001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Benoit Delbosc
018 */
019package org.nuxeo.importer.stream.automation;
020
021import static org.nuxeo.importer.stream.automation.RandomBlobProducers.DEFAULT_BLOB_LOG_NAME;
022
023import java.time.Duration;
024import java.util.concurrent.TimeUnit;
025
026import org.apache.commons.logging.Log;
027import org.apache.commons.logging.LogFactory;
028import org.nuxeo.ecm.automation.OperationContext;
029import org.nuxeo.ecm.automation.core.Constants;
030import org.nuxeo.ecm.automation.core.annotations.Context;
031import org.nuxeo.ecm.automation.core.annotations.Operation;
032import org.nuxeo.ecm.automation.core.annotations.OperationMethod;
033import org.nuxeo.ecm.automation.core.annotations.Param;
034import org.nuxeo.importer.stream.consumer.BlobInfoWriter;
035import org.nuxeo.importer.stream.consumer.BlobMessageConsumerFactory;
036import org.nuxeo.importer.stream.consumer.LogBlobInfoWriter;
037import org.nuxeo.importer.stream.message.BlobMessage;
038import org.nuxeo.lib.stream.log.LogManager;
039import org.nuxeo.lib.stream.pattern.consumer.BatchPolicy;
040import org.nuxeo.lib.stream.pattern.consumer.ConsumerPolicy;
041import org.nuxeo.lib.stream.pattern.consumer.ConsumerPool;
042import org.nuxeo.runtime.api.Framework;
043import org.nuxeo.runtime.stream.StreamService;
044
045import net.jodah.failsafe.RetryPolicy;
046
047/**
048 * @since 9.1
049 */
050@Operation(id = BlobConsumers.ID, category = Constants.CAT_SERVICES, label = "Import blobs", since = "9.1", description = "Import blob into the binarystore.")
051public class BlobConsumers {
052    private static final Log log = LogFactory.getLog(BlobConsumers.class);
053
054    public static final String ID = "StreamImporter.runBlobConsumers";
055
056    public static final String DEFAULT_LOG_BLOB_INFO_NAME = "import-blob-info";
057
058    public static final String DEFAULT_LOG_CONFIG = "default";
059
060    @Context
061    protected OperationContext ctx;
062
063    @Param(name = "nbThreads", required = false)
064    protected Integer nbThreads;
065
066    @Param(name = "blobProviderName", required = false)
067    protected String blobProviderName = "default";
068
069    @Param(name = "batchSize", required = false)
070    protected Integer batchSize = 10;
071
072    @Param(name = "batchThresholdS", required = false)
073    protected Integer batchThresholdS = 20;
074
075    @Param(name = "retryMax", required = false)
076    protected Integer retryMax = 3;
077
078    @Param(name = "retryDelayS", required = false)
079    protected Integer retryDelayS = 2;
080
081    @Param(name = "logName", required = false)
082    protected String logName;
083
084    @Param(name = "logBlobInfo", required = false)
085    protected String logBlobInfoName;
086
087    @Param(name = "logConfig", required = false)
088    protected String logConfig;
089
090    @Param(name = "waitMessageTimeoutSeconds", required = false)
091    protected Integer waitMessageTimeoutSeconds = 20;
092
093    @OperationMethod
094    public void run() {
095        RandomBlobProducers.checkAccess(ctx);
096        ConsumerPolicy consumerPolicy = ConsumerPolicy.builder()
097                                                      .name(ID)
098                                                      // we set the batch policy but batch is not used by the blob
099                                                      // consumer
100                                                      .batchPolicy(
101                                                              BatchPolicy.builder()
102                                                                         .capacity(batchSize)
103                                                                         .timeThreshold(
104                                                                                 Duration.ofSeconds(batchThresholdS))
105                                                                         .build())
106                                                      .retryPolicy(new RetryPolicy().withMaxRetries(retryMax).withDelay(
107                                                              retryDelayS, TimeUnit.SECONDS))
108                                                      .maxThreads(getNbThreads())
109                                                      .waitMessageTimeout(Duration.ofSeconds(waitMessageTimeoutSeconds))
110                                                      .build();
111        StreamService service = Framework.getService(StreamService.class);
112        LogManager manager = service.getLogManager(getLogConfig());
113        try (BlobInfoWriter blobInfoWriter = getBlobInfoWriter(manager)) {
114            ConsumerPool<BlobMessage> consumers = new ConsumerPool<>(getLogName(), manager,
115                    new BlobMessageConsumerFactory(blobProviderName, blobInfoWriter), consumerPolicy);
116            consumers.start().get();
117        } catch (Exception e) {
118            log.error(e.getMessage(), e);
119        }
120    }
121
122    protected BlobInfoWriter getBlobInfoWriter(LogManager managerBlobInfo) {
123        initBlobInfoMQ(managerBlobInfo);
124        return new LogBlobInfoWriter(managerBlobInfo.getAppender(getLogBlobInfoName()));
125    }
126
127    protected void initBlobInfoMQ(LogManager manager) {
128        manager.createIfNotExists(getLogBlobInfoName(), 1);
129    }
130
131    protected short getNbThreads() {
132        if (nbThreads != null) {
133            return nbThreads.shortValue();
134        }
135        return 0;
136    }
137
138    protected String getLogName() {
139        if (logName != null) {
140            return logName;
141        }
142        return DEFAULT_BLOB_LOG_NAME;
143    }
144
145    protected String getLogBlobInfoName() {
146        if (logBlobInfoName != null) {
147            return logBlobInfoName;
148        }
149        return DEFAULT_LOG_BLOB_INFO_NAME;
150    }
151
152    protected String getLogConfig() {
153        if (logConfig != null) {
154            return logConfig;
155        }
156        return DEFAULT_LOG_CONFIG;
157    }
158}