001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Benoit Delbosc
018 */
019package org.nuxeo.importer.stream.automation;
020
021import static org.nuxeo.importer.stream.automation.RandomBlobProducers.DEFAULT_BLOB_LOG_NAME;
022
023import java.time.Duration;
024import java.util.concurrent.ExecutionException;
025import java.util.concurrent.TimeUnit;
026
027import org.apache.commons.logging.Log;
028import org.apache.commons.logging.LogFactory;
029import org.nuxeo.ecm.automation.OperationContext;
030import org.nuxeo.ecm.automation.OperationException;
031import org.nuxeo.ecm.automation.core.Constants;
032import org.nuxeo.ecm.automation.core.annotations.Context;
033import org.nuxeo.ecm.automation.core.annotations.Operation;
034import org.nuxeo.ecm.automation.core.annotations.OperationMethod;
035import org.nuxeo.ecm.automation.core.annotations.Param;
036import org.nuxeo.importer.stream.consumer.BlobInfoWriter;
037import org.nuxeo.importer.stream.consumer.BlobMessageConsumerFactory;
038import org.nuxeo.importer.stream.consumer.LogBlobInfoWriter;
039import org.nuxeo.importer.stream.message.BlobMessage;
040import org.nuxeo.lib.stream.log.LogManager;
041import org.nuxeo.lib.stream.pattern.consumer.BatchPolicy;
042import org.nuxeo.lib.stream.pattern.consumer.ConsumerPolicy;
043import org.nuxeo.lib.stream.pattern.consumer.ConsumerPool;
044import org.nuxeo.runtime.api.Framework;
045import org.nuxeo.runtime.stream.StreamService;
046
047import net.jodah.failsafe.RetryPolicy;
048
049/**
050 * @since 9.1
051 */
052@Operation(id = BlobConsumers.ID, category = Constants.CAT_SERVICES, label = "Import blobs", since = "9.1", description = "Import blob into the binarystore.")
053public class BlobConsumers {
054    private static final Log log = LogFactory.getLog(BlobConsumers.class);
055
056    public static final String ID = "StreamImporter.runBlobConsumers";
057
058    public static final String DEFAULT_LOG_BLOB_INFO_NAME = "import-blob-info";
059
060    public static final String DEFAULT_LOG_CONFIG = "default";
061
062    @Context
063    protected OperationContext ctx;
064
065    @Param(name = "nbThreads", required = false)
066    protected Integer nbThreads;
067
068    @Param(name = "blobProviderName", required = false)
069    protected String blobProviderName = "default";
070
071    @Param(name = "batchSize", required = false)
072    protected Integer batchSize = 10;
073
074    @Param(name = "batchThresholdS", required = false)
075    protected Integer batchThresholdS = 20;
076
077    @Param(name = "retryMax", required = false)
078    protected Integer retryMax = 3;
079
080    @Param(name = "retryDelayS", required = false)
081    protected Integer retryDelayS = 2;
082
083    @Param(name = "logName", required = false)
084    protected String logName;
085
086    @Param(name = "logBlobInfo", required = false)
087    protected String logBlobInfoName;
088
089    @Param(name = "logConfig", required = false)
090    protected String logConfig;
091
092    @Param(name = "waitMessageTimeoutSeconds", required = false)
093    protected Integer waitMessageTimeoutSeconds = 20;
094
095    @Param(name = "watermark", required = false)
096    protected String watermark;
097
098    @Param(name = "persistBlobPath", required = false)
099    protected String persistBlobPath;
100
101    @OperationMethod
102    public void run() throws OperationException {
103        RandomBlobProducers.checkAccess(ctx);
104        ConsumerPolicy consumerPolicy = ConsumerPolicy.builder()
105                                                      .name(ID)
106                                                      // we set the batch policy but batch is not used by the blob
107                                                      // consumer
108                                                      .batchPolicy(
109                                                              BatchPolicy.builder()
110                                                                         .capacity(batchSize)
111                                                                         .timeThreshold(
112                                                                                 Duration.ofSeconds(batchThresholdS))
113                                                                         .build())
114                                                      .retryPolicy(new RetryPolicy().withMaxRetries(retryMax).withDelay(
115                                                              retryDelayS, TimeUnit.SECONDS))
116                                                      .maxThreads(getNbThreads())
117                                                      .waitMessageTimeout(Duration.ofSeconds(waitMessageTimeoutSeconds))
118                                                      .build();
119        StreamService service = Framework.getService(StreamService.class);
120        LogManager manager = service.getLogManager(getLogConfig());
121        try (BlobInfoWriter blobInfoWriter = getBlobInfoWriter(manager)) {
122            ConsumerPool<BlobMessage> consumers = new ConsumerPool<>(getLogName(), manager,
123                    new BlobMessageConsumerFactory(blobProviderName, blobInfoWriter, watermark, persistBlobPath),
124                    consumerPolicy);
125            consumers.start().get();
126        } catch (InterruptedException e) {
127            Thread.currentThread().interrupt();
128            log.warn("Operation interrupted");
129            throw new RuntimeException(e);
130        } catch (ExecutionException e) {
131            log.error("Operation fails", e);
132            throw new OperationException(e);
133        }
134    }
135
136    protected BlobInfoWriter getBlobInfoWriter(LogManager managerBlobInfo) {
137        initBlobInfoMQ(managerBlobInfo);
138        return new LogBlobInfoWriter(managerBlobInfo.getAppender(getLogBlobInfoName()));
139    }
140
141    protected void initBlobInfoMQ(LogManager manager) {
142        manager.createIfNotExists(getLogBlobInfoName(), 1);
143    }
144
145    protected short getNbThreads() {
146        if (nbThreads != null) {
147            return nbThreads.shortValue();
148        }
149        return 0;
150    }
151
152    protected String getLogName() {
153        if (logName != null) {
154            return logName;
155        }
156        return DEFAULT_BLOB_LOG_NAME;
157    }
158
159    protected String getLogBlobInfoName() {
160        if (logBlobInfoName != null) {
161            return logBlobInfoName;
162        }
163        return DEFAULT_LOG_BLOB_INFO_NAME;
164    }
165
166    protected String getLogConfig() {
167        if (logConfig != null) {
168            return logConfig;
169        }
170        return DEFAULT_LOG_CONFIG;
171    }
172}