001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Benoit Delbosc
018 */
019package org.nuxeo.importer.stream.automation;
020
021import static org.nuxeo.importer.stream.StreamImporters.DEFAULT_LOG_BLOB_INFO_NAME;
022import static org.nuxeo.importer.stream.StreamImporters.DEFAULT_LOG_BLOB_NAME;
023import static org.nuxeo.importer.stream.StreamImporters.DEFAULT_LOG_CONFIG;
024
025import java.time.Duration;
026import java.util.concurrent.ExecutionException;
027import java.util.concurrent.TimeUnit;
028
029import org.apache.commons.logging.Log;
030import org.apache.commons.logging.LogFactory;
031import org.nuxeo.ecm.automation.OperationContext;
032import org.nuxeo.ecm.automation.OperationException;
033import org.nuxeo.ecm.automation.core.Constants;
034import org.nuxeo.ecm.automation.core.annotations.Context;
035import org.nuxeo.ecm.automation.core.annotations.Operation;
036import org.nuxeo.ecm.automation.core.annotations.OperationMethod;
037import org.nuxeo.ecm.automation.core.annotations.Param;
038import org.nuxeo.importer.stream.StreamImporters;
039import org.nuxeo.importer.stream.consumer.BlobInfoWriter;
040import org.nuxeo.importer.stream.consumer.BlobMessageConsumerFactory;
041import org.nuxeo.importer.stream.consumer.LogBlobInfoWriter;
042import org.nuxeo.importer.stream.message.BlobInfoMessage;
043import org.nuxeo.importer.stream.message.BlobMessage;
044import org.nuxeo.lib.stream.codec.Codec;
045import org.nuxeo.lib.stream.log.LogManager;
046import org.nuxeo.lib.stream.log.Name;
047import org.nuxeo.lib.stream.pattern.consumer.BatchPolicy;
048import org.nuxeo.lib.stream.pattern.consumer.ConsumerPolicy;
049import org.nuxeo.lib.stream.pattern.consumer.ConsumerPool;
050import org.nuxeo.runtime.api.Framework;
051import org.nuxeo.runtime.stream.StreamService;
052
053import net.jodah.failsafe.RetryPolicy;
054
055/**
056 * @since 9.1
057 */
058@Operation(id = BlobConsumers.ID, category = Constants.CAT_SERVICES, label = "Import blobs", since = "9.1", description = "Import blob into the binarystore.")
059public class BlobConsumers {
060    private static final Log log = LogFactory.getLog(BlobConsumers.class);
061
062    public static final String ID = "StreamImporter.runBlobConsumers";
063
064    @Context
065    protected OperationContext ctx;
066
067    @Param(name = "nbThreads", required = false)
068    protected Integer nbThreads;
069
070    @Param(name = "blobProviderName", required = false)
071    protected String blobProviderName = "default";
072
073    @Param(name = "batchSize", required = false)
074    protected Integer batchSize = 10;
075
076    @Param(name = "batchThresholdS", required = false)
077    protected Integer batchThresholdS = 5;
078
079    @Param(name = "retryMax", required = false)
080    protected Integer retryMax = 3;
081
082    @Param(name = "retryDelayS", required = false)
083    protected Integer retryDelayS = 2;
084
085    @Param(name = "logName", required = false)
086    protected String logName = DEFAULT_LOG_BLOB_NAME;
087
088    @Param(name = "logBlobInfo", required = false)
089    protected String logBlobInfoName = DEFAULT_LOG_BLOB_INFO_NAME;
090
091    @Param(name = "waitMessageTimeoutSeconds", required = false)
092    protected Integer waitMessageTimeoutSeconds = 20;
093
094    @Param(name = "watermark", required = false)
095    protected String watermark;
096
097    @Param(name = "persistBlobPath", required = false)
098    protected String persistBlobPath;
099
100    @OperationMethod
101    public void run() throws OperationException {
102        RandomBlobProducers.checkAccess(ctx);
103        ConsumerPolicy consumerPolicy = ConsumerPolicy.builder()
104                                                      .name(ID)
105                                                      // we set the batch policy but batch is not used by the blob
106                                                      // consumer
107                                                      .batchPolicy(
108                                                              BatchPolicy.builder()
109                                                                         .capacity(batchSize)
110                                                                         .timeThreshold(
111                                                                                 Duration.ofSeconds(batchThresholdS))
112                                                                         .build())
113                                                      .retryPolicy(new RetryPolicy().withMaxRetries(retryMax)
114                                                                                    .withDelay(retryDelayS,
115                                                                                            TimeUnit.SECONDS))
116                                                      .maxThreads(getNbThreads())
117                                                      .waitMessageTimeout(Duration.ofSeconds(waitMessageTimeoutSeconds))
118                                                      .build();
119        LogManager manager = Framework.getService(StreamService.class).getLogManager();
120        Codec<BlobMessage> codec = StreamImporters.getBlobCodec();
121        try (BlobInfoWriter blobInfoWriter = getBlobInfoWriter(manager)) {
122            ConsumerPool<BlobMessage> consumers = new ConsumerPool<>(logName, manager, codec,
123                    new BlobMessageConsumerFactory(blobProviderName, blobInfoWriter, watermark, persistBlobPath),
124                    consumerPolicy);
125            consumers.start().get();
126        } catch (InterruptedException e) {
127            Thread.currentThread().interrupt();
128            log.warn("Operation interrupted");
129            throw new RuntimeException(e);
130        } catch (ExecutionException e) {
131            log.error("Operation fails", e);
132            throw new OperationException(e);
133        }
134    }
135
136    protected BlobInfoWriter getBlobInfoWriter(LogManager managerBlobInfo) {
137        initBlobInfoMQ(managerBlobInfo);
138        Codec<BlobInfoMessage> blobInfoCodec = StreamImporters.getBlobInfoCodec();
139        return new LogBlobInfoWriter(managerBlobInfo.getAppender(Name.ofUrn(logBlobInfoName), blobInfoCodec));
140    }
141
142    protected void initBlobInfoMQ(LogManager manager) {
143        manager.createIfNotExists(Name.ofUrn(logBlobInfoName), 1);
144    }
145
146    protected short getNbThreads() {
147        if (nbThreads != null) {
148            return nbThreads.shortValue();
149        }
150        return 0;
151    }
152}