001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Benoit Delbosc
018 */
019package org.nuxeo.ecm.platform.importer.mqueues.automation;
020
021import net.jodah.failsafe.RetryPolicy;
022import org.apache.commons.logging.Log;
023import org.apache.commons.logging.LogFactory;
024import org.nuxeo.ecm.automation.OperationContext;
025import org.nuxeo.ecm.automation.core.Constants;
026import org.nuxeo.ecm.automation.core.annotations.Context;
027import org.nuxeo.ecm.automation.core.annotations.Operation;
028import org.nuxeo.ecm.automation.core.annotations.OperationMethod;
029import org.nuxeo.ecm.automation.core.annotations.Param;
030import org.nuxeo.ecm.platform.importer.mqueues.chronicle.ChronicleConfig;
031import org.nuxeo.ecm.platform.importer.mqueues.kafka.KafkaConfigService;
032import org.nuxeo.ecm.platform.importer.mqueues.mqueues.MQManager;
033import org.nuxeo.ecm.platform.importer.mqueues.mqueues.chronicle.ChronicleMQManager;
034import org.nuxeo.ecm.platform.importer.mqueues.mqueues.kafka.KafkaMQManager;
035import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.BatchPolicy;
036import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.ConsumerPolicy;
037import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.DocumentConsumerPolicy;
038import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.DocumentConsumerPool;
039import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.DocumentMessageConsumerFactory;
040import org.nuxeo.ecm.platform.importer.mqueues.pattern.message.DocumentMessage;
041import org.nuxeo.runtime.api.Framework;
042
043import java.time.Duration;
044import java.util.concurrent.TimeUnit;
045
046/**
047 * @since 9.1
048 */
049@Operation(id = DocumentConsumers.ID, category = Constants.CAT_SERVICES, label = "Imports document", since = "9.1",
050        description = "Import mqueues document into repository.")
051public class DocumentConsumers {
052    private static final Log log = LogFactory.getLog(DocumentConsumers.class);
053    public static final String ID = "MQImporter.runDocumentConsumers";
054
055    @Context
056    protected OperationContext ctx;
057
058    @Param(name = "nbThreads", required = false)
059    protected Integer nbThreads;
060
061    @Param(name = "rootFolder")
062    protected String rootFolder;
063
064    @Param(name = "repositoryName", required = false)
065    protected String repositoryName;
066
067    @Param(name = "batchSize", required = false)
068    protected Integer batchSize = 10;
069
070    @Param(name = "batchThresholdS", required = false)
071    protected Integer batchThresholdS = 20;
072
073    @Param(name = "retryMax", required = false)
074    protected Integer retryMax = 3;
075
076    @Param(name = "retryDelayS", required = false)
077    protected Integer retryDelayS = 2;
078
079    @Param(name = "mqName", required = false)
080    protected String mqName;
081
082    @Param(name = "kafkaConfig", required = false)
083    protected String kafkaConfig;
084
085    @Param(name = "blockIndexing", required = false)
086    protected Boolean blockIndexing = false;
087
088    @Param(name = "blockAsyncListeners", required = false)
089    protected Boolean blockAsyncListeners = false;
090
091    @Param(name = "blockPostCommitListeners", required = false)
092    protected Boolean blockPostCommitListeners = false;
093
094    @Param(name = "blockDefaultSyncListeners", required = false)
095    protected Boolean blockSyncListeners = false;
096
097    @Param(name = "useBulkMode", required = false)
098    protected Boolean useBulkMode = false;
099
100
101    @OperationMethod
102    public void run() {
103        RandomBlobProducers.checkAccess(ctx);
104        repositoryName = getRepositoryName();
105        ConsumerPolicy consumerPolicy = DocumentConsumerPolicy.builder()
106                .blockIndexing(blockIndexing)
107                .blockAsyncListeners(blockAsyncListeners)
108                .blockPostCommitListeners(blockPostCommitListeners)
109                .blockDefaultSyncListener(blockSyncListeners)
110                .useBulkMode(useBulkMode)
111                .name(ID)
112                .batchPolicy(BatchPolicy.builder().capacity(batchSize)
113                        .timeThreshold(Duration.ofSeconds(batchThresholdS))
114                        .build())
115                .retryPolicy(new RetryPolicy().withMaxRetries(retryMax).withDelay(retryDelayS, TimeUnit.SECONDS))
116                .maxThreads(getNbThreads())
117                .salted()
118                .build();
119        log.warn(String.format("Import documents from mqueue: %s into: %s/%s, with policy: %s",
120                getMQName(), repositoryName, rootFolder, (DocumentConsumerPolicy) consumerPolicy));
121        try (MQManager<DocumentMessage> manager = getManager();
122             DocumentConsumerPool<DocumentMessage> consumers = new DocumentConsumerPool<>(getMQName(), manager,
123                     new DocumentMessageConsumerFactory(repositoryName, rootFolder),
124                     consumerPolicy)) {
125            consumers.start().get();
126        } catch (Exception e) {
127            log.error(e.getMessage(), e);
128        }
129    }
130
131    private short getNbThreads() {
132        if (nbThreads != null) {
133            return nbThreads.shortValue();
134        }
135        return 0;
136    }
137
138    protected String getRepositoryName() {
139        if (repositoryName != null && !repositoryName.isEmpty()) {
140            return repositoryName;
141        }
142        return ctx.getCoreSession().getRepositoryName();
143    }
144
145    protected String getMQName() {
146        if (mqName != null) {
147            return mqName;
148        }
149        return RandomDocumentProducers.DEFAULT_MQ_NAME;
150    }
151
152    protected MQManager<DocumentMessage> getManager() {
153        if (kafkaConfig == null || kafkaConfig.isEmpty()) {
154            return new ChronicleMQManager<>(ChronicleConfig.getBasePath("import"),
155                    ChronicleConfig.getRetentionDuration());
156        }
157        KafkaConfigService service = Framework.getService(KafkaConfigService.class);
158        return new KafkaMQManager<>(service.getZkServers(kafkaConfig),
159                service.getTopicPrefix(kafkaConfig),
160                service.getProducerProperties(kafkaConfig),
161                service.getConsumerProperties(kafkaConfig));
162    }
163}