001/* 002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Benoit Delbosc 018 */ 019package org.nuxeo.ecm.platform.importer.mqueues.automation; 020 021import net.jodah.failsafe.RetryPolicy; 022import org.apache.commons.logging.Log; 023import org.apache.commons.logging.LogFactory; 024import org.nuxeo.ecm.automation.OperationContext; 025import org.nuxeo.ecm.automation.core.Constants; 026import org.nuxeo.ecm.automation.core.annotations.Context; 027import org.nuxeo.ecm.automation.core.annotations.Operation; 028import org.nuxeo.ecm.automation.core.annotations.OperationMethod; 029import org.nuxeo.ecm.automation.core.annotations.Param; 030import org.nuxeo.ecm.platform.importer.mqueues.chronicle.ChronicleConfig; 031import org.nuxeo.ecm.platform.importer.mqueues.kafka.KafkaConfigService; 032import org.nuxeo.ecm.platform.importer.mqueues.mqueues.MQManager; 033import org.nuxeo.ecm.platform.importer.mqueues.mqueues.chronicle.ChronicleMQManager; 034import org.nuxeo.ecm.platform.importer.mqueues.mqueues.kafka.KafkaMQManager; 035import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.BatchPolicy; 036import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.ConsumerPolicy; 037import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.DocumentConsumerPolicy; 038import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.DocumentConsumerPool; 039import org.nuxeo.ecm.platform.importer.mqueues.pattern.consumer.DocumentMessageConsumerFactory; 040import org.nuxeo.ecm.platform.importer.mqueues.pattern.message.DocumentMessage; 041import org.nuxeo.runtime.api.Framework; 042 043import java.time.Duration; 044import java.util.concurrent.TimeUnit; 045 046/** 047 * @since 9.1 048 */ 049@Operation(id = DocumentConsumers.ID, category = Constants.CAT_SERVICES, label = "Imports document", since = "9.1", 050 description = "Import mqueues document into repository.") 051public class DocumentConsumers { 052 private static final Log log = LogFactory.getLog(DocumentConsumers.class); 053 public static final String ID = "MQImporter.runDocumentConsumers"; 054 055 @Context 056 protected OperationContext ctx; 057 058 @Param(name = "nbThreads", required = false) 059 protected Integer nbThreads; 060 061 @Param(name = "rootFolder") 062 protected String rootFolder; 063 064 @Param(name = "repositoryName", required = false) 065 protected String repositoryName; 066 067 @Param(name = "batchSize", required = false) 068 protected Integer batchSize = 10; 069 070 @Param(name = "batchThresholdS", required = false) 071 protected Integer batchThresholdS = 20; 072 073 @Param(name = "retryMax", required = false) 074 protected Integer retryMax = 3; 075 076 @Param(name = "retryDelayS", required = false) 077 protected Integer retryDelayS = 2; 078 079 @Param(name = "mqName", required = false) 080 protected String mqName; 081 082 @Param(name = "kafkaConfig", required = false) 083 protected String kafkaConfig; 084 085 @Param(name = "blockIndexing", required = false) 086 protected Boolean blockIndexing = false; 087 088 @Param(name = "blockAsyncListeners", required = false) 089 protected Boolean blockAsyncListeners = false; 090 091 @Param(name = "blockPostCommitListeners", required = false) 092 protected Boolean blockPostCommitListeners = false; 093 094 @Param(name = "blockDefaultSyncListeners", required = false) 095 protected Boolean blockSyncListeners = false; 096 097 @Param(name = "useBulkMode", required = false) 098 protected Boolean useBulkMode = false; 099 100 101 @OperationMethod 102 public void run() { 103 RandomBlobProducers.checkAccess(ctx); 104 repositoryName = getRepositoryName(); 105 ConsumerPolicy consumerPolicy = DocumentConsumerPolicy.builder() 106 .blockIndexing(blockIndexing) 107 .blockAsyncListeners(blockAsyncListeners) 108 .blockPostCommitListeners(blockPostCommitListeners) 109 .blockDefaultSyncListener(blockSyncListeners) 110 .useBulkMode(useBulkMode) 111 .name(ID) 112 .batchPolicy(BatchPolicy.builder().capacity(batchSize) 113 .timeThreshold(Duration.ofSeconds(batchThresholdS)) 114 .build()) 115 .retryPolicy(new RetryPolicy().withMaxRetries(retryMax).withDelay(retryDelayS, TimeUnit.SECONDS)) 116 .maxThreads(getNbThreads()) 117 .salted() 118 .build(); 119 log.warn(String.format("Import documents from mqueue: %s into: %s/%s, with policy: %s", 120 getMQName(), repositoryName, rootFolder, (DocumentConsumerPolicy) consumerPolicy)); 121 try (MQManager<DocumentMessage> manager = getManager(); 122 DocumentConsumerPool<DocumentMessage> consumers = new DocumentConsumerPool<>(getMQName(), manager, 123 new DocumentMessageConsumerFactory(repositoryName, rootFolder), 124 consumerPolicy)) { 125 consumers.start().get(); 126 } catch (Exception e) { 127 log.error(e.getMessage(), e); 128 } 129 } 130 131 private short getNbThreads() { 132 if (nbThreads != null) { 133 return nbThreads.shortValue(); 134 } 135 return 0; 136 } 137 138 protected String getRepositoryName() { 139 if (repositoryName != null && !repositoryName.isEmpty()) { 140 return repositoryName; 141 } 142 return ctx.getCoreSession().getRepositoryName(); 143 } 144 145 protected String getMQName() { 146 if (mqName != null) { 147 return mqName; 148 } 149 return RandomDocumentProducers.DEFAULT_MQ_NAME; 150 } 151 152 protected MQManager<DocumentMessage> getManager() { 153 if (kafkaConfig == null || kafkaConfig.isEmpty()) { 154 return new ChronicleMQManager<>(ChronicleConfig.getBasePath("import"), 155 ChronicleConfig.getRetentionDuration()); 156 } 157 KafkaConfigService service = Framework.getService(KafkaConfigService.class); 158 return new KafkaMQManager<>(service.getZkServers(kafkaConfig), 159 service.getTopicPrefix(kafkaConfig), 160 service.getProducerProperties(kafkaConfig), 161 service.getConsumerProperties(kafkaConfig)); 162 } 163}