001/* 002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Benoit Delbosc 018 */ 019package org.nuxeo.importer.stream.automation; 020 021import static org.nuxeo.importer.stream.automation.RandomBlobProducers.DEFAULT_BLOB_LOG_NAME; 022 023import java.time.Duration; 024import java.util.concurrent.TimeUnit; 025 026import org.apache.commons.logging.Log; 027import org.apache.commons.logging.LogFactory; 028import org.nuxeo.ecm.automation.OperationContext; 029import org.nuxeo.ecm.automation.core.Constants; 030import org.nuxeo.ecm.automation.core.annotations.Context; 031import org.nuxeo.ecm.automation.core.annotations.Operation; 032import org.nuxeo.ecm.automation.core.annotations.OperationMethod; 033import org.nuxeo.ecm.automation.core.annotations.Param; 034import org.nuxeo.importer.stream.consumer.BlobInfoWriter; 035import org.nuxeo.importer.stream.consumer.BlobMessageConsumerFactory; 036import org.nuxeo.importer.stream.consumer.LogBlobInfoWriter; 037import org.nuxeo.importer.stream.message.BlobMessage; 038import org.nuxeo.lib.stream.log.LogManager; 039import org.nuxeo.lib.stream.pattern.consumer.BatchPolicy; 040import org.nuxeo.lib.stream.pattern.consumer.ConsumerPolicy; 041import org.nuxeo.lib.stream.pattern.consumer.ConsumerPool; 042import org.nuxeo.runtime.api.Framework; 043import org.nuxeo.runtime.stream.StreamService; 044 045import net.jodah.failsafe.RetryPolicy; 046 047/** 048 * @since 9.1 049 */ 050@Operation(id = BlobConsumers.ID, category = Constants.CAT_SERVICES, label = "Import blobs", since = "9.1", description = "Import blob into the binarystore.") 051public class BlobConsumers { 052 private static final Log log = LogFactory.getLog(BlobConsumers.class); 053 054 public static final String ID = "StreamImporter.runBlobConsumers"; 055 056 public static final String DEFAULT_LOG_BLOB_INFO_NAME = "import-blob-info"; 057 058 public static final String DEFAULT_LOG_CONFIG = "default"; 059 060 @Context 061 protected OperationContext ctx; 062 063 @Param(name = "nbThreads", required = false) 064 protected Integer nbThreads; 065 066 @Param(name = "blobProviderName", required = false) 067 protected String blobProviderName = "default"; 068 069 @Param(name = "batchSize", required = false) 070 protected Integer batchSize = 10; 071 072 @Param(name = "batchThresholdS", required = false) 073 protected Integer batchThresholdS = 20; 074 075 @Param(name = "retryMax", required = false) 076 protected Integer retryMax = 3; 077 078 @Param(name = "retryDelayS", required = false) 079 protected Integer retryDelayS = 2; 080 081 @Param(name = "logName", required = false) 082 protected String logName; 083 084 @Param(name = "logBlobInfo", required = false) 085 protected String logBlobInfoName; 086 087 @Param(name = "logConfig", required = false) 088 protected String logConfig; 089 090 @Param(name = "waitMessageTimeoutSeconds", required = false) 091 protected Integer waitMessageTimeoutSeconds = 20; 092 093 @OperationMethod 094 public void run() { 095 RandomBlobProducers.checkAccess(ctx); 096 ConsumerPolicy consumerPolicy = ConsumerPolicy.builder() 097 .name(ID) 098 // we set the batch policy but batch is not used by the blob 099 // consumer 100 .batchPolicy( 101 BatchPolicy.builder() 102 .capacity(batchSize) 103 .timeThreshold( 104 Duration.ofSeconds(batchThresholdS)) 105 .build()) 106 .retryPolicy(new RetryPolicy().withMaxRetries(retryMax).withDelay( 107 retryDelayS, TimeUnit.SECONDS)) 108 .maxThreads(getNbThreads()) 109 .waitMessageTimeout(Duration.ofSeconds(waitMessageTimeoutSeconds)) 110 .build(); 111 StreamService service = Framework.getService(StreamService.class); 112 LogManager manager = service.getLogManager(getLogConfig()); 113 try (BlobInfoWriter blobInfoWriter = getBlobInfoWriter(manager)) { 114 ConsumerPool<BlobMessage> consumers = new ConsumerPool<>(getLogName(), manager, 115 new BlobMessageConsumerFactory(blobProviderName, blobInfoWriter), consumerPolicy); 116 consumers.start().get(); 117 } catch (Exception e) { 118 log.error(e.getMessage(), e); 119 } 120 } 121 122 protected BlobInfoWriter getBlobInfoWriter(LogManager managerBlobInfo) { 123 initBlobInfoMQ(managerBlobInfo); 124 return new LogBlobInfoWriter(managerBlobInfo.getAppender(getLogBlobInfoName())); 125 } 126 127 protected void initBlobInfoMQ(LogManager manager) { 128 manager.createIfNotExists(getLogBlobInfoName(), 1); 129 } 130 131 protected short getNbThreads() { 132 if (nbThreads != null) { 133 return nbThreads.shortValue(); 134 } 135 return 0; 136 } 137 138 protected String getLogName() { 139 if (logName != null) { 140 return logName; 141 } 142 return DEFAULT_BLOB_LOG_NAME; 143 } 144 145 protected String getLogBlobInfoName() { 146 if (logBlobInfoName != null) { 147 return logBlobInfoName; 148 } 149 return DEFAULT_LOG_BLOB_INFO_NAME; 150 } 151 152 protected String getLogConfig() { 153 if (logConfig != null) { 154 return logConfig; 155 } 156 return DEFAULT_LOG_CONFIG; 157 } 158}