001/* 002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Benoit Delbosc 018 */ 019package org.nuxeo.importer.stream.automation; 020 021import static org.nuxeo.importer.stream.StreamImporters.DEFAULT_LOG_BLOB_INFO_NAME; 022import static org.nuxeo.importer.stream.StreamImporters.DEFAULT_LOG_BLOB_NAME; 023import static org.nuxeo.importer.stream.StreamImporters.DEFAULT_LOG_CONFIG; 024 025import java.time.Duration; 026import java.util.concurrent.ExecutionException; 027import java.util.concurrent.TimeUnit; 028 029import org.apache.commons.logging.Log; 030import org.apache.commons.logging.LogFactory; 031import org.nuxeo.ecm.automation.OperationContext; 032import org.nuxeo.ecm.automation.OperationException; 033import org.nuxeo.ecm.automation.core.Constants; 034import org.nuxeo.ecm.automation.core.annotations.Context; 035import org.nuxeo.ecm.automation.core.annotations.Operation; 036import org.nuxeo.ecm.automation.core.annotations.OperationMethod; 037import org.nuxeo.ecm.automation.core.annotations.Param; 038import org.nuxeo.importer.stream.StreamImporters; 039import org.nuxeo.importer.stream.consumer.BlobInfoWriter; 040import org.nuxeo.importer.stream.consumer.BlobMessageConsumerFactory; 041import org.nuxeo.importer.stream.consumer.LogBlobInfoWriter; 042import org.nuxeo.importer.stream.message.BlobInfoMessage; 043import org.nuxeo.importer.stream.message.BlobMessage; 044import org.nuxeo.lib.stream.codec.Codec; 045import org.nuxeo.lib.stream.log.LogManager; 046import org.nuxeo.lib.stream.log.Name; 047import org.nuxeo.lib.stream.pattern.consumer.BatchPolicy; 048import org.nuxeo.lib.stream.pattern.consumer.ConsumerPolicy; 049import org.nuxeo.lib.stream.pattern.consumer.ConsumerPool; 050import org.nuxeo.runtime.api.Framework; 051import org.nuxeo.runtime.stream.StreamService; 052 053import net.jodah.failsafe.RetryPolicy; 054 055/** 056 * @since 9.1 057 */ 058@Operation(id = BlobConsumers.ID, category = Constants.CAT_SERVICES, label = "Import blobs", since = "9.1", description = "Import blob into the binarystore.") 059public class BlobConsumers { 060 private static final Log log = LogFactory.getLog(BlobConsumers.class); 061 062 public static final String ID = "StreamImporter.runBlobConsumers"; 063 064 @Context 065 protected OperationContext ctx; 066 067 @Param(name = "nbThreads", required = false) 068 protected Integer nbThreads; 069 070 @Param(name = "blobProviderName", required = false) 071 protected String blobProviderName = "default"; 072 073 @Param(name = "batchSize", required = false) 074 protected Integer batchSize = 10; 075 076 @Param(name = "batchThresholdS", required = false) 077 protected Integer batchThresholdS = 5; 078 079 @Param(name = "retryMax", required = false) 080 protected Integer retryMax = 3; 081 082 @Param(name = "retryDelayS", required = false) 083 protected Integer retryDelayS = 2; 084 085 @Param(name = "logName", required = false) 086 protected String logName = DEFAULT_LOG_BLOB_NAME; 087 088 @Param(name = "logBlobInfo", required = false) 089 protected String logBlobInfoName = DEFAULT_LOG_BLOB_INFO_NAME; 090 091 @Param(name = "waitMessageTimeoutSeconds", required = false) 092 protected Integer waitMessageTimeoutSeconds = 20; 093 094 @Param(name = "watermark", required = false) 095 protected String watermark; 096 097 @Param(name = "persistBlobPath", required = false) 098 protected String persistBlobPath; 099 100 @OperationMethod 101 public void run() throws OperationException { 102 RandomBlobProducers.checkAccess(ctx); 103 ConsumerPolicy consumerPolicy = ConsumerPolicy.builder() 104 .name(ID) 105 // we set the batch policy but batch is not used by the blob 106 // consumer 107 .batchPolicy( 108 BatchPolicy.builder() 109 .capacity(batchSize) 110 .timeThreshold( 111 Duration.ofSeconds(batchThresholdS)) 112 .build()) 113 .retryPolicy(new RetryPolicy().withMaxRetries(retryMax) 114 .withDelay(retryDelayS, 115 TimeUnit.SECONDS)) 116 .maxThreads(getNbThreads()) 117 .waitMessageTimeout(Duration.ofSeconds(waitMessageTimeoutSeconds)) 118 .build(); 119 LogManager manager = Framework.getService(StreamService.class).getLogManager(); 120 Codec<BlobMessage> codec = StreamImporters.getBlobCodec(); 121 try (BlobInfoWriter blobInfoWriter = getBlobInfoWriter(manager)) { 122 ConsumerPool<BlobMessage> consumers = new ConsumerPool<>(logName, manager, codec, 123 new BlobMessageConsumerFactory(blobProviderName, blobInfoWriter, watermark, persistBlobPath), 124 consumerPolicy); 125 consumers.start().get(); 126 } catch (InterruptedException e) { 127 Thread.currentThread().interrupt(); 128 log.warn("Operation interrupted"); 129 throw new RuntimeException(e); 130 } catch (ExecutionException e) { 131 log.error("Operation fails", e); 132 throw new OperationException(e); 133 } 134 } 135 136 protected BlobInfoWriter getBlobInfoWriter(LogManager managerBlobInfo) { 137 initBlobInfoMQ(managerBlobInfo); 138 Codec<BlobInfoMessage> blobInfoCodec = StreamImporters.getBlobInfoCodec(); 139 return new LogBlobInfoWriter(managerBlobInfo.getAppender(Name.ofUrn(logBlobInfoName), blobInfoCodec)); 140 } 141 142 protected void initBlobInfoMQ(LogManager manager) { 143 manager.createIfNotExists(Name.ofUrn(logBlobInfoName), 1); 144 } 145 146 protected short getNbThreads() { 147 if (nbThreads != null) { 148 return nbThreads.shortValue(); 149 } 150 return 0; 151 } 152}