001/* 002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Tiry 018 * Florent Guillaume 019 * Estelle Giuly <egiuly@nuxeo.com> 020 */ 021package org.nuxeo.ecm.core.convert.service; 022 023import java.io.File; 024import java.io.IOException; 025import java.io.Serializable; 026import java.nio.file.Path; 027import java.util.ArrayList; 028import java.util.HashMap; 029import java.util.List; 030import java.util.Map; 031import java.util.function.Function; 032import java.util.regex.Matcher; 033import java.util.regex.Pattern; 034 035import javax.ws.rs.core.MediaType; 036 037import org.apache.commons.io.FilenameUtils; 038import org.apache.commons.lang.StringUtils; 039import org.apache.commons.logging.Log; 040import org.apache.commons.logging.LogFactory; 041import org.nuxeo.common.utils.FileUtils; 042import org.nuxeo.ecm.core.api.Blob; 043import org.nuxeo.ecm.core.api.blobholder.BlobHolder; 044import org.nuxeo.ecm.core.api.blobholder.SimpleBlobHolder; 045import org.nuxeo.ecm.core.api.impl.blob.StringBlob; 046import org.nuxeo.ecm.core.convert.api.ConversionException; 047import org.nuxeo.ecm.core.convert.api.ConversionService; 048import org.nuxeo.ecm.core.convert.api.ConversionStatus; 049import org.nuxeo.ecm.core.convert.api.ConverterCheckResult; 050import org.nuxeo.ecm.core.convert.api.ConverterNotAvailable; 051import org.nuxeo.ecm.core.convert.api.ConverterNotRegistered; 052import org.nuxeo.ecm.core.convert.cache.CacheKeyGenerator; 053import org.nuxeo.ecm.core.convert.cache.ConversionCacheHolder; 054import org.nuxeo.ecm.core.convert.cache.GCTask; 055import org.nuxeo.ecm.core.convert.extension.ChainedConverter; 056import org.nuxeo.ecm.core.convert.extension.Converter; 057import org.nuxeo.ecm.core.convert.extension.ConverterDescriptor; 058import org.nuxeo.ecm.core.convert.extension.ExternalConverter; 059import org.nuxeo.ecm.core.convert.extension.GlobalConfigDescriptor; 060import org.nuxeo.ecm.core.io.download.DownloadService; 061import org.nuxeo.ecm.core.transientstore.work.TransientStoreWork; 062import org.nuxeo.ecm.core.work.api.Work; 063import org.nuxeo.ecm.core.work.api.WorkManager; 064import org.nuxeo.ecm.platform.mimetype.interfaces.MimetypeEntry; 065import org.nuxeo.ecm.platform.mimetype.interfaces.MimetypeRegistry; 066import org.nuxeo.runtime.RuntimeServiceEvent; 067import org.nuxeo.runtime.RuntimeServiceListener; 068import org.nuxeo.runtime.api.Framework; 069import org.nuxeo.runtime.model.ComponentContext; 070import org.nuxeo.runtime.model.ComponentInstance; 071import org.nuxeo.runtime.model.DefaultComponent; 072import org.nuxeo.runtime.reload.ReloadService; 073import org.nuxeo.runtime.services.event.Event; 074import org.nuxeo.runtime.services.event.EventListener; 075import org.nuxeo.runtime.services.event.EventService; 076 077/** 078 * Runtime Component that also provides the POJO implementation of the {@link ConversionService}. 079 */ 080public class ConversionServiceImpl extends DefaultComponent implements ConversionService { 081 082 protected static final Log log = LogFactory.getLog(ConversionServiceImpl.class); 083 084 public static final String CONVERTER_EP = "converter"; 085 086 public static final String CONFIG_EP = "configuration"; 087 088 protected final Map<String, ConverterDescriptor> converterDescriptors = new HashMap<>(); 089 090 protected final MimeTypeTranslationHelper translationHelper = new MimeTypeTranslationHelper(); 091 092 protected final GlobalConfigDescriptor config = new GlobalConfigDescriptor(); 093 094 protected static ConversionServiceImpl self; 095 096 protected Thread gcThread; 097 098 protected GCTask gcTask; 099 100 ReloadListener reloadListener; 101 102 class ReloadListener implements EventListener { 103 104 @Override 105 public void handleEvent(Event event) { 106 if (ReloadService.AFTER_RELOAD_EVENT_ID.equals(event.getId())) { 107 startGC(); 108 } else if (ReloadService.BEFORE_RELOAD_EVENT_ID.equals(event.getId())) { 109 endGC(); 110 } 111 } 112 113 } 114 115 @Override 116 public void activate(ComponentContext context) { 117 converterDescriptors.clear(); 118 translationHelper.clear(); 119 self = this; 120 config.clearCachingDirectory(); 121 Framework.addListener(new RuntimeServiceListener() { 122 123 @Override 124 public void handleEvent(RuntimeServiceEvent event) { 125 if (RuntimeServiceEvent.RUNTIME_ABOUT_TO_STOP != event.id) { 126 return; 127 } 128 Framework.removeListener(this); 129 Framework.getService(EventService.class).removeListener(ReloadService.RELOAD_TOPIC, reloadListener); 130 endGC(); 131 } 132 }); 133 Framework.getService(EventService.class).addListener(ReloadService.RELOAD_TOPIC, 134 reloadListener = new ReloadListener()); 135 } 136 137 @Override 138 public void deactivate(ComponentContext context) { 139 if (config.isCacheEnabled()) { 140 ConversionCacheHolder.deleteCache(); 141 } 142 self = null; 143 converterDescriptors.clear(); 144 translationHelper.clear(); 145 } 146 147 /** 148 * Component implementation. 149 */ 150 @Override 151 public void registerContribution(Object contribution, String extensionPoint, ComponentInstance contributor) { 152 153 if (CONVERTER_EP.equals(extensionPoint)) { 154 ConverterDescriptor desc = (ConverterDescriptor) contribution; 155 registerConverter(desc); 156 } else if (CONFIG_EP.equals(extensionPoint)) { 157 GlobalConfigDescriptor desc = (GlobalConfigDescriptor) contribution; 158 config.update(desc); 159 config.clearCachingDirectory(); 160 } else { 161 log.error("Unable to handle unknown extensionPoint " + extensionPoint); 162 } 163 } 164 165 @Override 166 public void unregisterContribution(Object contribution, String extensionPoint, ComponentInstance contributor) { 167 } 168 169 /* Component API */ 170 171 public static Converter getConverter(String converterName) { 172 ConverterDescriptor desc = self.converterDescriptors.get(converterName); 173 if (desc == null) { 174 return null; 175 } 176 return desc.getConverterInstance(); 177 } 178 179 public static ConverterDescriptor getConverterDescriptor(String converterName) { 180 return self.converterDescriptors.get(converterName); 181 } 182 183 public static long getGCIntervalInMinutes() { 184 return self.config.getGCInterval(); 185 } 186 187 public static void setGCIntervalInMinutes(long interval) { 188 self.config.setGCInterval(interval); 189 } 190 191 public static void registerConverter(ConverterDescriptor desc) { 192 193 if (self.converterDescriptors.containsKey(desc.getConverterName())) { 194 195 ConverterDescriptor existing = self.converterDescriptors.get(desc.getConverterName()); 196 desc = existing.merge(desc); 197 } 198 desc.initConverter(); 199 self.translationHelper.addConverter(desc); 200 self.converterDescriptors.put(desc.getConverterName(), desc); 201 } 202 203 public static int getMaxCacheSizeInKB() { 204 return self.config.getDiskCacheSize(); 205 } 206 207 public static void setMaxCacheSizeInKB(int size) { 208 self.config.setDiskCacheSize(size); 209 } 210 211 public static boolean isCacheEnabled() { 212 return self.config.isCacheEnabled(); 213 } 214 215 public static String getCacheBasePath() { 216 return self.config.getCachingDirectory(); 217 } 218 219 /* Service API */ 220 221 @Override 222 public List<String> getRegistredConverters() { 223 List<String> converterNames = new ArrayList<>(); 224 converterNames.addAll(converterDescriptors.keySet()); 225 return converterNames; 226 } 227 228 @Override 229 public Blob convertBlobToPDF(Blob blob) throws IOException { 230 String mimetype = blob.getMimeType(); 231 String filename = blob.getFilename(); 232 if (MimetypeRegistry.PDF_MIMETYPE.equals(mimetype)) { 233 return blob; 234 } 235 Blob result; 236 if (MediaType.TEXT_PLAIN.equals(mimetype)) { 237 result = convertBlobToMimeType(blob, MimetypeRegistry.PDF_MIMETYPE); 238 } else { 239 // Convert the blob to HTML 240 if (!MediaType.TEXT_HTML.equals(mimetype)) { 241 blob = convertBlobToMimeType(blob, MediaType.TEXT_HTML); 242 blob.setFilename(filename); 243 } 244 Path tempDirectory = Framework.createTempDirectory("blobs"); 245 try { 246 // Replace the image URLs by absolute paths 247 DownloadService downloadService = Framework.getService(DownloadService.class); 248 blob = replaceURLsByAbsolutePaths(blob, tempDirectory, downloadService::resolveBlobFromDownloadUrl); 249 // Convert the blob to PDF 250 result = convertBlobToMimeType(blob, MimetypeRegistry.PDF_MIMETYPE); 251 } finally { 252 org.apache.commons.io.FileUtils.deleteQuietly(tempDirectory.toFile()); 253 } 254 } 255 if (result != null) { 256 adjustPDFBlobName(filename, result); 257 } 258 return result; 259 } 260 261 protected Blob convertBlobToMimeType(Blob blob, String destinationMimeType) { 262 BlobHolder bh = new SimpleBlobHolder(blob); 263 bh = convertToMimeType(destinationMimeType, bh, null); 264 return bh == null ? null : bh.getBlob(); 265 } 266 267 protected static void adjustPDFBlobName(String filename, Blob blob) { 268 if (StringUtils.isBlank(filename)) { 269 filename = "file_" + System.currentTimeMillis(); 270 } else { 271 filename = FilenameUtils.removeExtension(FilenameUtils.getName(filename)); 272 } 273 blob.setFilename(filename + MimetypeRegistry.PDF_EXTENSION); 274 blob.setMimeType(MimetypeRegistry.PDF_MIMETYPE); 275 } 276 277 /** 278 * Replace the image URLs of an HTML blob by absolute local paths. 279 * 280 * @throws IOException 281 * @since 9.1 282 */ 283 protected static Blob replaceURLsByAbsolutePaths(Blob blob, Path tempDirectory, Function<String, Blob> blobResolver) 284 throws IOException { 285 String initialBlobContent = blob.getString(); 286 // Find images links in the blob 287 Pattern pattern = Pattern.compile("(src=([\"']))(.*?)(\\2)"); 288 Matcher matcher = pattern.matcher(initialBlobContent); 289 StringBuffer sb = new StringBuffer(); 290 while (matcher.find()) { 291 // Retrieve the image from the URL 292 String url = matcher.group(3); 293 Blob imageBlob = blobResolver.apply(url); 294 if (imageBlob == null) { 295 break; 296 } 297 // Export the image to a temporary directory in File System 298 String safeFilename = FileUtils.getSafeFilename(imageBlob.getFilename()); 299 File imageFile = tempDirectory.resolve(safeFilename).toFile(); 300 imageBlob.transferTo(imageFile); 301 // Replace the image URL by its absolute local path 302 matcher.appendReplacement(sb, "$1" + Matcher.quoteReplacement(imageFile.toPath().toString()) + "$4"); 303 } 304 matcher.appendTail(sb); 305 String blobContentWithAbsolutePaths = sb.toString(); 306 if (blobContentWithAbsolutePaths.equals(initialBlobContent)) { 307 return blob; 308 } 309 // Create a new blob with the new content 310 Blob newBlob = new StringBlob(blobContentWithAbsolutePaths, blob.getMimeType(), blob.getEncoding()); 311 newBlob.setFilename(blob.getFilename()); 312 return newBlob; 313 } 314 315 @Override 316 public BlobHolder convert(String converterName, BlobHolder blobHolder, Map<String, Serializable> parameters) 317 throws ConversionException { 318 319 // set parameters if null to avoid NPE in converters 320 if (parameters == null) { 321 parameters = new HashMap<>(); 322 } 323 324 // exist if not registered 325 ConverterCheckResult check = isConverterAvailable(converterName); 326 if (!check.isAvailable()) { 327 // exist is not installed / configured 328 throw new ConverterNotAvailable(converterName); 329 } 330 331 ConverterDescriptor desc = converterDescriptors.get(converterName); 332 if (desc == null) { 333 throw new ConversionException("Converter " + converterName + " can not be found"); 334 } 335 336 String cacheKey = CacheKeyGenerator.computeKey(converterName, blobHolder, parameters); 337 338 BlobHolder result = ConversionCacheHolder.getFromCache(cacheKey); 339 340 if (result == null) { 341 Converter converter = desc.getConverterInstance(); 342 result = converter.convert(blobHolder, parameters); 343 344 if (config.isCacheEnabled()) { 345 ConversionCacheHolder.addToCache(cacheKey, result); 346 } 347 } 348 349 if (result != null) { 350 updateResultBlobMimeType(result, desc); 351 updateResultBlobFileName(blobHolder, result); 352 } 353 354 return result; 355 } 356 357 protected void updateResultBlobMimeType(BlobHolder resultBh, ConverterDescriptor desc) { 358 Blob mainBlob = resultBh.getBlob(); 359 if (mainBlob == null) { 360 return; 361 } 362 String mimeType = mainBlob.getMimeType(); 363 if (StringUtils.isBlank(mimeType) || mimeType.equals("application/octet-stream")) { 364 mainBlob.setMimeType(desc.getDestinationMimeType()); 365 } 366 } 367 368 protected void updateResultBlobFileName(BlobHolder srcBh, BlobHolder resultBh) { 369 Blob mainBlob = resultBh.getBlob(); 370 if (mainBlob == null) { 371 return; 372 } 373 String filename = mainBlob.getFilename(); 374 if (StringUtils.isBlank(filename) || filename.startsWith("nxblob-")) { 375 Blob srcBlob = srcBh.getBlob(); 376 if (srcBlob != null && StringUtils.isNotBlank(srcBlob.getFilename())) { 377 String baseName = FilenameUtils.getBaseName(srcBlob.getFilename()); 378 379 MimetypeRegistry mimetypeRegistry = Framework.getLocalService(MimetypeRegistry.class); 380 MimetypeEntry mimeTypeEntry = mimetypeRegistry.getMimetypeEntryByMimeType(mainBlob.getMimeType()); 381 List<String> extensions = mimeTypeEntry.getExtensions(); 382 String extension; 383 if (!extensions.isEmpty()) { 384 extension = extensions.get(0); 385 } else { 386 extension = FilenameUtils.getExtension(filename); 387 if (extension == null) { 388 extension = "bin"; 389 } 390 } 391 mainBlob.setFilename(baseName + "." + extension); 392 } 393 394 } 395 } 396 397 @Override 398 public BlobHolder convertToMimeType(String destinationMimeType, BlobHolder blobHolder, 399 Map<String, Serializable> parameters) throws ConversionException { 400 String srcMt = blobHolder.getBlob().getMimeType(); 401 String converterName = translationHelper.getConverterName(srcMt, destinationMimeType); 402 if (converterName == null) { 403 throw new ConversionException( 404 "Cannot find converter from type " + srcMt + " to type " + destinationMimeType); 405 } 406 return convert(converterName, blobHolder, parameters); 407 } 408 409 @Override 410 public List<String> getConverterNames(String sourceMimeType, String destinationMimeType) { 411 return translationHelper.getConverterNames(sourceMimeType, destinationMimeType); 412 } 413 414 @Override 415 public String getConverterName(String sourceMimeType, String destinationMimeType) { 416 List<String> converterNames = getConverterNames(sourceMimeType, destinationMimeType); 417 if (!converterNames.isEmpty()) { 418 return converterNames.get(converterNames.size() - 1); 419 } 420 return null; 421 } 422 423 @Override 424 public ConverterCheckResult isConverterAvailable(String converterName) throws ConversionException { 425 return isConverterAvailable(converterName, false); 426 } 427 428 protected final Map<String, ConverterCheckResult> checkResultCache = new HashMap<>(); 429 430 @Override 431 public ConverterCheckResult isConverterAvailable(String converterName, boolean refresh) 432 throws ConverterNotRegistered { 433 434 if (!refresh) { 435 if (checkResultCache.containsKey(converterName)) { 436 return checkResultCache.get(converterName); 437 } 438 } 439 440 ConverterDescriptor descriptor = converterDescriptors.get(converterName); 441 if (descriptor == null) { 442 throw new ConverterNotRegistered(converterName); 443 } 444 445 Converter converter = descriptor.getConverterInstance(); 446 447 ConverterCheckResult result; 448 if (converter instanceof ExternalConverter) { 449 ExternalConverter exConverter = (ExternalConverter) converter; 450 result = exConverter.isConverterAvailable(); 451 } else if (converter instanceof ChainedConverter) { 452 ChainedConverter chainedConverter = (ChainedConverter) converter; 453 result = new ConverterCheckResult(); 454 if (chainedConverter.isSubConvertersBased()) { 455 for (String subConverterName : chainedConverter.getSubConverters()) { 456 result = isConverterAvailable(subConverterName, refresh); 457 if (!result.isAvailable()) { 458 break; 459 } 460 } 461 } 462 } else { 463 // return success since there is nothing to test 464 result = new ConverterCheckResult(); 465 } 466 467 result.setSupportedInputMimeTypes(descriptor.getSourceMimeTypes()); 468 checkResultCache.put(converterName, result); 469 470 return result; 471 } 472 473 @Override 474 public boolean isSourceMimeTypeSupported(String converterName, String sourceMimeType) { 475 return getConverterDescriptor(converterName).getSourceMimeTypes().contains(sourceMimeType); 476 } 477 478 @Override 479 public String scheduleConversion(String converterName, BlobHolder blobHolder, 480 Map<String, Serializable> parameters) { 481 WorkManager workManager = Framework.getService(WorkManager.class); 482 ConversionWork work = new ConversionWork(converterName, null, blobHolder, parameters); 483 workManager.schedule(work); 484 return work.getId(); 485 } 486 487 @Override 488 public String scheduleConversionToMimeType(String destinationMimeType, BlobHolder blobHolder, 489 Map<String, Serializable> parameters) { 490 WorkManager workManager = Framework.getService(WorkManager.class); 491 ConversionWork work = new ConversionWork(null, destinationMimeType, blobHolder, parameters); 492 workManager.schedule(work); 493 return work.getId(); 494 } 495 496 @Override 497 public ConversionStatus getConversionStatus(String id) { 498 WorkManager workManager = Framework.getService(WorkManager.class); 499 Work.State workState = workManager.getWorkState(id); 500 if (workState == null) { 501 String entryKey = TransientStoreWork.computeEntryKey(id); 502 if (TransientStoreWork.containsBlobHolder(entryKey)) { 503 return new ConversionStatus(id, ConversionStatus.Status.COMPLETED); 504 } 505 return null; 506 } 507 508 return new ConversionStatus(id, ConversionStatus.Status.valueOf(workState.name())); 509 } 510 511 @Override 512 public BlobHolder getConversionResult(String id, boolean cleanTransientStoreEntry) { 513 String entryKey = TransientStoreWork.computeEntryKey(id); 514 BlobHolder bh = TransientStoreWork.getBlobHolder(entryKey); 515 if (cleanTransientStoreEntry) { 516 TransientStoreWork.removeBlobHolder(entryKey); 517 } 518 return bh; 519 } 520 521 @Override 522 public <T> T getAdapter(Class<T> adapter) { 523 if (adapter.isAssignableFrom(MimeTypeTranslationHelper.class)) { 524 return adapter.cast(translationHelper); 525 } 526 return super.getAdapter(adapter); 527 } 528 529 @Override 530 public void applicationStarted(ComponentContext context) { 531 startGC(); 532 } 533 534 protected void startGC() { 535 log.debug("CasheCGTaskActivator activated starting GC thread"); 536 gcTask = new GCTask(); 537 gcThread = new Thread(gcTask, "Nuxeo-Convert-GC"); 538 gcThread.setDaemon(true); 539 gcThread.start(); 540 log.debug("GC Thread started"); 541 542 } 543 544 public void endGC() { 545 if (gcTask == null) { 546 return; 547 } 548 log.debug("Stopping GC Thread"); 549 gcTask.GCEnabled = false; 550 gcTask = null; 551 gcThread.interrupt(); 552 gcThread = null; 553 } 554 555}