001/*
002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Tiry
018 *     Florent Guillaume
019 *     Estelle Giuly <egiuly@nuxeo.com>
020 */
021package org.nuxeo.ecm.core.convert.service;
022
023import java.io.File;
024import java.io.IOException;
025import java.io.Serializable;
026import java.nio.file.Path;
027import java.util.ArrayList;
028import java.util.HashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.function.Function;
032import java.util.regex.Matcher;
033import java.util.regex.Pattern;
034
035import javax.ws.rs.core.MediaType;
036
037import org.apache.commons.io.FilenameUtils;
038import org.apache.commons.lang.StringUtils;
039import org.apache.commons.logging.Log;
040import org.apache.commons.logging.LogFactory;
041import org.nuxeo.common.utils.FileUtils;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.blobholder.BlobHolder;
044import org.nuxeo.ecm.core.api.blobholder.SimpleBlobHolder;
045import org.nuxeo.ecm.core.api.impl.blob.StringBlob;
046import org.nuxeo.ecm.core.convert.api.ConversionException;
047import org.nuxeo.ecm.core.convert.api.ConversionService;
048import org.nuxeo.ecm.core.convert.api.ConversionStatus;
049import org.nuxeo.ecm.core.convert.api.ConverterCheckResult;
050import org.nuxeo.ecm.core.convert.api.ConverterNotAvailable;
051import org.nuxeo.ecm.core.convert.api.ConverterNotRegistered;
052import org.nuxeo.ecm.core.convert.cache.CacheKeyGenerator;
053import org.nuxeo.ecm.core.convert.cache.ConversionCacheHolder;
054import org.nuxeo.ecm.core.convert.cache.GCTask;
055import org.nuxeo.ecm.core.convert.extension.ChainedConverter;
056import org.nuxeo.ecm.core.convert.extension.Converter;
057import org.nuxeo.ecm.core.convert.extension.ConverterDescriptor;
058import org.nuxeo.ecm.core.convert.extension.ExternalConverter;
059import org.nuxeo.ecm.core.convert.extension.GlobalConfigDescriptor;
060import org.nuxeo.ecm.core.io.download.DownloadService;
061import org.nuxeo.ecm.core.transientstore.work.TransientStoreWork;
062import org.nuxeo.ecm.core.work.api.Work;
063import org.nuxeo.ecm.core.work.api.WorkManager;
064import org.nuxeo.ecm.platform.mimetype.interfaces.MimetypeEntry;
065import org.nuxeo.ecm.platform.mimetype.interfaces.MimetypeRegistry;
066import org.nuxeo.runtime.RuntimeServiceEvent;
067import org.nuxeo.runtime.RuntimeServiceListener;
068import org.nuxeo.runtime.api.Framework;
069import org.nuxeo.runtime.model.ComponentContext;
070import org.nuxeo.runtime.model.ComponentInstance;
071import org.nuxeo.runtime.model.DefaultComponent;
072import org.nuxeo.runtime.reload.ReloadService;
073import org.nuxeo.runtime.services.event.Event;
074import org.nuxeo.runtime.services.event.EventListener;
075import org.nuxeo.runtime.services.event.EventService;
076
077/**
078 * Runtime Component that also provides the POJO implementation of the {@link ConversionService}.
079 */
080public class ConversionServiceImpl extends DefaultComponent implements ConversionService {
081
082    protected static final Log log = LogFactory.getLog(ConversionServiceImpl.class);
083
084    public static final String CONVERTER_EP = "converter";
085
086    public static final String CONFIG_EP = "configuration";
087
088    protected final Map<String, ConverterDescriptor> converterDescriptors = new HashMap<>();
089
090    protected final MimeTypeTranslationHelper translationHelper = new MimeTypeTranslationHelper();
091
092    protected final GlobalConfigDescriptor config = new GlobalConfigDescriptor();
093
094    protected static ConversionServiceImpl self;
095
096    protected Thread gcThread;
097
098    protected GCTask gcTask;
099
100    ReloadListener reloadListener;
101
102    class ReloadListener implements EventListener {
103
104        @Override
105        public void handleEvent(Event event) {
106            if (ReloadService.AFTER_RELOAD_EVENT_ID.equals(event.getId())) {
107                startGC();
108            } else if (ReloadService.BEFORE_RELOAD_EVENT_ID.equals(event.getId())) {
109                endGC();
110            }
111        }
112
113    }
114
115    @Override
116    public void activate(ComponentContext context) {
117        converterDescriptors.clear();
118        translationHelper.clear();
119        self = this;
120        config.clearCachingDirectory();
121        Framework.addListener(new RuntimeServiceListener() {
122
123            @Override
124            public void handleEvent(RuntimeServiceEvent event) {
125                if (RuntimeServiceEvent.RUNTIME_ABOUT_TO_STOP != event.id) {
126                    return;
127                }
128                Framework.removeListener(this);
129                Framework.getService(EventService.class).removeListener(ReloadService.RELOAD_TOPIC, reloadListener);
130                endGC();
131            }
132        });
133        Framework.getService(EventService.class).addListener(ReloadService.RELOAD_TOPIC,
134                reloadListener = new ReloadListener());
135    }
136
137    @Override
138    public void deactivate(ComponentContext context) {
139        if (config.isCacheEnabled()) {
140            ConversionCacheHolder.deleteCache();
141        }
142        self = null;
143        converterDescriptors.clear();
144        translationHelper.clear();
145    }
146
147    /**
148     * Component implementation.
149     */
150    @Override
151    public void registerContribution(Object contribution, String extensionPoint, ComponentInstance contributor) {
152
153        if (CONVERTER_EP.equals(extensionPoint)) {
154            ConverterDescriptor desc = (ConverterDescriptor) contribution;
155            registerConverter(desc);
156        } else if (CONFIG_EP.equals(extensionPoint)) {
157            GlobalConfigDescriptor desc = (GlobalConfigDescriptor) contribution;
158            config.update(desc);
159            config.clearCachingDirectory();
160        } else {
161            log.error("Unable to handle unknown extensionPoint " + extensionPoint);
162        }
163    }
164
165    @Override
166    public void unregisterContribution(Object contribution, String extensionPoint, ComponentInstance contributor) {
167    }
168
169    /* Component API */
170
171    public static Converter getConverter(String converterName) {
172        ConverterDescriptor desc = self.converterDescriptors.get(converterName);
173        if (desc == null) {
174            return null;
175        }
176        return desc.getConverterInstance();
177    }
178
179    public static ConverterDescriptor getConverterDescriptor(String converterName) {
180        return self.converterDescriptors.get(converterName);
181    }
182
183    public static long getGCIntervalInMinutes() {
184        return self.config.getGCInterval();
185    }
186
187    public static void setGCIntervalInMinutes(long interval) {
188        self.config.setGCInterval(interval);
189    }
190
191    public static void registerConverter(ConverterDescriptor desc) {
192
193        if (self.converterDescriptors.containsKey(desc.getConverterName())) {
194
195            ConverterDescriptor existing = self.converterDescriptors.get(desc.getConverterName());
196            desc = existing.merge(desc);
197        }
198        desc.initConverter();
199        self.translationHelper.addConverter(desc);
200        self.converterDescriptors.put(desc.getConverterName(), desc);
201    }
202
203    public static int getMaxCacheSizeInKB() {
204        return self.config.getDiskCacheSize();
205    }
206
207    public static void setMaxCacheSizeInKB(int size) {
208        self.config.setDiskCacheSize(size);
209    }
210
211    public static boolean isCacheEnabled() {
212        return self.config.isCacheEnabled();
213    }
214
215    public static String getCacheBasePath() {
216        return self.config.getCachingDirectory();
217    }
218
219    /* Service API */
220
221    @Override
222    public List<String> getRegistredConverters() {
223        List<String> converterNames = new ArrayList<>();
224        converterNames.addAll(converterDescriptors.keySet());
225        return converterNames;
226    }
227
228    @Override
229    public Blob convertBlobToPDF(Blob blob) throws IOException {
230        String mimetype = blob.getMimeType();
231        String filename = blob.getFilename();
232        if (MimetypeRegistry.PDF_MIMETYPE.equals(mimetype)) {
233            return blob;
234        }
235        Blob result;
236        if (MediaType.TEXT_PLAIN.equals(mimetype)) {
237            result = convertBlobToMimeType(blob, MimetypeRegistry.PDF_MIMETYPE);
238        } else {
239            // Convert the blob to HTML
240            if (!MediaType.TEXT_HTML.equals(mimetype)) {
241                blob = convertBlobToMimeType(blob, MediaType.TEXT_HTML);
242                blob.setFilename(filename);
243            }
244            Path tempDirectory = Framework.createTempDirectory("blobs");
245            try {
246                // Replace the image URLs by absolute paths
247                DownloadService downloadService = Framework.getService(DownloadService.class);
248                blob = replaceURLsByAbsolutePaths(blob, tempDirectory, downloadService::resolveBlobFromDownloadUrl);
249                // Convert the blob to PDF
250                result = convertBlobToMimeType(blob, MimetypeRegistry.PDF_MIMETYPE);
251            } finally {
252                org.apache.commons.io.FileUtils.deleteQuietly(tempDirectory.toFile());
253            }
254        }
255        if (result != null) {
256            adjustPDFBlobName(filename, result);
257        }
258        return result;
259    }
260
261    protected Blob convertBlobToMimeType(Blob blob, String destinationMimeType) {
262        BlobHolder bh = new SimpleBlobHolder(blob);
263        bh = convertToMimeType(destinationMimeType, bh, null);
264        return bh == null ? null : bh.getBlob();
265    }
266
267    protected static void adjustPDFBlobName(String filename, Blob blob) {
268        if (StringUtils.isBlank(filename)) {
269            filename = "file_" + System.currentTimeMillis();
270        } else {
271            filename = FilenameUtils.removeExtension(FilenameUtils.getName(filename));
272        }
273        blob.setFilename(filename + MimetypeRegistry.PDF_EXTENSION);
274        blob.setMimeType(MimetypeRegistry.PDF_MIMETYPE);
275    }
276
277    /**
278     * Replace the image URLs of an HTML blob by absolute local paths.
279     *
280     * @throws IOException
281     * @since 9.1
282     */
283    protected static Blob replaceURLsByAbsolutePaths(Blob blob, Path tempDirectory, Function<String, Blob> blobResolver)
284            throws IOException {
285        String initialBlobContent = blob.getString();
286        // Find images links in the blob
287        Pattern pattern = Pattern.compile("(src=([\"']))(.*?)(\\2)");
288        Matcher matcher = pattern.matcher(initialBlobContent);
289        StringBuffer sb = new StringBuffer();
290        while (matcher.find()) {
291            // Retrieve the image from the URL
292            String url = matcher.group(3);
293            Blob imageBlob = blobResolver.apply(url);
294            if (imageBlob == null) {
295                break;
296            }
297            // Export the image to a temporary directory in File System
298            String safeFilename = FileUtils.getSafeFilename(imageBlob.getFilename());
299            File imageFile = tempDirectory.resolve(safeFilename).toFile();
300            imageBlob.transferTo(imageFile);
301            // Replace the image URL by its absolute local path
302            matcher.appendReplacement(sb, "$1" + Matcher.quoteReplacement(imageFile.toPath().toString()) + "$4");
303        }
304        matcher.appendTail(sb);
305        String blobContentWithAbsolutePaths = sb.toString();
306        if (blobContentWithAbsolutePaths.equals(initialBlobContent)) {
307            return blob;
308        }
309        // Create a new blob with the new content
310        Blob newBlob = new StringBlob(blobContentWithAbsolutePaths, blob.getMimeType(), blob.getEncoding());
311        newBlob.setFilename(blob.getFilename());
312        return newBlob;
313    }
314
315    @Override
316    public BlobHolder convert(String converterName, BlobHolder blobHolder, Map<String, Serializable> parameters)
317            throws ConversionException {
318
319        // set parameters if null to avoid NPE in converters
320        if (parameters == null) {
321            parameters = new HashMap<>();
322        }
323
324        // exist if not registered
325        ConverterCheckResult check = isConverterAvailable(converterName);
326        if (!check.isAvailable()) {
327            // exist is not installed / configured
328            throw new ConverterNotAvailable(converterName);
329        }
330
331        ConverterDescriptor desc = converterDescriptors.get(converterName);
332        if (desc == null) {
333            throw new ConversionException("Converter " + converterName + " can not be found");
334        }
335
336        String cacheKey = CacheKeyGenerator.computeKey(converterName, blobHolder, parameters);
337
338        BlobHolder result = ConversionCacheHolder.getFromCache(cacheKey);
339
340        if (result == null) {
341            Converter converter = desc.getConverterInstance();
342            result = converter.convert(blobHolder, parameters);
343
344            if (config.isCacheEnabled()) {
345                ConversionCacheHolder.addToCache(cacheKey, result);
346            }
347        }
348
349        if (result != null) {
350            updateResultBlobMimeType(result, desc);
351            updateResultBlobFileName(blobHolder, result);
352        }
353
354        return result;
355    }
356
357    protected void updateResultBlobMimeType(BlobHolder resultBh, ConverterDescriptor desc) {
358        Blob mainBlob = resultBh.getBlob();
359        if (mainBlob == null) {
360            return;
361        }
362        String mimeType = mainBlob.getMimeType();
363        if (StringUtils.isBlank(mimeType) || mimeType.equals("application/octet-stream")) {
364            mainBlob.setMimeType(desc.getDestinationMimeType());
365        }
366    }
367
368    protected void updateResultBlobFileName(BlobHolder srcBh, BlobHolder resultBh) {
369        Blob mainBlob = resultBh.getBlob();
370        if (mainBlob == null) {
371            return;
372        }
373        String filename = mainBlob.getFilename();
374        if (StringUtils.isBlank(filename) || filename.startsWith("nxblob-")) {
375            Blob srcBlob = srcBh.getBlob();
376            if (srcBlob != null && StringUtils.isNotBlank(srcBlob.getFilename())) {
377                String baseName = FilenameUtils.getBaseName(srcBlob.getFilename());
378
379                MimetypeRegistry mimetypeRegistry = Framework.getLocalService(MimetypeRegistry.class);
380                MimetypeEntry mimeTypeEntry = mimetypeRegistry.getMimetypeEntryByMimeType(mainBlob.getMimeType());
381                List<String> extensions = mimeTypeEntry.getExtensions();
382                String extension;
383                if (!extensions.isEmpty()) {
384                    extension = extensions.get(0);
385                } else {
386                    extension = FilenameUtils.getExtension(filename);
387                    if (extension == null) {
388                        extension = "bin";
389                    }
390                }
391                mainBlob.setFilename(baseName + "." + extension);
392            }
393
394        }
395    }
396
397    @Override
398    public BlobHolder convertToMimeType(String destinationMimeType, BlobHolder blobHolder,
399            Map<String, Serializable> parameters) throws ConversionException {
400        String srcMt = blobHolder.getBlob().getMimeType();
401        String converterName = translationHelper.getConverterName(srcMt, destinationMimeType);
402        if (converterName == null) {
403            throw new ConversionException(
404                    "Cannot find converter from type " + srcMt + " to type " + destinationMimeType);
405        }
406        return convert(converterName, blobHolder, parameters);
407    }
408
409    @Override
410    public List<String> getConverterNames(String sourceMimeType, String destinationMimeType) {
411        return translationHelper.getConverterNames(sourceMimeType, destinationMimeType);
412    }
413
414    @Override
415    public String getConverterName(String sourceMimeType, String destinationMimeType) {
416        List<String> converterNames = getConverterNames(sourceMimeType, destinationMimeType);
417        if (!converterNames.isEmpty()) {
418            return converterNames.get(converterNames.size() - 1);
419        }
420        return null;
421    }
422
423    @Override
424    public ConverterCheckResult isConverterAvailable(String converterName) throws ConversionException {
425        return isConverterAvailable(converterName, false);
426    }
427
428    protected final Map<String, ConverterCheckResult> checkResultCache = new HashMap<>();
429
430    @Override
431    public ConverterCheckResult isConverterAvailable(String converterName, boolean refresh)
432            throws ConverterNotRegistered {
433
434        if (!refresh) {
435            if (checkResultCache.containsKey(converterName)) {
436                return checkResultCache.get(converterName);
437            }
438        }
439
440        ConverterDescriptor descriptor = converterDescriptors.get(converterName);
441        if (descriptor == null) {
442            throw new ConverterNotRegistered(converterName);
443        }
444
445        Converter converter = descriptor.getConverterInstance();
446
447        ConverterCheckResult result;
448        if (converter instanceof ExternalConverter) {
449            ExternalConverter exConverter = (ExternalConverter) converter;
450            result = exConverter.isConverterAvailable();
451        } else if (converter instanceof ChainedConverter) {
452            ChainedConverter chainedConverter = (ChainedConverter) converter;
453            result = new ConverterCheckResult();
454            if (chainedConverter.isSubConvertersBased()) {
455                for (String subConverterName : chainedConverter.getSubConverters()) {
456                    result = isConverterAvailable(subConverterName, refresh);
457                    if (!result.isAvailable()) {
458                        break;
459                    }
460                }
461            }
462        } else {
463            // return success since there is nothing to test
464            result = new ConverterCheckResult();
465        }
466
467        result.setSupportedInputMimeTypes(descriptor.getSourceMimeTypes());
468        checkResultCache.put(converterName, result);
469
470        return result;
471    }
472
473    @Override
474    public boolean isSourceMimeTypeSupported(String converterName, String sourceMimeType) {
475        return getConverterDescriptor(converterName).getSourceMimeTypes().contains(sourceMimeType);
476    }
477
478    @Override
479    public String scheduleConversion(String converterName, BlobHolder blobHolder,
480            Map<String, Serializable> parameters) {
481        WorkManager workManager = Framework.getService(WorkManager.class);
482        ConversionWork work = new ConversionWork(converterName, null, blobHolder, parameters);
483        workManager.schedule(work);
484        return work.getId();
485    }
486
487    @Override
488    public String scheduleConversionToMimeType(String destinationMimeType, BlobHolder blobHolder,
489            Map<String, Serializable> parameters) {
490        WorkManager workManager = Framework.getService(WorkManager.class);
491        ConversionWork work = new ConversionWork(null, destinationMimeType, blobHolder, parameters);
492        workManager.schedule(work);
493        return work.getId();
494    }
495
496    @Override
497    public ConversionStatus getConversionStatus(String id) {
498        WorkManager workManager = Framework.getService(WorkManager.class);
499        Work.State workState = workManager.getWorkState(id);
500        if (workState == null) {
501            String entryKey = TransientStoreWork.computeEntryKey(id);
502            if (TransientStoreWork.containsBlobHolder(entryKey)) {
503                return new ConversionStatus(id, ConversionStatus.Status.COMPLETED);
504            }
505            return null;
506        }
507
508        return new ConversionStatus(id, ConversionStatus.Status.valueOf(workState.name()));
509    }
510
511    @Override
512    public BlobHolder getConversionResult(String id, boolean cleanTransientStoreEntry) {
513        String entryKey = TransientStoreWork.computeEntryKey(id);
514        BlobHolder bh = TransientStoreWork.getBlobHolder(entryKey);
515        if (cleanTransientStoreEntry) {
516            TransientStoreWork.removeBlobHolder(entryKey);
517        }
518        return bh;
519    }
520
521    @Override
522    public <T> T getAdapter(Class<T> adapter) {
523        if (adapter.isAssignableFrom(MimeTypeTranslationHelper.class)) {
524            return adapter.cast(translationHelper);
525        }
526        return super.getAdapter(adapter);
527    }
528
529    @Override
530    public void applicationStarted(ComponentContext context) {
531        startGC();
532    }
533
534    protected void startGC() {
535        log.debug("CasheCGTaskActivator activated starting GC thread");
536        gcTask = new GCTask();
537        gcThread = new Thread(gcTask, "Nuxeo-Convert-GC");
538        gcThread.setDaemon(true);
539        gcThread.start();
540        log.debug("GC Thread started");
541
542    }
543
544    public void endGC() {
545        if (gcTask == null) {
546            return;
547        }
548        log.debug("Stopping GC Thread");
549        gcTask.GCEnabled = false;
550        gcTask = null;
551        gcThread.interrupt();
552        gcThread = null;
553    }
554
555}