Source code

001/*
002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Tiago Cardoso <tcardoso@nuxeo.com>
018 */
019package org.nuxeo.ecm.platform.threed.convert;
020
021import org.apache.commons.io.FileUtils;
022import org.apache.commons.io.FilenameUtils;
023import org.apache.commons.io.IOUtils;
024import org.nuxeo.common.Environment;
025import org.nuxeo.common.utils.Path;
026import org.nuxeo.ecm.core.api.Blob;
027import org.nuxeo.ecm.core.api.CloseableFile;
028import org.nuxeo.ecm.core.api.blobholder.BlobHolder;
029import org.nuxeo.ecm.core.api.blobholder.SimpleBlobHolderWithProperties;
030import org.nuxeo.ecm.core.api.impl.blob.FileBlob;
031import org.nuxeo.ecm.platform.commandline.executor.api.*;
032import org.nuxeo.ecm.core.convert.api.ConversionException;
033import org.nuxeo.ecm.platform.convert.plugins.CommandLineBasedConverter;
034import org.nuxeo.runtime.api.Framework;
035
036import java.io.*;
037import java.util.*;
038import java.util.stream.Collectors;
039import java.util.zip.ZipEntry;
040import java.util.zip.ZipInputStream;
041
042import static org.nuxeo.ecm.platform.threed.ThreeDConstants.SUPPORTED_EXTENSIONS;
043import static org.nuxeo.ecm.platform.threed.convert.Constants.*;
044
045/**
046 * Base converter for blender pipeline. Processes scripts for operators and input blobs
047 *
048 * @since 8.4
049 */
050public abstract class BaseBlenderConverter extends CommandLineBasedConverter {
051
052    public static final String MIMETYPE_ZIP = "application/zip";
053
054    protected Path tempDirectory(Map<String, Serializable> parameters, String sufix) throws ConversionException {
055        Path directory = new Path(getTmpDirectory(parameters)).append(BLENDER_PATH_PREFIX + UUID.randomUUID() + sufix);
056        boolean dirCreated = new File(directory.toString()).mkdirs();
057        if (!dirCreated) {
058            throw new ConversionException("Unable to create tmp dir: " + directory);
059        }
060        return directory;
061    }
062
063    protected boolean isThreeDFile(File file) {
064        return SUPPORTED_EXTENSIONS.contains(FilenameUtils.getExtension(file.getName()));
065    }
066
067    private List<String> unpackZipFile(final File file, final File directory) throws IOException {
068        /* Extract ZIP contents */
069        List<String> expandedFiles = new ArrayList<>();
070        ZipEntry zipEntry;
071        ZipInputStream zipInputStream = null;
072        try {
073            zipInputStream = new ZipInputStream(new FileInputStream(file));
074            while ((zipEntry = zipInputStream.getNextEntry()) != null) {
075                final File destFile = new File(directory, zipEntry.getName());
076                if (!zipEntry.isDirectory()) {
077                    try (FileOutputStream destOutputStream = new FileOutputStream(destFile)) {
078                        IOUtils.copy(zipInputStream, destOutputStream);
079                    }
080                    zipInputStream.closeEntry();
081                    if (isThreeDFile(destFile)) {
082                        expandedFiles.add(0, destFile.getAbsolutePath());
083                    } else {
084                        expandedFiles.add(destFile.getAbsolutePath());
085                    }
086                } else {
087                    destFile.mkdirs();
088                }
089            }
090        } finally {
091            if (zipInputStream != null) {
092                zipInputStream.close();
093            }
094
095        }
096        return expandedFiles;
097    }
098
099    protected List<String> blobsToTempDir(BlobHolder blobHolder, Path directory) throws IOException {
100        List<Blob> blobs = blobHolder.getBlobs();
101        List<String> filesCreated = new ArrayList<>();
102        if (blobs.isEmpty()) {
103            return filesCreated;
104        }
105
106        if (MIMETYPE_ZIP.equals(blobs.get(0).getMimeType())) {
107            filesCreated.addAll(unpackZipFile(blobs.get(0).getFile(), new File(directory.toString())));
108            blobs = blobs.subList(1, blobs.size());
109        }
110
111        // Add the main and assets as params
112        // The params are not used by the command but the blobs are extracted to files and managed!
113        filesCreated.addAll(blobs.stream().map(blob -> {
114            File file = new File(directory.append(blob.getFilename()).toString());
115            try {
116                blob.transferTo(file);
117            } catch (IOException e) {
118                throw new UncheckedIOException(e);
119            }
120            return file.getAbsolutePath();
121        }).collect(Collectors.toList()));
122
123        filesCreated.add(directory.toString());
124        return filesCreated;
125    }
126
127    private void createPath(Path path) throws ConversionException {
128        File pathFile = new File(path.toString());
129        if (!pathFile.exists()) {
130            boolean dirCreated = pathFile.mkdir();
131            if (!dirCreated) {
132                throw new ConversionException("Unable to create tmp dir for scripts output: " + path);
133            }
134        }
135    }
136
137    private Path copyScript(Path pathDst, Path source) throws IOException {
138        String sourceFile = source.lastSegment();
139        // xxx : find a way to check if the correct version is already there.
140        File script = new File(pathDst.append(sourceFile).toString());
141        InputStream is = getClass().getResourceAsStream("/" + source.toString());
142        FileUtils.copyInputStreamToFile(is, script);
143        return new Path(script.getAbsolutePath());
144    }
145
146    /**
147     * Returns the absolute path to the main script (main and pipeline scripts). Copies the script to the filesystem if
148     * needed. Copies needed {@code operators} script to the filesystem if missing.
149     */
150    protected Path getScriptWith(List<String> operators) throws IOException {
151        Path dataPath = new Path(Environment.getDefault().getData().getAbsolutePath());
152        String sourceDir = initParameters.get(SCRIPTS_DIR_PARAMETER);
153        String sourceFile = initParameters.get(SCRIPT_FILE_PARAMETER);
154
155        // create scripts directory
156        Path scriptsPath = dataPath.append(SCRIPTS_DIRECTORY);
157        createPath(scriptsPath);
158
159        copyScript(scriptsPath, new Path(sourceDir).append(sourceFile));
160
161        if (operators.isEmpty()) {
162            return scriptsPath;
163        }
164
165        // create pipeline scripts directory
166        Path pipelinePath = scriptsPath.append(SCRIPTS_PIPELINE_DIRECTORY);
167        createPath(pipelinePath);
168
169        Path pipelineSourcePath = new Path(sourceDir).append("pipeline");
170        // copy operators scripts resources
171        operators.forEach(operator -> {
172            try {
173                copyScript(pipelinePath, pipelineSourcePath.append(operator + ".py"));
174            } catch (IOException e) {
175                throw new UncheckedIOException(e);
176            }
177        });
178
179        return scriptsPath;
180    }
181
182    private List<String> getParams(Map<String, Serializable> inParams, Map<String, String> initParams, String key) {
183        String values = "";
184        if (inParams.containsKey(key)) {
185            values = (String) inParams.get(key);
186        } else if (initParams.containsKey(key)) {
187            values = initParams.get(key);
188        }
189        return Arrays.asList(values.split(" "));
190    }
191
192    @Override
193    public BlobHolder convert(BlobHolder blobHolder, Map<String, Serializable> parameters) throws ConversionException {
194        String dataContainer = "data" + String.valueOf(Calendar.getInstance().getTime().getTime());
195        String convertContainer = "convert" + String.valueOf(Calendar.getInstance().getTime().getTime());
196        String commandName = getCommandName(blobHolder, parameters);
197        if (commandName == null) {
198            throw new ConversionException("Unable to determine target CommandLine name");
199        }
200
201        List<Closeable> toClose = new ArrayList<>();
202        Path inDirectory = tempDirectory(null, "_in");
203        try {
204            CmdParameters params = new CmdParameters();
205
206            // Deal with operators and script files (blender and pipeline)
207            List<String> operatorsList = getParams(parameters, initParameters, OPERATORS_PARAMETER);
208            params.addNamedParameter(OPERATORS_PARAMETER, operatorsList);
209
210            operatorsList = operatorsList.stream().distinct().collect(Collectors.toList());
211            Path mainScriptDir = getScriptWith(operatorsList);
212            // params.addNamedParameter(SCRIPTS_DIR_PARAMETER, mainScriptDir.toString());
213            params.addNamedParameter(SCRIPT_FILE_PARAMETER, initParameters.get(SCRIPT_FILE_PARAMETER));
214
215            // Initialize render id params
216            params.addNamedParameter(RENDER_IDS_PARAMETER, getParams(parameters, initParameters, RENDER_IDS_PARAMETER));
217
218            // Initialize LOD id params
219            params.addNamedParameter(LOD_IDS_PARAMETER, getParams(parameters, initParameters, LOD_IDS_PARAMETER));
220
221            // Initialize percentage polygon params
222            params.addNamedParameter(PERC_POLY_PARAMETER, getParams(parameters, initParameters, PERC_POLY_PARAMETER));
223
224            // Initialize max polygon params
225            params.addNamedParameter(MAX_POLY_PARAMETER, getParams(parameters, initParameters, MAX_POLY_PARAMETER));
226
227            // Initialize spherical coordinates params
228            params.addNamedParameter(COORDS_PARAMETER, getParams(parameters, initParameters, COORDS_PARAMETER));
229
230            // Initialize dimension params
231            params.addNamedParameter(DIMENSIONS_PARAMETER, getParams(parameters, initParameters, DIMENSIONS_PARAMETER));
232
233            // Deal with input blobs (main and assets)
234            List<String> inputFiles = blobsToTempDir(blobHolder, inDirectory);
235            Path mainFile = new Path(inputFiles.get(0));
236            // params.addNamedParameter(INPUT_DIR_PARAMETER, mainFile.removeLastSegments(1).toString() );
237            params.addNamedParameter(INPUT_FILE_PARAMETER, mainFile.lastSegment());
238
239            // Extra blob parameters
240            Map<String, Blob> blobParams = getCmdBlobParameters(blobHolder, parameters);
241
242            ExecResult createRes = DockerHelper.CreateContainer(dataContainer, "nuxeo/blender");
243            if (createRes == null || !createRes.isSuccessful()) {
244                throw new ConversionException("Unable to create data volume : " + dataContainer,
245                        (createRes != null) ? createRes.getError() : null);
246            }
247            ExecResult copyRes = DockerHelper.CopyData(
248                    mainFile.removeLastSegments(1).toString() + File.separatorChar + ".", dataContainer + ":/in/");
249            if (copyRes == null || !copyRes.isSuccessful()) {
250                throw new ConversionException("Unable to copy content to data volume : " + dataContainer,
251                        (copyRes != null) ? copyRes.getError() : null);
252            }
253            copyRes = DockerHelper.CopyData(mainScriptDir.toString() + File.separatorChar + ".",
254                    dataContainer + ":/scripts/");
255            if (copyRes == null || !copyRes.isSuccessful()) {
256                throw new ConversionException("Unable to copy to scripts data volume : " + dataContainer,
257                        (copyRes != null) ? copyRes.getError() : null);
258            }
259            params.addNamedParameter(NAME_PARAM, convertContainer);
260            params.addNamedParameter(DATA_PARAM, dataContainer);
261
262            if (blobParams != null) {
263                for (String blobParamName : blobParams.keySet()) {
264                    Blob blob = blobParams.get(blobParamName);
265                    // closed in finally block
266                    CloseableFile closeable = blob.getCloseableFile(
267                            "." + FilenameUtils.getExtension(blob.getFilename()));
268                    params.addNamedParameter(blobParamName, closeable.getFile());
269                    toClose.add(closeable);
270                }
271            }
272
273            // Extra string parameters
274            Map<String, String> strParams = getCmdStringParameters(blobHolder, parameters);
275
276            if (strParams != null) {
277                for (String paramName : strParams.keySet()) {
278                    if (RENDER_IDS_PARAMETER.equals(paramName) || LOD_IDS_PARAMETER.equals(paramName)
279                            || PERC_POLY_PARAMETER.equals(paramName) || MAX_POLY_PARAMETER.equals(paramName)
280                            || COORDS_PARAMETER.equals(paramName) || DIMENSIONS_PARAMETER.equals(paramName)) {
281                        if (strParams.get(paramName) != null) {
282                            params.addNamedParameter(paramName, Arrays.asList(strParams.get(paramName).split(" ")));
283                        }
284                    } else {
285                        params.addNamedParameter(paramName, strParams.get(paramName));
286                    }
287                }
288            }
289
290            // Deal with output directory
291            Path outDir = tempDirectory(null, "_out");
292            params.addNamedParameter(OUT_DIR_PARAMETER, outDir.toString());
293
294            ExecResult result = Framework.getService(CommandLineExecutorService.class).execCommand(commandName, params);
295            if (!result.isSuccessful()) {
296                throw result.getError();
297            }
298
299            copyRes = DockerHelper.CopyData(dataContainer + ":/out/.", outDir.toString());
300            if (copyRes == null || !copyRes.isSuccessful()) {
301                throw new ConversionException("Unable to copy from data volume : " + dataContainer,
302                        (copyRes != null) ? copyRes.getError() : null);
303            }
304            return buildResult(result.getOutput(), params);
305        } catch (CommandNotAvailable e) {
306            // XXX bubble installation instructions
307            throw new ConversionException("Unable to find targetCommand", e);
308        } catch (IOException | CommandException e) {
309            throw new ConversionException("Error while converting via CommandLineService", e);
310        } finally {
311            FileUtils.deleteQuietly(new File(inDirectory.toString()));
312            for (Closeable closeable : toClose) {
313                IOUtils.closeQuietly(closeable);
314            }
315            DockerHelper.RemoveContainer(dataContainer);
316            DockerHelper.RemoveContainer(convertContainer);
317        }
318
319    }
320
321    public List<String> getConversionLOD(String outDir) {
322        File directory = new File(outDir);
323        String[] files = directory.list((dir, name) -> true);
324        return (files == null) ? Collections.emptyList() : Arrays.asList(files);
325    }
326
327    public List<String> getRenders(String outDir) {
328        File directory = new File(outDir);
329        String[] files = directory.list((dir, name) -> name.startsWith("render") && name.endsWith(".png"));
330        return (files == null) ? Collections.emptyList() : Arrays.asList(files);
331    }
332
333    public List<String> getInfos(String outDir) {
334        File directory = new File(outDir);
335        String[] files = directory.list((dir, name) -> name.endsWith(".info"));
336        return (files == null) ? Collections.emptyList() : Arrays.asList(files);
337    }
338
339    @Override
340    protected BlobHolder buildResult(List<String> cmdOutput, CmdParameters cmdParams) throws ConversionException {
341        String outDir = cmdParams.getParameter(OUT_DIR_PARAMETER);
342        Map<String, Integer> lodBlobIndexes = new HashMap<>();
343        List<Integer> resourceIndexes = new ArrayList<>();
344        Map<String, Integer> infoIndexes = new HashMap<>();
345        List<Blob> blobs = new ArrayList<>();
346
347        String lodDir = outDir + File.separatorChar + "convert";
348        List<String> conversions = getConversionLOD(lodDir);
349        conversions.forEach(filename -> {
350            File file = new File(lodDir + File.separatorChar + filename);
351            Blob blob = new FileBlob(file);
352            blob.setFilename(file.getName());
353            if (FilenameUtils.getExtension(filename).toLowerCase().equals("dae")) {
354                String[] filenameArray = filename.split("-");
355                if (filenameArray.length != 4) {
356                    throw new ConversionException(filenameArray + " incompatible with conversion file name schema.");
357                }
358                lodBlobIndexes.put(filenameArray[1], blobs.size());
359            } else {
360                resourceIndexes.add(blobs.size());
361            }
362            blobs.add(blob);
363        });
364
365        String infoDir = outDir + File.separatorChar + "info";
366        List<String> infos = getInfos(infoDir);
367        infos.forEach(filename -> {
368            File file = new File(infoDir + File.separatorChar + filename);
369            Blob blob = new FileBlob(file);
370            blob.setFilename(file.getName());
371            if (FilenameUtils.getExtension(filename).toLowerCase().equals("info")) {
372                String lodId = FilenameUtils.getBaseName(filename);
373                infoIndexes.put(lodId, blobs.size());
374                blobs.add(blob);
375            }
376        });
377
378        String renderDir = outDir + File.separatorChar + "render";
379        List<String> renders = getRenders(renderDir);
380
381        Map<String, Serializable> properties = new HashMap<>();
382        properties.put("cmdOutput", (Serializable) cmdOutput);
383        properties.put("resourceIndexes", (Serializable) resourceIndexes);
384        properties.put("infoIndexes", (Serializable) infoIndexes);
385        properties.put("lodIdIndexes", (Serializable) lodBlobIndexes);
386        properties.put("renderStartIndex", blobs.size());
387
388        blobs.addAll(renders.stream().map(result -> {
389            File file = new File(renderDir + File.separatorChar + result);
390            Blob blob = new FileBlob(file);
391            blob.setFilename(file.getName());
392            return blob;
393        }).collect(Collectors.toList()));
394
395        return new SimpleBlobHolderWithProperties(blobs, properties);
396    }
397}