001/*
002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Nuxeo - initial API and implementation
018 *     bstefanescu, jcarsique
019 *     Anahide Tchertchian
020 */
021package org.nuxeo.runtime.deployment.preprocessor;
022
023import static java.nio.charset.StandardCharsets.UTF_8;
024
025import java.io.BufferedInputStream;
026import java.io.File;
027import java.io.IOException;
028import java.io.InputStream;
029import java.net.MalformedURLException;
030import java.net.URL;
031import java.util.ArrayList;
032import java.util.Arrays;
033import java.util.List;
034import java.util.Properties;
035import java.util.jar.Attributes;
036import java.util.jar.JarFile;
037import java.util.jar.Manifest;
038import java.util.regex.Matcher;
039import java.util.regex.Pattern;
040import java.util.zip.ZipEntry;
041
042import org.apache.commons.io.FileUtils;
043import org.apache.commons.logging.Log;
044import org.apache.commons.logging.LogFactory;
045import org.nuxeo.common.collections.DependencyTree;
046import org.nuxeo.common.utils.JarUtils;
047import org.nuxeo.common.utils.Path;
048import org.nuxeo.common.utils.StringUtils;
049import org.nuxeo.common.xmap.XMap;
050import org.nuxeo.launcher.config.ConfigurationGenerator;
051import org.nuxeo.runtime.deployment.preprocessor.install.CommandContext;
052import org.nuxeo.runtime.deployment.preprocessor.install.CommandContextImpl;
053import org.nuxeo.runtime.deployment.preprocessor.template.TemplateContribution;
054import org.nuxeo.runtime.deployment.preprocessor.template.TemplateParser;
055
056/**
057 * Initializer for the deployment skeleton, taking care of creating templates, aggregating default components before
058 * runtime is started.
059 *
060 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a>
061 */
062public class DeploymentPreprocessor {
063
064    public static final String FRAGMENT_FILE = "OSGI-INF/deployment-fragment.xml";
065
066    public static final String CONTAINER_FILE = "META-INF/nuxeo-preprocessor.xml";
067
068    public static final String CONTAINER_FILE_COMPAT = "OSGI-INF/deployment-container.xml";
069
070    private static final Pattern ARTIFACT_NAME_PATTERN = Pattern.compile("-[0-9]+");
071
072    private static final Log log = LogFactory.getLog(DeploymentPreprocessor.class);
073
074    private final File dir;
075
076    private final XMap xmap;
077
078    private ContainerDescriptor root;
079
080    public DeploymentPreprocessor(File dir) {
081        this.dir = dir;
082        xmap = new XMap();
083        xmap.register(ContainerDescriptor.class);
084        xmap.register(FragmentDescriptor.class);
085    }
086
087    public ContainerDescriptor getRootContainer() {
088        return root;
089    }
090
091    public void init() throws IOException {
092        root = getDefaultContainer(dir);
093        if (root != null) {
094            // run container commands
095            init(root);
096        }
097    }
098
099    public void init(File metadata, File[] files) throws IOException {
100        if (metadata == null) {
101            root = getDefaultContainer(dir);
102        } else {
103            root = getContainer(dir, metadata);
104        }
105        if (root != null) {
106            root.files = files;
107            // run container commands
108            init(root);
109        }
110    }
111
112    protected void init(ContainerDescriptor cd) throws IOException {
113        cd.context = new CommandContextImpl(cd.directory);
114        initContextProperties(cd.context);
115        // run container install instructions if any
116        if (cd.install != null) {
117            cd.install.setLogger(log);
118            log.info("Running custom installation for container: " + cd.name);
119            cd.install.exec(cd.context);
120        }
121        if (cd.files != null) {
122            init(cd, cd.files);
123        } else {
124            // scan directories
125            if (cd.directories == null || cd.directories.isEmpty()) {
126                init(cd, dir);
127            } else {
128                for (String dirPath : cd.directories) {
129                    init(cd, new File(dir, dirPath));
130                }
131            }
132        }
133    }
134
135    protected void initContextProperties(CommandContext ctx) {
136        ConfigurationGenerator confGen = new ConfigurationGenerator();
137        confGen.init();
138        Properties props = confGen.getUserConfig();
139        for (String key : props.stringPropertyNames()) {
140            ctx.put(key, props.getProperty(key));
141        }
142    }
143
144    protected void processFile(ContainerDescriptor cd, File file) throws IOException {
145        String fileName = file.getName();
146        FragmentDescriptor fd = null;
147        boolean isBundle = false;
148        if (fileName.endsWith("-fragment.xml")) {
149            fd = getXMLFragment(file);
150        } else if (fileName.endsWith("-fragments.xml")) {
151            // we allow declaring multiple fragments in the same file
152            // this is useful to deploy libraries
153            collectXMLFragments(cd, file);
154            return;
155        } else if (fileName.endsWith(".jar") || fileName.endsWith(".war") || fileName.endsWith(".sar")
156                || fileName.endsWith(".rar")) {
157            isBundle = true;
158            if (file.isDirectory()) {
159                fd = getDirectoryFragment(file);
160            } else {
161                fd = getJARFragment(file);
162            }
163        }
164        // register the fragment if any was found
165        if (fd != null) {
166            fd.fileName = fileName;
167            fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath());
168            cd.fragments.add(fd);
169            if (fd.templates != null) {
170                for (TemplateDescriptor td : fd.templates.values()) {
171                    td.baseDir = file;
172                    cd.templates.put(td.name, td);
173                }
174            }
175        } else if (isBundle) {
176            // create markers - for compatibility with versions < 5.4
177            String name = getSymbolicName(file);
178            if (name != null) {
179                cd.fragments.add(new FragmentDescriptor(name, true));
180            }
181        }
182    }
183
184    protected String getSymbolicName(File file) {
185        Manifest mf = JarUtils.getManifest(file);
186        if (mf != null) {
187            Attributes attrs = mf.getMainAttributes();
188            String id = attrs.getValue("Bundle-SymbolicName");
189            if (id != null) {
190                int p = id.indexOf(';');
191                if (p > -1) { // remove properties part if any
192                    id = id.substring(0, p);
193                }
194                return id;
195            }
196        }
197        return null;
198    }
199
200    protected String getJarArtifactName(String name) {
201        if (name.endsWith(".jar")) {
202            name = name.substring(0, name.length() - 4);
203        }
204        Matcher m = ARTIFACT_NAME_PATTERN.matcher(name);
205        if (m.find()) {
206            name = name.substring(0, m.start());
207        }
208        return name;
209    }
210
211    protected void init(ContainerDescriptor cd, File[] files) throws IOException {
212        for (File file : files) {
213            processFile(cd, file);
214        }
215    }
216
217    protected void init(ContainerDescriptor cd, File dir) throws IOException {
218        log.info("Scanning directory: " + dir.getName());
219        if (!dir.exists()) {
220            log.warn("Directory doesn't exist: " + dir.getPath());
221            return;
222        }
223        // sort input files in alphabetic order -> this way we are sure we get
224        // the same deploying order on all machines.
225        File[] files = dir.listFiles();
226        Arrays.sort(files);
227        init(cd, files);
228    }
229
230    public void predeploy() throws IOException {
231        if (root != null) {
232            predeploy(root);
233        }
234    }
235
236    protected static String listFragmentDescriptor(FragmentDescriptor fd) {
237        return fd.name + " (" + fd.fileName + ")";
238    }
239
240    protected static void printInfo(FragmentRegistry fragments) {
241        List<DependencyTree.Entry<String, FragmentDescriptor>> entries = fragments.getResolvedEntries();
242        StringBuilder buf = new StringBuilder("Preprocessing order: ");
243        for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) {
244            FragmentDescriptor fd = entry.get();
245            if (fd != null && !fd.isMarker()) {
246                buf.append("\n\t");
247                buf.append(listFragmentDescriptor(entry.get()));
248            }
249        }
250        log.info(buf);
251
252        StringBuilder errors = new StringBuilder();
253        List<DependencyTree.Entry<String, FragmentDescriptor>> missing = fragments.getMissingRequirements();
254        for (DependencyTree.Entry<String, FragmentDescriptor> entry : missing) {
255            buf = new StringBuilder("Unknown bundle: ");
256            buf.append(entry.getKey());
257            buf.append(" required by: ");
258            boolean first = true;
259            for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getDependsOnMe()) {
260                if (!first) {
261                    buf.append(", "); // length 2
262                }
263                first = false;
264                buf.append(listFragmentDescriptor(dep.get()));
265            }
266            log.error(buf);
267            errors.append(buf);
268            errors.append("\n");
269        }
270        for (DependencyTree.Entry<String, FragmentDescriptor> entry : fragments.getPendingEntries()) {
271            if (!entry.isRegistered()) {
272                continue;
273            }
274            buf = new StringBuilder("Bundle not preprocessed: ");
275            buf.append(listFragmentDescriptor(entry.get()));
276            buf.append(" waiting for: ");
277            boolean first = true;
278            for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getWaitsFor()) {
279                if (!first) {
280                    buf.append(", "); // length 2
281                }
282                first = false;
283                buf.append(dep.getKey());
284            }
285            log.error(buf);
286            errors.append(buf);
287            errors.append("\n");
288        }
289        if (errors.length() != 0) {
290            // set system property to log startup errors
291            // this is read by AbstractRuntimeService
292            System.setProperty("org.nuxeo.runtime.deployment.errors", errors.toString());
293        }
294    }
295
296    protected static void predeploy(ContainerDescriptor cd) throws IOException {
297        // run installer and register contributions for each fragment
298        List<DependencyTree.Entry<String, FragmentDescriptor>> entries = cd.fragments.getResolvedEntries();
299        printInfo(cd.fragments);
300        for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) {
301            FragmentDescriptor fd = entry.get();
302            if (fd == null || fd.isMarker()) {
303                continue; // should be a marker entry like the "all" one.
304            }
305            cd.context.put("bundle.fileName", fd.filePath);
306            cd.context.put("bundle.shortName", fd.fileName);
307            cd.context.put("bundle", fd.name);
308
309            // execute install instructions if any
310            if (fd.install != null) {
311                fd.install.setLogger(log);
312                log.info("Running custom installation for fragment: " + fd.name);
313                fd.install.exec(cd.context);
314            }
315
316            if (fd.contributions == null) {
317                continue; // no contributions
318            }
319
320            // get fragment contributions and register them
321            for (TemplateContribution tc : fd.contributions) {
322
323                // register template contributions if any
324                // get the target template
325                TemplateDescriptor td = cd.templates.get(tc.getTemplate());
326                if (td != null) {
327                    if (td.baseDir == null) {
328                        td.baseDir = cd.directory;
329                    }
330                    if (td.template == null) { // template not yet compiled
331                        File file = new File(td.baseDir, td.src);
332                        // compile it
333                        td.template = TemplateParser.parse(file);
334                    }
335                } else {
336                    log.warn("No template '" + tc.getTemplate() + "' found for deployment fragment:  " + fd.name);
337                    continue;
338                }
339                // get the marker where contribution should be inserted
340                td.template.update(tc, cd.context);
341            }
342        }
343
344        // process and write templates
345        // fragments where imported. write down templates
346        for (TemplateDescriptor td : cd.templates.values()) {
347            if (td.baseDir == null) {
348                td.baseDir = cd.directory;
349            }
350            // if required process the template even if no contributions were
351            // made
352            if (td.template == null && td.isRequired) {
353                // compile the template
354                File file = new File(td.baseDir, td.src);
355                td.template = TemplateParser.parse(file);
356            }
357            // process the template
358            if (td.template != null) {
359                File file = new File(td.baseDir, td.installPath);
360                file.getParentFile().mkdirs(); // make sure parents exists
361                FileUtils.writeStringToFile(file, td.template.getText(), UTF_8);
362            }
363        }
364
365        // process sub containers if any
366        for (ContainerDescriptor subCd : cd.subContainers) {
367            predeploy(subCd);
368        }
369    }
370
371    protected FragmentDescriptor getXMLFragment(File file) throws IOException {
372        URL url;
373        try {
374            url = file.toURI().toURL();
375        } catch (MalformedURLException e) {
376            throw new RuntimeException(e);
377        }
378        FragmentDescriptor fd = (FragmentDescriptor) xmap.load(url);
379        if (fd != null && fd.name == null) {
380            fd.name = file.getName();
381        }
382        return fd;
383    }
384
385    protected void collectXMLFragments(ContainerDescriptor cd, File file) throws IOException {
386        String fileName = file.getName();
387        URL url;
388        try {
389            url = file.toURI().toURL();
390        } catch (MalformedURLException e) {
391            throw new RuntimeException(e);
392        }
393        Object[] result = xmap.loadAll(url);
394        for (Object entry : result) {
395            FragmentDescriptor fd = (FragmentDescriptor) entry;
396            assert fd != null;
397            if (fd.name == null) {
398                log.error("Invalid fragments file: " + file.getName()
399                        + ". Fragments declared in a -fragments.xml file must have names.");
400            } else {
401                cd.fragments.add(fd);
402                fd.fileName = fileName;
403                fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath());
404            }
405        }
406    }
407
408    protected void processBundleForCompat(FragmentDescriptor fd, File file) {
409        // TODO disable for now the warning
410        log.warn("Entering compatibility mode - Please update the deployment-fragment.xml in " + file.getName()
411                + " to use new dependency management");
412        Manifest mf = JarUtils.getManifest(file);
413        if (mf != null) {
414            fd.name = file.getName();
415            processManifest(fd, fd.name, mf);
416        } else {
417            throw new RuntimeException("Compat: Fragments without a name must reside in an OSGi bundle");
418        }
419    }
420
421    protected FragmentDescriptor getDirectoryFragment(File directory) throws IOException {
422        FragmentDescriptor fd;
423        File file = new File(directory.getAbsolutePath() + '/' + FRAGMENT_FILE);
424        if (file.isFile()) {
425            URL url;
426            try {
427                url = file.toURI().toURL();
428            } catch (MalformedURLException e) {
429                throw new RuntimeException(e);
430            }
431            fd = (FragmentDescriptor) xmap.load(url);
432        } else {
433            return null; // don't need preprocessing
434        }
435        if (fd.name == null) {
436            // fallback on symbolic name
437            fd.name = getSymbolicName(directory);
438        }
439        if (fd.name == null) {
440            // fallback on artifact id
441            fd.name = getJarArtifactName(directory.getName());
442        }
443        if (fd.version == 0) { // compat with versions < 5.4
444            processBundleForCompat(fd, directory);
445        }
446        return fd;
447    }
448
449    protected FragmentDescriptor getJARFragment(File file) throws IOException {
450        FragmentDescriptor fd = null;
451        try (JarFile jar = new JarFile(file)) {
452            ZipEntry ze = jar.getEntry(FRAGMENT_FILE);
453            if (ze != null) {
454                try (InputStream in = new BufferedInputStream(jar.getInputStream(ze))) {
455                    fd = (FragmentDescriptor) xmap.load(in);
456                }
457                if (fd.name == null) {
458                    // fallback on symbolic name
459                    fd.name = getSymbolicName(file);
460                }
461                if (fd.name == null) {
462                    // fallback on artifact id
463                    fd.name = getJarArtifactName(file.getName());
464                }
465                if (fd.version == 0) { // compat with versions < 5.4
466                    processBundleForCompat(fd, file);
467                }
468            }
469        }
470        return fd;
471    }
472
473    protected void processManifest(FragmentDescriptor fd, String fileName, Manifest mf) {
474        Attributes attrs = mf.getMainAttributes();
475        String id = attrs.getValue("Bundle-SymbolicName");
476        int p = id.indexOf(';');
477        if (p > -1) { // remove properties part if any
478            id = id.substring(0, p);
479        }
480        fd.name = id;
481        if (fd.requires != null && !fd.requires.isEmpty()) {
482            throw new RuntimeException(
483                    "In compatibility mode you must not use <require> tags for OSGi bundles - use Require-Bundle manifest header instead. Bundle: "
484                            + fileName);
485        }
486        // needed to control start-up order (which differs from
487        // Require-Bundle)
488        String requires = attrs.getValue("Nuxeo-Require");
489        if (requires == null) { // if not specific requirement is met use
490                                // Require-Bundle
491            requires = attrs.getValue("Require-Bundle");
492        }
493        if (requires != null) {
494            String[] ids = StringUtils.split(requires, ',', true);
495            fd.requires = new ArrayList<>(ids.length);
496            for (int i = 0; i < ids.length; i++) {
497                String rid = ids[i];
498                p = rid.indexOf(';');
499                if (p > -1) { // remove properties part if any
500                    ids[i] = rid.substring(0, p);
501                }
502                fd.requires.add(ids[i]);
503            }
504        }
505
506        String requiredBy = attrs.getValue("Nuxeo-RequiredBy");
507        if (requiredBy != null) {
508            String[] ids = StringUtils.split(requiredBy, ',', true);
509            for (int i = 0; i < ids.length; i++) {
510                String rid = ids[i];
511                p = rid.indexOf(';');
512                if (p > -1) { // remove properties part if any
513                    ids[i] = rid.substring(0, p);
514                }
515            }
516            fd.requiredBy = ids;
517        }
518
519    }
520
521    /**
522     * Reads a container fragment metadata file and returns the container descriptor.
523     */
524    protected ContainerDescriptor getContainer(File home, File file) throws IOException {
525        URL url;
526        try {
527            url = file.toURI().toURL();
528        } catch (MalformedURLException e) {
529            throw new RuntimeException(e);
530        }
531        ContainerDescriptor cd = (ContainerDescriptor) xmap.load(url);
532        if (cd != null) {
533            cd.directory = home;
534            if (cd.name == null) {
535                cd.name = home.getName();
536            }
537        }
538        return cd;
539    }
540
541    protected ContainerDescriptor getDefaultContainer(File directory) throws IOException {
542        File file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE);
543        if (!file.isFile()) {
544            file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE_COMPAT);
545        }
546        ContainerDescriptor cd = null;
547        if (file.isFile()) {
548            cd = getContainer(directory, file);
549        }
550        return cd;
551    }
552
553    public static String getRelativeChildPath(String parent, String child) {
554        // TODO optimize this method
555        // fix win32 case
556        if (parent.indexOf('\\') > -1) {
557            parent = parent.replace('\\', '/');
558        }
559        if (child.indexOf('\\') > -1) {
560            child = child.replace('\\', '/');
561        } // end fix win32
562        Path parentPath = new Path(parent);
563        Path childPath = new Path(child);
564        if (parentPath.isPrefixOf(childPath)) {
565            return childPath.removeFirstSegments(parentPath.segmentCount()).makeRelative().toString();
566        }
567        return null;
568    }
569
570    /**
571     * Run preprocessing in the given home directory and using the given list of bundles. Bundles must be ordered by the
572     * caller to have same deployment order on all computers.
573     * <p>
574     * The metadata file is the metadat file to be used to configure the processor. If null the default location will be
575     * used (relative to home): {@link #CONTAINER_FILE}.
576     */
577    public static void process(File home, File metadata, File[] files) throws IOException {
578        DeploymentPreprocessor processor = new DeploymentPreprocessor(home);
579        // initialize
580        processor.init(metadata, files);
581        // run preprocessor
582        processor.predeploy();
583    }
584
585    public static void main(String[] args) throws IOException {
586        File root;
587        if (args.length > 0) {
588            root = new File(args[0]);
589        } else {
590            root = new File(".");
591        }
592        System.out.println("Preprocessing: " + root);
593        DeploymentPreprocessor processor = new DeploymentPreprocessor(root);
594        // initialize
595        processor.init();
596        // and predeploy
597        processor.predeploy();
598        System.out.println("Done.");
599    }
600
601}