001/*
002 * (C) Copyright 2006-2010 Nuxeo SAS (http://nuxeo.com/) and contributors.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the GNU Lesser General Public License
006 * (LGPL) version 2.1 which accompanies this distribution, and is available at
007 * http://www.gnu.org/licenses/lgpl.html
008 *
009 * This library is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 * Lesser General Public License for more details.
013 *
014 * Contributors:
015 *     Nuxeo - initial API and implementation
016 *     bstefanescu, jcarsique
017 *     Anahide Tchertchian
018 *
019 * $Id$
020 */
021
022package org.nuxeo.runtime.deployment.preprocessor;
023
024import java.io.BufferedInputStream;
025import java.io.File;
026import java.io.IOException;
027import java.io.InputStream;
028import java.net.MalformedURLException;
029import java.net.URL;
030import java.util.ArrayList;
031import java.util.Arrays;
032import java.util.HashMap;
033import java.util.List;
034import java.util.Map;
035import java.util.Properties;
036import java.util.jar.Attributes;
037import java.util.jar.JarFile;
038import java.util.jar.Manifest;
039import java.util.regex.Matcher;
040import java.util.regex.Pattern;
041import java.util.zip.ZipEntry;
042
043import org.apache.commons.logging.Log;
044import org.apache.commons.logging.LogFactory;
045import org.nuxeo.common.collections.DependencyTree;
046import org.nuxeo.common.utils.FileUtils;
047import org.nuxeo.common.utils.JarUtils;
048import org.nuxeo.common.utils.Path;
049import org.nuxeo.common.utils.StringUtils;
050import org.nuxeo.common.xmap.XMap;
051import org.nuxeo.launcher.config.ConfigurationGenerator;
052import org.nuxeo.runtime.deployment.preprocessor.install.CommandContext;
053import org.nuxeo.runtime.deployment.preprocessor.install.CommandContextImpl;
054import org.nuxeo.runtime.deployment.preprocessor.template.TemplateContribution;
055import org.nuxeo.runtime.deployment.preprocessor.template.TemplateParser;
056
057/**
058 * Initializer for the deployment skeleton, taking care of creating templates, aggregating default components before
059 * runtime is started.
060 *
061 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a>
062 */
063public class DeploymentPreprocessor {
064
065    public static final String FRAGMENT_FILE = "OSGI-INF/deployment-fragment.xml";
066
067    public static final String CONTAINER_FILE = "META-INF/nuxeo-preprocessor.xml";
068
069    public static final String CONTAINER_FILE_COMPAT = "OSGI-INF/deployment-container.xml";
070
071    private static final Pattern ARTIFACT_NAME_PATTERN = Pattern.compile("-[0-9]+");
072
073    private static final Log log = LogFactory.getLog(DeploymentPreprocessor.class);
074
075    private final File dir;
076
077    private final XMap xmap;
078
079    private ContainerDescriptor root;
080
081    // map jar names to bundle symbolic ids - WARN: no more used - will be
082    // removed in future,
083    @Deprecated
084    private final Map<String, String> jar2Id = new HashMap<String, String>();
085
086    public DeploymentPreprocessor(File dir) {
087        this.dir = dir;
088        xmap = new XMap();
089        xmap.register(ContainerDescriptor.class);
090        xmap.register(FragmentDescriptor.class);
091    }
092
093    @Deprecated
094    public String getJarId(String jarName) {
095        return jar2Id.get(jarName);
096    }
097
098    public ContainerDescriptor getRootContainer() {
099        return root;
100    }
101
102    public void init() throws IOException {
103        root = getDefaultContainer(dir);
104        if (root != null) {
105            // run container commands
106            init(root);
107        }
108    }
109
110    public void init(File metadata, File[] files) throws IOException {
111        if (metadata == null) {
112            root = getDefaultContainer(dir);
113        } else {
114            root = getContainer(dir, metadata);
115        }
116        if (root != null) {
117            root.files = files;
118            // run container commands
119            init(root);
120        }
121    }
122
123    protected void init(ContainerDescriptor cd) throws IOException {
124        cd.context = new CommandContextImpl(cd.directory);
125        initContextProperties(cd.context);
126        // run container install instructions if any
127        if (cd.install != null) {
128            cd.install.setLogger(log);
129            log.info("Running custom installation for container: " + cd.name);
130            cd.install.exec(cd.context);
131        }
132        if (cd.files != null) {
133            init(cd, cd.files);
134        } else {
135            // scan directories
136            if (cd.directories == null || cd.directories.isEmpty()) {
137                init(cd, dir);
138            } else {
139                for (String dirPath : cd.directories) {
140                    init(cd, new File(dir, dirPath));
141                }
142            }
143        }
144    }
145
146    protected void initContextProperties(CommandContext ctx) {
147        ConfigurationGenerator confGen = new ConfigurationGenerator();
148        confGen.init();
149        Properties props = confGen.getUserConfig();
150        for (String key : props.stringPropertyNames()) {
151            ctx.put(key, props.getProperty(key));
152        }
153    }
154
155    protected void processFile(ContainerDescriptor cd, File file) throws IOException {
156        String fileName = file.getName();
157        FragmentDescriptor fd = null;
158        boolean isBundle = false;
159        if (fileName.endsWith("-fragment.xml")) {
160            fd = getXMLFragment(file);
161        } else if (fileName.endsWith("-fragments.xml")) {
162            // we allow declaring multiple fragments in the same file
163            // this is useful to deploy libraries
164            collectXMLFragments(cd, file);
165            return;
166        } else if (fileName.endsWith(".jar") || fileName.endsWith(".war") || fileName.endsWith(".sar")
167                || fileName.endsWith(".rar")) {
168            isBundle = true;
169            if (file.isDirectory()) {
170                fd = getDirectoryFragment(file);
171            } else {
172                fd = getJARFragment(file);
173            }
174        }
175        // register the fragment if any was found
176        if (fd != null) {
177            fd.fileName = fileName;
178            fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath());
179            cd.fragments.add(fd);
180            if (fd.templates != null) {
181                for (TemplateDescriptor td : fd.templates.values()) {
182                    td.baseDir = file;
183                    cd.templates.put(td.name, td);
184                }
185            }
186        } else if (isBundle) {
187            // create markers - for compatibility with versions < 5.4
188            String name = getSymbolicName(file);
189            if (name != null) {
190                cd.fragments.add(new FragmentDescriptor(name, true));
191            }
192        }
193    }
194
195    protected String getSymbolicName(File file) {
196        Manifest mf = JarUtils.getManifest(file);
197        if (mf != null) {
198            Attributes attrs = mf.getMainAttributes();
199            String id = attrs.getValue("Bundle-SymbolicName");
200            if (id != null) {
201                int p = id.indexOf(';');
202                if (p > -1) { // remove properties part if any
203                    id = id.substring(0, p);
204                }
205                return id;
206            }
207        }
208        return null;
209    }
210
211    protected String getJarArtifactName(String name) {
212        if (name.endsWith(".jar")) {
213            name = name.substring(0, name.length() - 4);
214        }
215        Matcher m = ARTIFACT_NAME_PATTERN.matcher(name);
216        if (m.find()) {
217            name = name.substring(0, m.start());
218        }
219        return name;
220    }
221
222    protected void init(ContainerDescriptor cd, File[] files) throws IOException {
223        for (File file : files) {
224            processFile(cd, file);
225        }
226    }
227
228    protected void init(ContainerDescriptor cd, File dir) throws IOException {
229        log.info("Scanning directory: " + dir.getName());
230        if (!dir.exists()) {
231            log.warn("Directory doesn't exist: " + dir.getPath());
232            return;
233        }
234        // sort input files in alphabetic order -> this way we are sure we get
235        // the same deploying order on all machines.
236        File[] files = dir.listFiles();
237        Arrays.sort(files);
238        init(cd, files);
239    }
240
241    public void predeploy() throws IOException {
242        if (root != null) {
243            predeploy(root);
244        }
245    }
246
247    protected static String listFragmentDescriptor(FragmentDescriptor fd) {
248        return fd.name + " (" + fd.fileName + ")";
249    }
250
251    protected static void printInfo(FragmentRegistry fragments) {
252        List<DependencyTree.Entry<String, FragmentDescriptor>> entries = fragments.getResolvedEntries();
253        StringBuilder buf = new StringBuilder("Preprocessing order: ");
254        for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) {
255            FragmentDescriptor fd = entry.get();
256            if (fd != null && !fd.isMarker()) {
257                buf.append("\n\t");
258                buf.append(listFragmentDescriptor(entry.get()));
259            }
260        }
261        log.info(buf);
262
263        StringBuilder errors = new StringBuilder();
264        List<DependencyTree.Entry<String, FragmentDescriptor>> missing = fragments.getMissingRequirements();
265        for (DependencyTree.Entry<String, FragmentDescriptor> entry : missing) {
266            buf = new StringBuilder("Unknown bundle: ");
267            buf.append(entry.getKey());
268            buf.append(" required by: ");
269            boolean first = true;
270            for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getDependsOnMe()) {
271                if (!first) {
272                    buf.append(", "); // length 2
273                }
274                first = false;
275                buf.append(listFragmentDescriptor(dep.get()));
276            }
277            log.error(buf);
278            errors.append(buf);
279            errors.append("\n");
280        }
281        for (DependencyTree.Entry<String, FragmentDescriptor> entry : fragments.getPendingEntries()) {
282            if (!entry.isRegistered()) {
283                continue;
284            }
285            buf = new StringBuilder("Bundle not preprocessed: ");
286            buf.append(listFragmentDescriptor(entry.get()));
287            buf.append(" waiting for: ");
288            boolean first = true;
289            for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getWaitsFor()) {
290                if (!first) {
291                    buf.append(", "); // length 2
292                }
293                first = false;
294                buf.append(dep.getKey());
295            }
296            log.error(buf);
297            errors.append(buf);
298            errors.append("\n");
299        }
300        if (errors.length() != 0) {
301            // set system property to log startup errors
302            // this is read by AbstractRuntimeService
303            System.setProperty("org.nuxeo.runtime.deployment.errors", errors.toString());
304        }
305    }
306
307    protected static void predeploy(ContainerDescriptor cd) throws IOException {
308        // run installer and register contributions for each fragment
309        List<DependencyTree.Entry<String, FragmentDescriptor>> entries = cd.fragments.getResolvedEntries();
310        printInfo(cd.fragments);
311        for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) {
312            FragmentDescriptor fd = entry.get();
313            if (fd == null || fd.isMarker()) {
314                continue; // should be a marker entry like the "all" one.
315            }
316            cd.context.put("bundle.fileName", fd.filePath);
317            cd.context.put("bundle.shortName", fd.fileName);
318            cd.context.put("bundle", fd.name);
319
320            // execute install instructions if any
321            if (fd.install != null) {
322                fd.install.setLogger(log);
323                log.info("Running custom installation for fragment: " + fd.name);
324                fd.install.exec(cd.context);
325            }
326
327            if (fd.contributions == null) {
328                continue; // no contributions
329            }
330
331            // get fragment contributions and register them
332            for (TemplateContribution tc : fd.contributions) {
333
334                // register template contributions if any
335                // get the target template
336                TemplateDescriptor td = cd.templates.get(tc.getTemplate());
337                if (td != null) {
338                    if (td.baseDir == null) {
339                        td.baseDir = cd.directory;
340                    }
341                    if (td.template == null) { // template not yet compiled
342                        File file = new File(td.baseDir, td.src);
343                        // compile it
344                        td.template = TemplateParser.parse(file);
345                    }
346                } else {
347                    log.warn("No template '" + tc.getTemplate() + "' found for deployment fragment:  " + fd.name);
348                    continue;
349                }
350                // get the marker where contribution should be inserted
351                td.template.update(tc, cd.context);
352            }
353        }
354
355        // process and write templates
356        // fragments where imported. write down templates
357        for (TemplateDescriptor td : cd.templates.values()) {
358            if (td.baseDir == null) {
359                td.baseDir = cd.directory;
360            }
361            // if required process the template even if no contributions were
362            // made
363            if (td.template == null && td.isRequired) {
364                // compile the template
365                File file = new File(td.baseDir, td.src);
366                td.template = TemplateParser.parse(file);
367            }
368            // process the template
369            if (td.template != null) {
370                File file = new File(td.baseDir, td.installPath);
371                file.getParentFile().mkdirs(); // make sure parents exists
372                FileUtils.writeFile(file, td.template.getText());
373            }
374        }
375
376        // process sub containers if any
377        for (ContainerDescriptor subCd : cd.subContainers) {
378            predeploy(subCd);
379        }
380    }
381
382    protected FragmentDescriptor getXMLFragment(File file) throws IOException {
383        URL url;
384        try {
385            url = file.toURI().toURL();
386        } catch (MalformedURLException e) {
387            throw new RuntimeException(e);
388        }
389        FragmentDescriptor fd = (FragmentDescriptor) xmap.load(url);
390        if (fd != null && fd.name == null) {
391            fd.name = file.getName();
392        }
393        return fd;
394    }
395
396    protected void collectXMLFragments(ContainerDescriptor cd, File file) throws IOException {
397        String fileName = file.getName();
398        URL url;
399        try {
400            url = file.toURI().toURL();
401        } catch (MalformedURLException e) {
402            throw new RuntimeException(e);
403        }
404        Object[] result = xmap.loadAll(url);
405        for (Object entry : result) {
406            FragmentDescriptor fd = (FragmentDescriptor) entry;
407            assert fd != null;
408            if (fd.name == null) {
409                log.error("Invalid fragments file: " + file.getName()
410                        + ". Fragments declared in a -fragments.xml file must have names.");
411            } else {
412                cd.fragments.add(fd);
413                fd.fileName = fileName;
414                fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath());
415            }
416        }
417    }
418
419    protected void processBundleForCompat(FragmentDescriptor fd, File file) {
420        // TODO disable for now the warning
421        log.warn("Entering compatibility mode - Please update the deployment-fragment.xml in " + file.getName()
422                + " to use new dependency management");
423        Manifest mf = JarUtils.getManifest(file);
424        if (mf != null) {
425            fd.name = file.getName();
426            processManifest(fd, fd.name, mf);
427        } else {
428            throw new RuntimeException("Compat: Fragments without a name must reside in an OSGi bundle");
429        }
430    }
431
432    protected FragmentDescriptor getDirectoryFragment(File directory) throws IOException {
433        FragmentDescriptor fd = null;
434        File file = new File(directory.getAbsolutePath() + '/' + FRAGMENT_FILE);
435        if (file.isFile()) {
436            URL url;
437            try {
438                url = file.toURI().toURL();
439            } catch (MalformedURLException e) {
440                throw new RuntimeException(e);
441            }
442            fd = (FragmentDescriptor) xmap.load(url);
443        } else {
444            return null; // don't need preprocessing
445        }
446        if (fd.name == null) {
447            // fallback on symbolic name
448            fd.name = getSymbolicName(directory);
449        }
450        if (fd.name == null) {
451            // fallback on artifact id
452            fd.name = getJarArtifactName(directory.getName());
453        }
454        if (fd.version == 0) { // compat with versions < 5.4
455            processBundleForCompat(fd, directory);
456        }
457        return fd;
458    }
459
460    protected FragmentDescriptor getJARFragment(File file) throws IOException {
461        FragmentDescriptor fd = null;
462        JarFile jar = new JarFile(file);
463        try {
464            ZipEntry ze = jar.getEntry(FRAGMENT_FILE);
465            if (ze != null) {
466                InputStream in = new BufferedInputStream(jar.getInputStream(ze));
467                try {
468                    fd = (FragmentDescriptor) xmap.load(in);
469                } finally {
470                    in.close();
471                }
472                if (fd.name == null) {
473                    // fallback on symbolic name
474                    fd.name = getSymbolicName(file);
475                }
476                if (fd.name == null) {
477                    // fallback on artifact id
478                    fd.name = getJarArtifactName(file.getName());
479                }
480                if (fd.version == 0) { // compat with versions < 5.4
481                    processBundleForCompat(fd, file);
482                }
483            }
484        } finally {
485            jar.close();
486        }
487        return fd;
488    }
489
490    protected void processManifest(FragmentDescriptor fd, String fileName, Manifest mf) {
491        Attributes attrs = mf.getMainAttributes();
492        String id = attrs.getValue("Bundle-SymbolicName");
493        if (id != null) {
494            int p = id.indexOf(';');
495            if (p > -1) { // remove properties part if any
496                id = id.substring(0, p);
497            }
498            jar2Id.put(fileName, id);
499            fd.name = id;
500            if (fd.requires != null && !fd.requires.isEmpty()) {
501                throw new RuntimeException(
502                        "In compatibility mode you must not use <require> tags for OSGi bundles - use Require-Bundle manifest header instead. Bundle: "
503                                + fileName);
504            }
505            // needed to control start-up order (which differs from
506            // Require-Bundle)
507            String requires = attrs.getValue("Nuxeo-Require");
508            if (requires == null) { // if not specific requirement is met use
509                                    // Require-Bundle
510                requires = attrs.getValue("Require-Bundle");
511            }
512            if (requires != null) {
513                String[] ids = StringUtils.split(requires, ',', true);
514                fd.requires = new ArrayList<String>(ids.length);
515                for (int i = 0; i < ids.length; i++) {
516                    String rid = ids[i];
517                    p = rid.indexOf(';');
518                    if (p > -1) { // remove properties part if any
519                        ids[i] = rid.substring(0, p);
520                    }
521                    fd.requires.add(ids[i]);
522                }
523            }
524
525            String requiredBy = attrs.getValue("Nuxeo-RequiredBy");
526            if (requiredBy != null) {
527                String[] ids = StringUtils.split(requiredBy, ',', true);
528                for (int i = 0; i < ids.length; i++) {
529                    String rid = ids[i];
530                    p = rid.indexOf(';');
531                    if (p > -1) { // remove properties part if any
532                        ids[i] = rid.substring(0, p);
533                    }
534                }
535                fd.requiredBy = ids;
536            }
537
538        } else {
539            jar2Id.put(fileName, fd.name);
540        }
541    }
542
543    /**
544     * Reads a container fragment metadata file and returns the container descriptor.
545     */
546    protected ContainerDescriptor getContainer(File home, File file) throws IOException {
547        URL url;
548        try {
549            url = file.toURI().toURL();
550        } catch (MalformedURLException e) {
551            throw new RuntimeException(e);
552        }
553        ContainerDescriptor cd = (ContainerDescriptor) xmap.load(url);
554        if (cd != null) {
555            cd.directory = home;
556            if (cd.name == null) {
557                cd.name = home.getName();
558            }
559        }
560        return cd;
561    }
562
563    protected ContainerDescriptor getDefaultContainer(File directory) throws IOException {
564        File file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE);
565        if (!file.isFile()) {
566            file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE_COMPAT);
567        }
568        ContainerDescriptor cd = null;
569        if (file.isFile()) {
570            cd = getContainer(directory, file);
571        }
572        return cd;
573    }
574
575    public static String getRelativeChildPath(String parent, String child) {
576        // TODO optimize this method
577        // fix win32 case
578        if (parent.indexOf('\\') > -1) {
579            parent = parent.replace('\\', '/');
580        }
581        if (child.indexOf('\\') > -1) {
582            child = child.replace('\\', '/');
583        } // end fix win32
584        Path parentPath = new Path(parent);
585        Path childPath = new Path(child);
586        if (parentPath.isPrefixOf(childPath)) {
587            return childPath.removeFirstSegments(parentPath.segmentCount()).makeRelative().toString();
588        }
589        return null;
590    }
591
592    /**
593     * Run preprocessing in the given home directory and using the given list of bundles. Bundles must be ordered by the
594     * caller to have same deployment order on all computers.
595     * <p>
596     * The metadata file is the metadat file to be used to configure the processor. If null the default location will be
597     * used (relative to home): {@link #CONTAINER_FILE}.
598     */
599    public static void process(File home, File metadata, File[] files) throws IOException {
600        DeploymentPreprocessor processor = new DeploymentPreprocessor(home);
601        // initialize
602        processor.init(metadata, files);
603        // run preprocessor
604        processor.predeploy();
605    }
606
607    public static void main(String[] args) throws IOException {
608        File root;
609        if (args.length > 0) {
610            root = new File(args[0]);
611        } else {
612            root = new File(".");
613        }
614        System.out.println("Preprocessing: " + root);
615        DeploymentPreprocessor processor = new DeploymentPreprocessor(root);
616        // initialize
617        processor.init();
618        // and predeploy
619        processor.predeploy();
620        System.out.println("Done.");
621    }
622
623}