001/*
002 * (C) Copyright 2006-2010 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Nuxeo - initial API and implementation
018 *     bstefanescu, jcarsique
019 *     Anahide Tchertchian
020 *
021 * $Id$
022 */
023
024package org.nuxeo.runtime.deployment.preprocessor;
025
026import java.io.BufferedInputStream;
027import java.io.File;
028import java.io.IOException;
029import java.io.InputStream;
030import java.net.MalformedURLException;
031import java.net.URL;
032import java.util.ArrayList;
033import java.util.Arrays;
034import java.util.HashMap;
035import java.util.List;
036import java.util.Map;
037import java.util.Properties;
038import java.util.jar.Attributes;
039import java.util.jar.JarFile;
040import java.util.jar.Manifest;
041import java.util.regex.Matcher;
042import java.util.regex.Pattern;
043import java.util.zip.ZipEntry;
044
045import org.apache.commons.logging.Log;
046import org.apache.commons.logging.LogFactory;
047import org.nuxeo.common.collections.DependencyTree;
048import org.nuxeo.common.utils.FileUtils;
049import org.nuxeo.common.utils.JarUtils;
050import org.nuxeo.common.utils.Path;
051import org.nuxeo.common.utils.StringUtils;
052import org.nuxeo.common.xmap.XMap;
053import org.nuxeo.launcher.config.ConfigurationGenerator;
054import org.nuxeo.runtime.deployment.preprocessor.install.CommandContext;
055import org.nuxeo.runtime.deployment.preprocessor.install.CommandContextImpl;
056import org.nuxeo.runtime.deployment.preprocessor.template.TemplateContribution;
057import org.nuxeo.runtime.deployment.preprocessor.template.TemplateParser;
058
059/**
060 * Initializer for the deployment skeleton, taking care of creating templates, aggregating default components before
061 * runtime is started.
062 *
063 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a>
064 */
065public class DeploymentPreprocessor {
066
067    public static final String FRAGMENT_FILE = "OSGI-INF/deployment-fragment.xml";
068
069    public static final String CONTAINER_FILE = "META-INF/nuxeo-preprocessor.xml";
070
071    public static final String CONTAINER_FILE_COMPAT = "OSGI-INF/deployment-container.xml";
072
073    private static final Pattern ARTIFACT_NAME_PATTERN = Pattern.compile("-[0-9]+");
074
075    private static final Log log = LogFactory.getLog(DeploymentPreprocessor.class);
076
077    private final File dir;
078
079    private final XMap xmap;
080
081    private ContainerDescriptor root;
082
083    // map jar names to bundle symbolic ids - WARN: no more used - will be
084    // removed in future,
085    @Deprecated
086    private final Map<String, String> jar2Id = new HashMap<String, String>();
087
088    public DeploymentPreprocessor(File dir) {
089        this.dir = dir;
090        xmap = new XMap();
091        xmap.register(ContainerDescriptor.class);
092        xmap.register(FragmentDescriptor.class);
093    }
094
095    @Deprecated
096    public String getJarId(String jarName) {
097        return jar2Id.get(jarName);
098    }
099
100    public ContainerDescriptor getRootContainer() {
101        return root;
102    }
103
104    public void init() throws IOException {
105        root = getDefaultContainer(dir);
106        if (root != null) {
107            // run container commands
108            init(root);
109        }
110    }
111
112    public void init(File metadata, File[] files) throws IOException {
113        if (metadata == null) {
114            root = getDefaultContainer(dir);
115        } else {
116            root = getContainer(dir, metadata);
117        }
118        if (root != null) {
119            root.files = files;
120            // run container commands
121            init(root);
122        }
123    }
124
125    protected void init(ContainerDescriptor cd) throws IOException {
126        cd.context = new CommandContextImpl(cd.directory);
127        initContextProperties(cd.context);
128        // run container install instructions if any
129        if (cd.install != null) {
130            cd.install.setLogger(log);
131            log.info("Running custom installation for container: " + cd.name);
132            cd.install.exec(cd.context);
133        }
134        if (cd.files != null) {
135            init(cd, cd.files);
136        } else {
137            // scan directories
138            if (cd.directories == null || cd.directories.isEmpty()) {
139                init(cd, dir);
140            } else {
141                for (String dirPath : cd.directories) {
142                    init(cd, new File(dir, dirPath));
143                }
144            }
145        }
146    }
147
148    protected void initContextProperties(CommandContext ctx) {
149        ConfigurationGenerator confGen = new ConfigurationGenerator();
150        confGen.init();
151        Properties props = confGen.getUserConfig();
152        for (String key : props.stringPropertyNames()) {
153            ctx.put(key, props.getProperty(key));
154        }
155    }
156
157    protected void processFile(ContainerDescriptor cd, File file) throws IOException {
158        String fileName = file.getName();
159        FragmentDescriptor fd = null;
160        boolean isBundle = false;
161        if (fileName.endsWith("-fragment.xml")) {
162            fd = getXMLFragment(file);
163        } else if (fileName.endsWith("-fragments.xml")) {
164            // we allow declaring multiple fragments in the same file
165            // this is useful to deploy libraries
166            collectXMLFragments(cd, file);
167            return;
168        } else if (fileName.endsWith(".jar") || fileName.endsWith(".war") || fileName.endsWith(".sar")
169                || fileName.endsWith(".rar")) {
170            isBundle = true;
171            if (file.isDirectory()) {
172                fd = getDirectoryFragment(file);
173            } else {
174                fd = getJARFragment(file);
175            }
176        }
177        // register the fragment if any was found
178        if (fd != null) {
179            fd.fileName = fileName;
180            fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath());
181            cd.fragments.add(fd);
182            if (fd.templates != null) {
183                for (TemplateDescriptor td : fd.templates.values()) {
184                    td.baseDir = file;
185                    cd.templates.put(td.name, td);
186                }
187            }
188        } else if (isBundle) {
189            // create markers - for compatibility with versions < 5.4
190            String name = getSymbolicName(file);
191            if (name != null) {
192                cd.fragments.add(new FragmentDescriptor(name, true));
193            }
194        }
195    }
196
197    protected String getSymbolicName(File file) {
198        Manifest mf = JarUtils.getManifest(file);
199        if (mf != null) {
200            Attributes attrs = mf.getMainAttributes();
201            String id = attrs.getValue("Bundle-SymbolicName");
202            if (id != null) {
203                int p = id.indexOf(';');
204                if (p > -1) { // remove properties part if any
205                    id = id.substring(0, p);
206                }
207                return id;
208            }
209        }
210        return null;
211    }
212
213    protected String getJarArtifactName(String name) {
214        if (name.endsWith(".jar")) {
215            name = name.substring(0, name.length() - 4);
216        }
217        Matcher m = ARTIFACT_NAME_PATTERN.matcher(name);
218        if (m.find()) {
219            name = name.substring(0, m.start());
220        }
221        return name;
222    }
223
224    protected void init(ContainerDescriptor cd, File[] files) throws IOException {
225        for (File file : files) {
226            processFile(cd, file);
227        }
228    }
229
230    protected void init(ContainerDescriptor cd, File dir) throws IOException {
231        log.info("Scanning directory: " + dir.getName());
232        if (!dir.exists()) {
233            log.warn("Directory doesn't exist: " + dir.getPath());
234            return;
235        }
236        // sort input files in alphabetic order -> this way we are sure we get
237        // the same deploying order on all machines.
238        File[] files = dir.listFiles();
239        Arrays.sort(files);
240        init(cd, files);
241    }
242
243    public void predeploy() throws IOException {
244        if (root != null) {
245            predeploy(root);
246        }
247    }
248
249    protected static String listFragmentDescriptor(FragmentDescriptor fd) {
250        return fd.name + " (" + fd.fileName + ")";
251    }
252
253    protected static void printInfo(FragmentRegistry fragments) {
254        List<DependencyTree.Entry<String, FragmentDescriptor>> entries = fragments.getResolvedEntries();
255        StringBuilder buf = new StringBuilder("Preprocessing order: ");
256        for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) {
257            FragmentDescriptor fd = entry.get();
258            if (fd != null && !fd.isMarker()) {
259                buf.append("\n\t");
260                buf.append(listFragmentDescriptor(entry.get()));
261            }
262        }
263        log.info(buf);
264
265        StringBuilder errors = new StringBuilder();
266        List<DependencyTree.Entry<String, FragmentDescriptor>> missing = fragments.getMissingRequirements();
267        for (DependencyTree.Entry<String, FragmentDescriptor> entry : missing) {
268            buf = new StringBuilder("Unknown bundle: ");
269            buf.append(entry.getKey());
270            buf.append(" required by: ");
271            boolean first = true;
272            for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getDependsOnMe()) {
273                if (!first) {
274                    buf.append(", "); // length 2
275                }
276                first = false;
277                buf.append(listFragmentDescriptor(dep.get()));
278            }
279            log.error(buf);
280            errors.append(buf);
281            errors.append("\n");
282        }
283        for (DependencyTree.Entry<String, FragmentDescriptor> entry : fragments.getPendingEntries()) {
284            if (!entry.isRegistered()) {
285                continue;
286            }
287            buf = new StringBuilder("Bundle not preprocessed: ");
288            buf.append(listFragmentDescriptor(entry.get()));
289            buf.append(" waiting for: ");
290            boolean first = true;
291            for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getWaitsFor()) {
292                if (!first) {
293                    buf.append(", "); // length 2
294                }
295                first = false;
296                buf.append(dep.getKey());
297            }
298            log.error(buf);
299            errors.append(buf);
300            errors.append("\n");
301        }
302        if (errors.length() != 0) {
303            // set system property to log startup errors
304            // this is read by AbstractRuntimeService
305            System.setProperty("org.nuxeo.runtime.deployment.errors", errors.toString());
306        }
307    }
308
309    protected static void predeploy(ContainerDescriptor cd) throws IOException {
310        // run installer and register contributions for each fragment
311        List<DependencyTree.Entry<String, FragmentDescriptor>> entries = cd.fragments.getResolvedEntries();
312        printInfo(cd.fragments);
313        for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) {
314            FragmentDescriptor fd = entry.get();
315            if (fd == null || fd.isMarker()) {
316                continue; // should be a marker entry like the "all" one.
317            }
318            cd.context.put("bundle.fileName", fd.filePath);
319            cd.context.put("bundle.shortName", fd.fileName);
320            cd.context.put("bundle", fd.name);
321
322            // execute install instructions if any
323            if (fd.install != null) {
324                fd.install.setLogger(log);
325                log.info("Running custom installation for fragment: " + fd.name);
326                fd.install.exec(cd.context);
327            }
328
329            if (fd.contributions == null) {
330                continue; // no contributions
331            }
332
333            // get fragment contributions and register them
334            for (TemplateContribution tc : fd.contributions) {
335
336                // register template contributions if any
337                // get the target template
338                TemplateDescriptor td = cd.templates.get(tc.getTemplate());
339                if (td != null) {
340                    if (td.baseDir == null) {
341                        td.baseDir = cd.directory;
342                    }
343                    if (td.template == null) { // template not yet compiled
344                        File file = new File(td.baseDir, td.src);
345                        // compile it
346                        td.template = TemplateParser.parse(file);
347                    }
348                } else {
349                    log.warn("No template '" + tc.getTemplate() + "' found for deployment fragment:  " + fd.name);
350                    continue;
351                }
352                // get the marker where contribution should be inserted
353                td.template.update(tc, cd.context);
354            }
355        }
356
357        // process and write templates
358        // fragments where imported. write down templates
359        for (TemplateDescriptor td : cd.templates.values()) {
360            if (td.baseDir == null) {
361                td.baseDir = cd.directory;
362            }
363            // if required process the template even if no contributions were
364            // made
365            if (td.template == null && td.isRequired) {
366                // compile the template
367                File file = new File(td.baseDir, td.src);
368                td.template = TemplateParser.parse(file);
369            }
370            // process the template
371            if (td.template != null) {
372                File file = new File(td.baseDir, td.installPath);
373                file.getParentFile().mkdirs(); // make sure parents exists
374                FileUtils.writeFile(file, td.template.getText());
375            }
376        }
377
378        // process sub containers if any
379        for (ContainerDescriptor subCd : cd.subContainers) {
380            predeploy(subCd);
381        }
382    }
383
384    protected FragmentDescriptor getXMLFragment(File file) throws IOException {
385        URL url;
386        try {
387            url = file.toURI().toURL();
388        } catch (MalformedURLException e) {
389            throw new RuntimeException(e);
390        }
391        FragmentDescriptor fd = (FragmentDescriptor) xmap.load(url);
392        if (fd != null && fd.name == null) {
393            fd.name = file.getName();
394        }
395        return fd;
396    }
397
398    protected void collectXMLFragments(ContainerDescriptor cd, File file) throws IOException {
399        String fileName = file.getName();
400        URL url;
401        try {
402            url = file.toURI().toURL();
403        } catch (MalformedURLException e) {
404            throw new RuntimeException(e);
405        }
406        Object[] result = xmap.loadAll(url);
407        for (Object entry : result) {
408            FragmentDescriptor fd = (FragmentDescriptor) entry;
409            assert fd != null;
410            if (fd.name == null) {
411                log.error("Invalid fragments file: " + file.getName()
412                        + ". Fragments declared in a -fragments.xml file must have names.");
413            } else {
414                cd.fragments.add(fd);
415                fd.fileName = fileName;
416                fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath());
417            }
418        }
419    }
420
421    protected void processBundleForCompat(FragmentDescriptor fd, File file) {
422        // TODO disable for now the warning
423        log.warn("Entering compatibility mode - Please update the deployment-fragment.xml in " + file.getName()
424                + " to use new dependency management");
425        Manifest mf = JarUtils.getManifest(file);
426        if (mf != null) {
427            fd.name = file.getName();
428            processManifest(fd, fd.name, mf);
429        } else {
430            throw new RuntimeException("Compat: Fragments without a name must reside in an OSGi bundle");
431        }
432    }
433
434    protected FragmentDescriptor getDirectoryFragment(File directory) throws IOException {
435        FragmentDescriptor fd = null;
436        File file = new File(directory.getAbsolutePath() + '/' + FRAGMENT_FILE);
437        if (file.isFile()) {
438            URL url;
439            try {
440                url = file.toURI().toURL();
441            } catch (MalformedURLException e) {
442                throw new RuntimeException(e);
443            }
444            fd = (FragmentDescriptor) xmap.load(url);
445        } else {
446            return null; // don't need preprocessing
447        }
448        if (fd.name == null) {
449            // fallback on symbolic name
450            fd.name = getSymbolicName(directory);
451        }
452        if (fd.name == null) {
453            // fallback on artifact id
454            fd.name = getJarArtifactName(directory.getName());
455        }
456        if (fd.version == 0) { // compat with versions < 5.4
457            processBundleForCompat(fd, directory);
458        }
459        return fd;
460    }
461
462    protected FragmentDescriptor getJARFragment(File file) throws IOException {
463        FragmentDescriptor fd = null;
464        JarFile jar = new JarFile(file);
465        try {
466            ZipEntry ze = jar.getEntry(FRAGMENT_FILE);
467            if (ze != null) {
468                InputStream in = new BufferedInputStream(jar.getInputStream(ze));
469                try {
470                    fd = (FragmentDescriptor) xmap.load(in);
471                } finally {
472                    in.close();
473                }
474                if (fd.name == null) {
475                    // fallback on symbolic name
476                    fd.name = getSymbolicName(file);
477                }
478                if (fd.name == null) {
479                    // fallback on artifact id
480                    fd.name = getJarArtifactName(file.getName());
481                }
482                if (fd.version == 0) { // compat with versions < 5.4
483                    processBundleForCompat(fd, file);
484                }
485            }
486        } finally {
487            jar.close();
488        }
489        return fd;
490    }
491
492    protected void processManifest(FragmentDescriptor fd, String fileName, Manifest mf) {
493        Attributes attrs = mf.getMainAttributes();
494        String id = attrs.getValue("Bundle-SymbolicName");
495        if (id != null) {
496            int p = id.indexOf(';');
497            if (p > -1) { // remove properties part if any
498                id = id.substring(0, p);
499            }
500            jar2Id.put(fileName, id);
501            fd.name = id;
502            if (fd.requires != null && !fd.requires.isEmpty()) {
503                throw new RuntimeException(
504                        "In compatibility mode you must not use <require> tags for OSGi bundles - use Require-Bundle manifest header instead. Bundle: "
505                                + fileName);
506            }
507            // needed to control start-up order (which differs from
508            // Require-Bundle)
509            String requires = attrs.getValue("Nuxeo-Require");
510            if (requires == null) { // if not specific requirement is met use
511                                    // Require-Bundle
512                requires = attrs.getValue("Require-Bundle");
513            }
514            if (requires != null) {
515                String[] ids = StringUtils.split(requires, ',', true);
516                fd.requires = new ArrayList<String>(ids.length);
517                for (int i = 0; i < ids.length; i++) {
518                    String rid = ids[i];
519                    p = rid.indexOf(';');
520                    if (p > -1) { // remove properties part if any
521                        ids[i] = rid.substring(0, p);
522                    }
523                    fd.requires.add(ids[i]);
524                }
525            }
526
527            String requiredBy = attrs.getValue("Nuxeo-RequiredBy");
528            if (requiredBy != null) {
529                String[] ids = StringUtils.split(requiredBy, ',', true);
530                for (int i = 0; i < ids.length; i++) {
531                    String rid = ids[i];
532                    p = rid.indexOf(';');
533                    if (p > -1) { // remove properties part if any
534                        ids[i] = rid.substring(0, p);
535                    }
536                }
537                fd.requiredBy = ids;
538            }
539
540        } else {
541            jar2Id.put(fileName, fd.name);
542        }
543    }
544
545    /**
546     * Reads a container fragment metadata file and returns the container descriptor.
547     */
548    protected ContainerDescriptor getContainer(File home, File file) throws IOException {
549        URL url;
550        try {
551            url = file.toURI().toURL();
552        } catch (MalformedURLException e) {
553            throw new RuntimeException(e);
554        }
555        ContainerDescriptor cd = (ContainerDescriptor) xmap.load(url);
556        if (cd != null) {
557            cd.directory = home;
558            if (cd.name == null) {
559                cd.name = home.getName();
560            }
561        }
562        return cd;
563    }
564
565    protected ContainerDescriptor getDefaultContainer(File directory) throws IOException {
566        File file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE);
567        if (!file.isFile()) {
568            file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE_COMPAT);
569        }
570        ContainerDescriptor cd = null;
571        if (file.isFile()) {
572            cd = getContainer(directory, file);
573        }
574        return cd;
575    }
576
577    public static String getRelativeChildPath(String parent, String child) {
578        // TODO optimize this method
579        // fix win32 case
580        if (parent.indexOf('\\') > -1) {
581            parent = parent.replace('\\', '/');
582        }
583        if (child.indexOf('\\') > -1) {
584            child = child.replace('\\', '/');
585        } // end fix win32
586        Path parentPath = new Path(parent);
587        Path childPath = new Path(child);
588        if (parentPath.isPrefixOf(childPath)) {
589            return childPath.removeFirstSegments(parentPath.segmentCount()).makeRelative().toString();
590        }
591        return null;
592    }
593
594    /**
595     * Run preprocessing in the given home directory and using the given list of bundles. Bundles must be ordered by the
596     * caller to have same deployment order on all computers.
597     * <p>
598     * The metadata file is the metadat file to be used to configure the processor. If null the default location will be
599     * used (relative to home): {@link #CONTAINER_FILE}.
600     */
601    public static void process(File home, File metadata, File[] files) throws IOException {
602        DeploymentPreprocessor processor = new DeploymentPreprocessor(home);
603        // initialize
604        processor.init(metadata, files);
605        // run preprocessor
606        processor.predeploy();
607    }
608
609    public static void main(String[] args) throws IOException {
610        File root;
611        if (args.length > 0) {
612            root = new File(args[0]);
613        } else {
614            root = new File(".");
615        }
616        System.out.println("Preprocessing: " + root);
617        DeploymentPreprocessor processor = new DeploymentPreprocessor(root);
618        // initialize
619        processor.init();
620        // and predeploy
621        processor.predeploy();
622        System.out.println("Done.");
623    }
624
625}