001/* 002 * (C) Copyright 2006-2010 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Nuxeo - initial API and implementation 018 * bstefanescu, jcarsique 019 * Anahide Tchertchian 020 * 021 * $Id$ 022 */ 023 024package org.nuxeo.runtime.deployment.preprocessor; 025 026import java.io.BufferedInputStream; 027import java.io.File; 028import java.io.IOException; 029import java.io.InputStream; 030import java.net.MalformedURLException; 031import java.net.URL; 032import java.util.ArrayList; 033import java.util.Arrays; 034import java.util.HashMap; 035import java.util.List; 036import java.util.Map; 037import java.util.Properties; 038import java.util.jar.Attributes; 039import java.util.jar.JarFile; 040import java.util.jar.Manifest; 041import java.util.regex.Matcher; 042import java.util.regex.Pattern; 043import java.util.zip.ZipEntry; 044 045import org.apache.commons.logging.Log; 046import org.apache.commons.logging.LogFactory; 047import org.nuxeo.common.collections.DependencyTree; 048import org.nuxeo.common.utils.FileUtils; 049import org.nuxeo.common.utils.JarUtils; 050import org.nuxeo.common.utils.Path; 051import org.nuxeo.common.utils.StringUtils; 052import org.nuxeo.common.xmap.XMap; 053import org.nuxeo.launcher.config.ConfigurationGenerator; 054import org.nuxeo.runtime.deployment.preprocessor.install.CommandContext; 055import org.nuxeo.runtime.deployment.preprocessor.install.CommandContextImpl; 056import org.nuxeo.runtime.deployment.preprocessor.template.TemplateContribution; 057import org.nuxeo.runtime.deployment.preprocessor.template.TemplateParser; 058 059/** 060 * Initializer for the deployment skeleton, taking care of creating templates, aggregating default components before 061 * runtime is started. 062 * 063 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a> 064 */ 065public class DeploymentPreprocessor { 066 067 public static final String FRAGMENT_FILE = "OSGI-INF/deployment-fragment.xml"; 068 069 public static final String CONTAINER_FILE = "META-INF/nuxeo-preprocessor.xml"; 070 071 public static final String CONTAINER_FILE_COMPAT = "OSGI-INF/deployment-container.xml"; 072 073 private static final Pattern ARTIFACT_NAME_PATTERN = Pattern.compile("-[0-9]+"); 074 075 private static final Log log = LogFactory.getLog(DeploymentPreprocessor.class); 076 077 private final File dir; 078 079 private final XMap xmap; 080 081 private ContainerDescriptor root; 082 083 // map jar names to bundle symbolic ids - WARN: no more used - will be 084 // removed in future, 085 @Deprecated 086 private final Map<String, String> jar2Id = new HashMap<String, String>(); 087 088 public DeploymentPreprocessor(File dir) { 089 this.dir = dir; 090 xmap = new XMap(); 091 xmap.register(ContainerDescriptor.class); 092 xmap.register(FragmentDescriptor.class); 093 } 094 095 @Deprecated 096 public String getJarId(String jarName) { 097 return jar2Id.get(jarName); 098 } 099 100 public ContainerDescriptor getRootContainer() { 101 return root; 102 } 103 104 public void init() throws IOException { 105 root = getDefaultContainer(dir); 106 if (root != null) { 107 // run container commands 108 init(root); 109 } 110 } 111 112 public void init(File metadata, File[] files) throws IOException { 113 if (metadata == null) { 114 root = getDefaultContainer(dir); 115 } else { 116 root = getContainer(dir, metadata); 117 } 118 if (root != null) { 119 root.files = files; 120 // run container commands 121 init(root); 122 } 123 } 124 125 protected void init(ContainerDescriptor cd) throws IOException { 126 cd.context = new CommandContextImpl(cd.directory); 127 initContextProperties(cd.context); 128 // run container install instructions if any 129 if (cd.install != null) { 130 cd.install.setLogger(log); 131 log.info("Running custom installation for container: " + cd.name); 132 cd.install.exec(cd.context); 133 } 134 if (cd.files != null) { 135 init(cd, cd.files); 136 } else { 137 // scan directories 138 if (cd.directories == null || cd.directories.isEmpty()) { 139 init(cd, dir); 140 } else { 141 for (String dirPath : cd.directories) { 142 init(cd, new File(dir, dirPath)); 143 } 144 } 145 } 146 } 147 148 protected void initContextProperties(CommandContext ctx) { 149 ConfigurationGenerator confGen = new ConfigurationGenerator(); 150 confGen.init(); 151 Properties props = confGen.getUserConfig(); 152 for (String key : props.stringPropertyNames()) { 153 ctx.put(key, props.getProperty(key)); 154 } 155 } 156 157 protected void processFile(ContainerDescriptor cd, File file) throws IOException { 158 String fileName = file.getName(); 159 FragmentDescriptor fd = null; 160 boolean isBundle = false; 161 if (fileName.endsWith("-fragment.xml")) { 162 fd = getXMLFragment(file); 163 } else if (fileName.endsWith("-fragments.xml")) { 164 // we allow declaring multiple fragments in the same file 165 // this is useful to deploy libraries 166 collectXMLFragments(cd, file); 167 return; 168 } else if (fileName.endsWith(".jar") || fileName.endsWith(".war") || fileName.endsWith(".sar") 169 || fileName.endsWith(".rar")) { 170 isBundle = true; 171 if (file.isDirectory()) { 172 fd = getDirectoryFragment(file); 173 } else { 174 fd = getJARFragment(file); 175 } 176 } 177 // register the fragment if any was found 178 if (fd != null) { 179 fd.fileName = fileName; 180 fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath()); 181 cd.fragments.add(fd); 182 if (fd.templates != null) { 183 for (TemplateDescriptor td : fd.templates.values()) { 184 td.baseDir = file; 185 cd.templates.put(td.name, td); 186 } 187 } 188 } else if (isBundle) { 189 // create markers - for compatibility with versions < 5.4 190 String name = getSymbolicName(file); 191 if (name != null) { 192 cd.fragments.add(new FragmentDescriptor(name, true)); 193 } 194 } 195 } 196 197 protected String getSymbolicName(File file) { 198 Manifest mf = JarUtils.getManifest(file); 199 if (mf != null) { 200 Attributes attrs = mf.getMainAttributes(); 201 String id = attrs.getValue("Bundle-SymbolicName"); 202 if (id != null) { 203 int p = id.indexOf(';'); 204 if (p > -1) { // remove properties part if any 205 id = id.substring(0, p); 206 } 207 return id; 208 } 209 } 210 return null; 211 } 212 213 protected String getJarArtifactName(String name) { 214 if (name.endsWith(".jar")) { 215 name = name.substring(0, name.length() - 4); 216 } 217 Matcher m = ARTIFACT_NAME_PATTERN.matcher(name); 218 if (m.find()) { 219 name = name.substring(0, m.start()); 220 } 221 return name; 222 } 223 224 protected void init(ContainerDescriptor cd, File[] files) throws IOException { 225 for (File file : files) { 226 processFile(cd, file); 227 } 228 } 229 230 protected void init(ContainerDescriptor cd, File dir) throws IOException { 231 log.info("Scanning directory: " + dir.getName()); 232 if (!dir.exists()) { 233 log.warn("Directory doesn't exist: " + dir.getPath()); 234 return; 235 } 236 // sort input files in alphabetic order -> this way we are sure we get 237 // the same deploying order on all machines. 238 File[] files = dir.listFiles(); 239 Arrays.sort(files); 240 init(cd, files); 241 } 242 243 public void predeploy() throws IOException { 244 if (root != null) { 245 predeploy(root); 246 } 247 } 248 249 protected static String listFragmentDescriptor(FragmentDescriptor fd) { 250 return fd.name + " (" + fd.fileName + ")"; 251 } 252 253 protected static void printInfo(FragmentRegistry fragments) { 254 List<DependencyTree.Entry<String, FragmentDescriptor>> entries = fragments.getResolvedEntries(); 255 StringBuilder buf = new StringBuilder("Preprocessing order: "); 256 for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) { 257 FragmentDescriptor fd = entry.get(); 258 if (fd != null && !fd.isMarker()) { 259 buf.append("\n\t"); 260 buf.append(listFragmentDescriptor(entry.get())); 261 } 262 } 263 log.info(buf); 264 265 StringBuilder errors = new StringBuilder(); 266 List<DependencyTree.Entry<String, FragmentDescriptor>> missing = fragments.getMissingRequirements(); 267 for (DependencyTree.Entry<String, FragmentDescriptor> entry : missing) { 268 buf = new StringBuilder("Unknown bundle: "); 269 buf.append(entry.getKey()); 270 buf.append(" required by: "); 271 boolean first = true; 272 for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getDependsOnMe()) { 273 if (!first) { 274 buf.append(", "); // length 2 275 } 276 first = false; 277 buf.append(listFragmentDescriptor(dep.get())); 278 } 279 log.error(buf); 280 errors.append(buf); 281 errors.append("\n"); 282 } 283 for (DependencyTree.Entry<String, FragmentDescriptor> entry : fragments.getPendingEntries()) { 284 if (!entry.isRegistered()) { 285 continue; 286 } 287 buf = new StringBuilder("Bundle not preprocessed: "); 288 buf.append(listFragmentDescriptor(entry.get())); 289 buf.append(" waiting for: "); 290 boolean first = true; 291 for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getWaitsFor()) { 292 if (!first) { 293 buf.append(", "); // length 2 294 } 295 first = false; 296 buf.append(dep.getKey()); 297 } 298 log.error(buf); 299 errors.append(buf); 300 errors.append("\n"); 301 } 302 if (errors.length() != 0) { 303 // set system property to log startup errors 304 // this is read by AbstractRuntimeService 305 System.setProperty("org.nuxeo.runtime.deployment.errors", errors.toString()); 306 } 307 } 308 309 protected static void predeploy(ContainerDescriptor cd) throws IOException { 310 // run installer and register contributions for each fragment 311 List<DependencyTree.Entry<String, FragmentDescriptor>> entries = cd.fragments.getResolvedEntries(); 312 printInfo(cd.fragments); 313 for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) { 314 FragmentDescriptor fd = entry.get(); 315 if (fd == null || fd.isMarker()) { 316 continue; // should be a marker entry like the "all" one. 317 } 318 cd.context.put("bundle.fileName", fd.filePath); 319 cd.context.put("bundle.shortName", fd.fileName); 320 cd.context.put("bundle", fd.name); 321 322 // execute install instructions if any 323 if (fd.install != null) { 324 fd.install.setLogger(log); 325 log.info("Running custom installation for fragment: " + fd.name); 326 fd.install.exec(cd.context); 327 } 328 329 if (fd.contributions == null) { 330 continue; // no contributions 331 } 332 333 // get fragment contributions and register them 334 for (TemplateContribution tc : fd.contributions) { 335 336 // register template contributions if any 337 // get the target template 338 TemplateDescriptor td = cd.templates.get(tc.getTemplate()); 339 if (td != null) { 340 if (td.baseDir == null) { 341 td.baseDir = cd.directory; 342 } 343 if (td.template == null) { // template not yet compiled 344 File file = new File(td.baseDir, td.src); 345 // compile it 346 td.template = TemplateParser.parse(file); 347 } 348 } else { 349 log.warn("No template '" + tc.getTemplate() + "' found for deployment fragment: " + fd.name); 350 continue; 351 } 352 // get the marker where contribution should be inserted 353 td.template.update(tc, cd.context); 354 } 355 } 356 357 // process and write templates 358 // fragments where imported. write down templates 359 for (TemplateDescriptor td : cd.templates.values()) { 360 if (td.baseDir == null) { 361 td.baseDir = cd.directory; 362 } 363 // if required process the template even if no contributions were 364 // made 365 if (td.template == null && td.isRequired) { 366 // compile the template 367 File file = new File(td.baseDir, td.src); 368 td.template = TemplateParser.parse(file); 369 } 370 // process the template 371 if (td.template != null) { 372 File file = new File(td.baseDir, td.installPath); 373 file.getParentFile().mkdirs(); // make sure parents exists 374 FileUtils.writeFile(file, td.template.getText()); 375 } 376 } 377 378 // process sub containers if any 379 for (ContainerDescriptor subCd : cd.subContainers) { 380 predeploy(subCd); 381 } 382 } 383 384 protected FragmentDescriptor getXMLFragment(File file) throws IOException { 385 URL url; 386 try { 387 url = file.toURI().toURL(); 388 } catch (MalformedURLException e) { 389 throw new RuntimeException(e); 390 } 391 FragmentDescriptor fd = (FragmentDescriptor) xmap.load(url); 392 if (fd != null && fd.name == null) { 393 fd.name = file.getName(); 394 } 395 return fd; 396 } 397 398 protected void collectXMLFragments(ContainerDescriptor cd, File file) throws IOException { 399 String fileName = file.getName(); 400 URL url; 401 try { 402 url = file.toURI().toURL(); 403 } catch (MalformedURLException e) { 404 throw new RuntimeException(e); 405 } 406 Object[] result = xmap.loadAll(url); 407 for (Object entry : result) { 408 FragmentDescriptor fd = (FragmentDescriptor) entry; 409 assert fd != null; 410 if (fd.name == null) { 411 log.error("Invalid fragments file: " + file.getName() 412 + ". Fragments declared in a -fragments.xml file must have names."); 413 } else { 414 cd.fragments.add(fd); 415 fd.fileName = fileName; 416 fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath()); 417 } 418 } 419 } 420 421 protected void processBundleForCompat(FragmentDescriptor fd, File file) { 422 // TODO disable for now the warning 423 log.warn("Entering compatibility mode - Please update the deployment-fragment.xml in " + file.getName() 424 + " to use new dependency management"); 425 Manifest mf = JarUtils.getManifest(file); 426 if (mf != null) { 427 fd.name = file.getName(); 428 processManifest(fd, fd.name, mf); 429 } else { 430 throw new RuntimeException("Compat: Fragments without a name must reside in an OSGi bundle"); 431 } 432 } 433 434 protected FragmentDescriptor getDirectoryFragment(File directory) throws IOException { 435 FragmentDescriptor fd = null; 436 File file = new File(directory.getAbsolutePath() + '/' + FRAGMENT_FILE); 437 if (file.isFile()) { 438 URL url; 439 try { 440 url = file.toURI().toURL(); 441 } catch (MalformedURLException e) { 442 throw new RuntimeException(e); 443 } 444 fd = (FragmentDescriptor) xmap.load(url); 445 } else { 446 return null; // don't need preprocessing 447 } 448 if (fd.name == null) { 449 // fallback on symbolic name 450 fd.name = getSymbolicName(directory); 451 } 452 if (fd.name == null) { 453 // fallback on artifact id 454 fd.name = getJarArtifactName(directory.getName()); 455 } 456 if (fd.version == 0) { // compat with versions < 5.4 457 processBundleForCompat(fd, directory); 458 } 459 return fd; 460 } 461 462 protected FragmentDescriptor getJARFragment(File file) throws IOException { 463 FragmentDescriptor fd = null; 464 JarFile jar = new JarFile(file); 465 try { 466 ZipEntry ze = jar.getEntry(FRAGMENT_FILE); 467 if (ze != null) { 468 InputStream in = new BufferedInputStream(jar.getInputStream(ze)); 469 try { 470 fd = (FragmentDescriptor) xmap.load(in); 471 } finally { 472 in.close(); 473 } 474 if (fd.name == null) { 475 // fallback on symbolic name 476 fd.name = getSymbolicName(file); 477 } 478 if (fd.name == null) { 479 // fallback on artifact id 480 fd.name = getJarArtifactName(file.getName()); 481 } 482 if (fd.version == 0) { // compat with versions < 5.4 483 processBundleForCompat(fd, file); 484 } 485 } 486 } finally { 487 jar.close(); 488 } 489 return fd; 490 } 491 492 protected void processManifest(FragmentDescriptor fd, String fileName, Manifest mf) { 493 Attributes attrs = mf.getMainAttributes(); 494 String id = attrs.getValue("Bundle-SymbolicName"); 495 if (id != null) { 496 int p = id.indexOf(';'); 497 if (p > -1) { // remove properties part if any 498 id = id.substring(0, p); 499 } 500 jar2Id.put(fileName, id); 501 fd.name = id; 502 if (fd.requires != null && !fd.requires.isEmpty()) { 503 throw new RuntimeException( 504 "In compatibility mode you must not use <require> tags for OSGi bundles - use Require-Bundle manifest header instead. Bundle: " 505 + fileName); 506 } 507 // needed to control start-up order (which differs from 508 // Require-Bundle) 509 String requires = attrs.getValue("Nuxeo-Require"); 510 if (requires == null) { // if not specific requirement is met use 511 // Require-Bundle 512 requires = attrs.getValue("Require-Bundle"); 513 } 514 if (requires != null) { 515 String[] ids = StringUtils.split(requires, ',', true); 516 fd.requires = new ArrayList<String>(ids.length); 517 for (int i = 0; i < ids.length; i++) { 518 String rid = ids[i]; 519 p = rid.indexOf(';'); 520 if (p > -1) { // remove properties part if any 521 ids[i] = rid.substring(0, p); 522 } 523 fd.requires.add(ids[i]); 524 } 525 } 526 527 String requiredBy = attrs.getValue("Nuxeo-RequiredBy"); 528 if (requiredBy != null) { 529 String[] ids = StringUtils.split(requiredBy, ',', true); 530 for (int i = 0; i < ids.length; i++) { 531 String rid = ids[i]; 532 p = rid.indexOf(';'); 533 if (p > -1) { // remove properties part if any 534 ids[i] = rid.substring(0, p); 535 } 536 } 537 fd.requiredBy = ids; 538 } 539 540 } else { 541 jar2Id.put(fileName, fd.name); 542 } 543 } 544 545 /** 546 * Reads a container fragment metadata file and returns the container descriptor. 547 */ 548 protected ContainerDescriptor getContainer(File home, File file) throws IOException { 549 URL url; 550 try { 551 url = file.toURI().toURL(); 552 } catch (MalformedURLException e) { 553 throw new RuntimeException(e); 554 } 555 ContainerDescriptor cd = (ContainerDescriptor) xmap.load(url); 556 if (cd != null) { 557 cd.directory = home; 558 if (cd.name == null) { 559 cd.name = home.getName(); 560 } 561 } 562 return cd; 563 } 564 565 protected ContainerDescriptor getDefaultContainer(File directory) throws IOException { 566 File file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE); 567 if (!file.isFile()) { 568 file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE_COMPAT); 569 } 570 ContainerDescriptor cd = null; 571 if (file.isFile()) { 572 cd = getContainer(directory, file); 573 } 574 return cd; 575 } 576 577 public static String getRelativeChildPath(String parent, String child) { 578 // TODO optimize this method 579 // fix win32 case 580 if (parent.indexOf('\\') > -1) { 581 parent = parent.replace('\\', '/'); 582 } 583 if (child.indexOf('\\') > -1) { 584 child = child.replace('\\', '/'); 585 } // end fix win32 586 Path parentPath = new Path(parent); 587 Path childPath = new Path(child); 588 if (parentPath.isPrefixOf(childPath)) { 589 return childPath.removeFirstSegments(parentPath.segmentCount()).makeRelative().toString(); 590 } 591 return null; 592 } 593 594 /** 595 * Run preprocessing in the given home directory and using the given list of bundles. Bundles must be ordered by the 596 * caller to have same deployment order on all computers. 597 * <p> 598 * The metadata file is the metadat file to be used to configure the processor. If null the default location will be 599 * used (relative to home): {@link #CONTAINER_FILE}. 600 */ 601 public static void process(File home, File metadata, File[] files) throws IOException { 602 DeploymentPreprocessor processor = new DeploymentPreprocessor(home); 603 // initialize 604 processor.init(metadata, files); 605 // run preprocessor 606 processor.predeploy(); 607 } 608 609 public static void main(String[] args) throws IOException { 610 File root; 611 if (args.length > 0) { 612 root = new File(args[0]); 613 } else { 614 root = new File("."); 615 } 616 System.out.println("Preprocessing: " + root); 617 DeploymentPreprocessor processor = new DeploymentPreprocessor(root); 618 // initialize 619 processor.init(); 620 // and predeploy 621 processor.predeploy(); 622 System.out.println("Done."); 623 } 624 625}