001/* 002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Nuxeo - initial API and implementation 018 * bstefanescu, jcarsique 019 * Anahide Tchertchian 020 */ 021package org.nuxeo.runtime.deployment.preprocessor; 022 023import java.io.BufferedInputStream; 024import java.io.File; 025import java.io.IOException; 026import java.io.InputStream; 027import java.net.MalformedURLException; 028import java.net.URL; 029import java.util.ArrayList; 030import java.util.Arrays; 031import java.util.List; 032import java.util.Properties; 033import java.util.jar.Attributes; 034import java.util.jar.JarFile; 035import java.util.jar.Manifest; 036import java.util.regex.Matcher; 037import java.util.regex.Pattern; 038import java.util.zip.ZipEntry; 039 040import org.apache.commons.io.FileUtils; 041import org.apache.commons.logging.Log; 042import org.apache.commons.logging.LogFactory; 043import org.nuxeo.common.collections.DependencyTree; 044import org.nuxeo.common.utils.JarUtils; 045import org.nuxeo.common.utils.Path; 046import org.nuxeo.common.utils.StringUtils; 047import org.nuxeo.common.xmap.XMap; 048import org.nuxeo.launcher.config.ConfigurationGenerator; 049import org.nuxeo.runtime.deployment.preprocessor.install.CommandContext; 050import org.nuxeo.runtime.deployment.preprocessor.install.CommandContextImpl; 051import org.nuxeo.runtime.deployment.preprocessor.template.TemplateContribution; 052import org.nuxeo.runtime.deployment.preprocessor.template.TemplateParser; 053 054/** 055 * Initializer for the deployment skeleton, taking care of creating templates, aggregating default components before 056 * runtime is started. 057 * 058 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a> 059 */ 060public class DeploymentPreprocessor { 061 062 public static final String FRAGMENT_FILE = "OSGI-INF/deployment-fragment.xml"; 063 064 public static final String CONTAINER_FILE = "META-INF/nuxeo-preprocessor.xml"; 065 066 public static final String CONTAINER_FILE_COMPAT = "OSGI-INF/deployment-container.xml"; 067 068 private static final Pattern ARTIFACT_NAME_PATTERN = Pattern.compile("-[0-9]+"); 069 070 private static final Log log = LogFactory.getLog(DeploymentPreprocessor.class); 071 072 private final File dir; 073 074 private final XMap xmap; 075 076 private ContainerDescriptor root; 077 078 public DeploymentPreprocessor(File dir) { 079 this.dir = dir; 080 xmap = new XMap(); 081 xmap.register(ContainerDescriptor.class); 082 xmap.register(FragmentDescriptor.class); 083 } 084 085 public ContainerDescriptor getRootContainer() { 086 return root; 087 } 088 089 public void init() throws IOException { 090 root = getDefaultContainer(dir); 091 if (root != null) { 092 // run container commands 093 init(root); 094 } 095 } 096 097 public void init(File metadata, File[] files) throws IOException { 098 if (metadata == null) { 099 root = getDefaultContainer(dir); 100 } else { 101 root = getContainer(dir, metadata); 102 } 103 if (root != null) { 104 root.files = files; 105 // run container commands 106 init(root); 107 } 108 } 109 110 protected void init(ContainerDescriptor cd) throws IOException { 111 cd.context = new CommandContextImpl(cd.directory); 112 initContextProperties(cd.context); 113 // run container install instructions if any 114 if (cd.install != null) { 115 cd.install.setLogger(log); 116 log.info("Running custom installation for container: " + cd.name); 117 cd.install.exec(cd.context); 118 } 119 if (cd.files != null) { 120 init(cd, cd.files); 121 } else { 122 // scan directories 123 if (cd.directories == null || cd.directories.isEmpty()) { 124 init(cd, dir); 125 } else { 126 for (String dirPath : cd.directories) { 127 init(cd, new File(dir, dirPath)); 128 } 129 } 130 } 131 } 132 133 protected void initContextProperties(CommandContext ctx) { 134 ConfigurationGenerator confGen = new ConfigurationGenerator(); 135 confGen.init(); 136 Properties props = confGen.getUserConfig(); 137 for (String key : props.stringPropertyNames()) { 138 ctx.put(key, props.getProperty(key)); 139 } 140 } 141 142 protected void processFile(ContainerDescriptor cd, File file) throws IOException { 143 String fileName = file.getName(); 144 FragmentDescriptor fd = null; 145 boolean isBundle = false; 146 if (fileName.endsWith("-fragment.xml")) { 147 fd = getXMLFragment(file); 148 } else if (fileName.endsWith("-fragments.xml")) { 149 // we allow declaring multiple fragments in the same file 150 // this is useful to deploy libraries 151 collectXMLFragments(cd, file); 152 return; 153 } else if (fileName.endsWith(".jar") || fileName.endsWith(".war") || fileName.endsWith(".sar") 154 || fileName.endsWith(".rar")) { 155 isBundle = true; 156 if (file.isDirectory()) { 157 fd = getDirectoryFragment(file); 158 } else { 159 fd = getJARFragment(file); 160 } 161 } 162 // register the fragment if any was found 163 if (fd != null) { 164 fd.fileName = fileName; 165 fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath()); 166 cd.fragments.add(fd); 167 if (fd.templates != null) { 168 for (TemplateDescriptor td : fd.templates.values()) { 169 td.baseDir = file; 170 cd.templates.put(td.name, td); 171 } 172 } 173 } else if (isBundle) { 174 // create markers - for compatibility with versions < 5.4 175 String name = getSymbolicName(file); 176 if (name != null) { 177 cd.fragments.add(new FragmentDescriptor(name, true)); 178 } 179 } 180 } 181 182 protected String getSymbolicName(File file) { 183 Manifest mf = JarUtils.getManifest(file); 184 if (mf != null) { 185 Attributes attrs = mf.getMainAttributes(); 186 String id = attrs.getValue("Bundle-SymbolicName"); 187 if (id != null) { 188 int p = id.indexOf(';'); 189 if (p > -1) { // remove properties part if any 190 id = id.substring(0, p); 191 } 192 return id; 193 } 194 } 195 return null; 196 } 197 198 protected String getJarArtifactName(String name) { 199 if (name.endsWith(".jar")) { 200 name = name.substring(0, name.length() - 4); 201 } 202 Matcher m = ARTIFACT_NAME_PATTERN.matcher(name); 203 if (m.find()) { 204 name = name.substring(0, m.start()); 205 } 206 return name; 207 } 208 209 protected void init(ContainerDescriptor cd, File[] files) throws IOException { 210 for (File file : files) { 211 processFile(cd, file); 212 } 213 } 214 215 protected void init(ContainerDescriptor cd, File dir) throws IOException { 216 log.info("Scanning directory: " + dir.getName()); 217 if (!dir.exists()) { 218 log.warn("Directory doesn't exist: " + dir.getPath()); 219 return; 220 } 221 // sort input files in alphabetic order -> this way we are sure we get 222 // the same deploying order on all machines. 223 File[] files = dir.listFiles(); 224 Arrays.sort(files); 225 init(cd, files); 226 } 227 228 public void predeploy() throws IOException { 229 if (root != null) { 230 predeploy(root); 231 } 232 } 233 234 protected static String listFragmentDescriptor(FragmentDescriptor fd) { 235 return fd.name + " (" + fd.fileName + ")"; 236 } 237 238 protected static void printInfo(FragmentRegistry fragments) { 239 List<DependencyTree.Entry<String, FragmentDescriptor>> entries = fragments.getResolvedEntries(); 240 StringBuilder buf = new StringBuilder("Preprocessing order: "); 241 for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) { 242 FragmentDescriptor fd = entry.get(); 243 if (fd != null && !fd.isMarker()) { 244 buf.append("\n\t"); 245 buf.append(listFragmentDescriptor(entry.get())); 246 } 247 } 248 log.info(buf); 249 250 StringBuilder errors = new StringBuilder(); 251 List<DependencyTree.Entry<String, FragmentDescriptor>> missing = fragments.getMissingRequirements(); 252 for (DependencyTree.Entry<String, FragmentDescriptor> entry : missing) { 253 buf = new StringBuilder("Unknown bundle: "); 254 buf.append(entry.getKey()); 255 buf.append(" required by: "); 256 boolean first = true; 257 for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getDependsOnMe()) { 258 if (!first) { 259 buf.append(", "); // length 2 260 } 261 first = false; 262 buf.append(listFragmentDescriptor(dep.get())); 263 } 264 log.error(buf); 265 errors.append(buf); 266 errors.append("\n"); 267 } 268 for (DependencyTree.Entry<String, FragmentDescriptor> entry : fragments.getPendingEntries()) { 269 if (!entry.isRegistered()) { 270 continue; 271 } 272 buf = new StringBuilder("Bundle not preprocessed: "); 273 buf.append(listFragmentDescriptor(entry.get())); 274 buf.append(" waiting for: "); 275 boolean first = true; 276 for (DependencyTree.Entry<String, FragmentDescriptor> dep : entry.getWaitsFor()) { 277 if (!first) { 278 buf.append(", "); // length 2 279 } 280 first = false; 281 buf.append(dep.getKey()); 282 } 283 log.error(buf); 284 errors.append(buf); 285 errors.append("\n"); 286 } 287 if (errors.length() != 0) { 288 // set system property to log startup errors 289 // this is read by AbstractRuntimeService 290 System.setProperty("org.nuxeo.runtime.deployment.errors", errors.toString()); 291 } 292 } 293 294 protected static void predeploy(ContainerDescriptor cd) throws IOException { 295 // run installer and register contributions for each fragment 296 List<DependencyTree.Entry<String, FragmentDescriptor>> entries = cd.fragments.getResolvedEntries(); 297 printInfo(cd.fragments); 298 for (DependencyTree.Entry<String, FragmentDescriptor> entry : entries) { 299 FragmentDescriptor fd = entry.get(); 300 if (fd == null || fd.isMarker()) { 301 continue; // should be a marker entry like the "all" one. 302 } 303 cd.context.put("bundle.fileName", fd.filePath); 304 cd.context.put("bundle.shortName", fd.fileName); 305 cd.context.put("bundle", fd.name); 306 307 // execute install instructions if any 308 if (fd.install != null) { 309 fd.install.setLogger(log); 310 log.info("Running custom installation for fragment: " + fd.name); 311 fd.install.exec(cd.context); 312 } 313 314 if (fd.contributions == null) { 315 continue; // no contributions 316 } 317 318 // get fragment contributions and register them 319 for (TemplateContribution tc : fd.contributions) { 320 321 // register template contributions if any 322 // get the target template 323 TemplateDescriptor td = cd.templates.get(tc.getTemplate()); 324 if (td != null) { 325 if (td.baseDir == null) { 326 td.baseDir = cd.directory; 327 } 328 if (td.template == null) { // template not yet compiled 329 File file = new File(td.baseDir, td.src); 330 // compile it 331 td.template = TemplateParser.parse(file); 332 } 333 } else { 334 log.warn("No template '" + tc.getTemplate() + "' found for deployment fragment: " + fd.name); 335 continue; 336 } 337 // get the marker where contribution should be inserted 338 td.template.update(tc, cd.context); 339 } 340 } 341 342 // process and write templates 343 // fragments where imported. write down templates 344 for (TemplateDescriptor td : cd.templates.values()) { 345 if (td.baseDir == null) { 346 td.baseDir = cd.directory; 347 } 348 // if required process the template even if no contributions were 349 // made 350 if (td.template == null && td.isRequired) { 351 // compile the template 352 File file = new File(td.baseDir, td.src); 353 td.template = TemplateParser.parse(file); 354 } 355 // process the template 356 if (td.template != null) { 357 File file = new File(td.baseDir, td.installPath); 358 file.getParentFile().mkdirs(); // make sure parents exists 359 FileUtils.writeStringToFile(file, td.template.getText()); 360 } 361 } 362 363 // process sub containers if any 364 for (ContainerDescriptor subCd : cd.subContainers) { 365 predeploy(subCd); 366 } 367 } 368 369 protected FragmentDescriptor getXMLFragment(File file) throws IOException { 370 URL url; 371 try { 372 url = file.toURI().toURL(); 373 } catch (MalformedURLException e) { 374 throw new RuntimeException(e); 375 } 376 FragmentDescriptor fd = (FragmentDescriptor) xmap.load(url); 377 if (fd != null && fd.name == null) { 378 fd.name = file.getName(); 379 } 380 return fd; 381 } 382 383 protected void collectXMLFragments(ContainerDescriptor cd, File file) throws IOException { 384 String fileName = file.getName(); 385 URL url; 386 try { 387 url = file.toURI().toURL(); 388 } catch (MalformedURLException e) { 389 throw new RuntimeException(e); 390 } 391 Object[] result = xmap.loadAll(url); 392 for (Object entry : result) { 393 FragmentDescriptor fd = (FragmentDescriptor) entry; 394 assert fd != null; 395 if (fd.name == null) { 396 log.error("Invalid fragments file: " + file.getName() 397 + ". Fragments declared in a -fragments.xml file must have names."); 398 } else { 399 cd.fragments.add(fd); 400 fd.fileName = fileName; 401 fd.filePath = getRelativeChildPath(cd.directory.getAbsolutePath(), file.getAbsolutePath()); 402 } 403 } 404 } 405 406 protected void processBundleForCompat(FragmentDescriptor fd, File file) { 407 // TODO disable for now the warning 408 log.warn("Entering compatibility mode - Please update the deployment-fragment.xml in " + file.getName() 409 + " to use new dependency management"); 410 Manifest mf = JarUtils.getManifest(file); 411 if (mf != null) { 412 fd.name = file.getName(); 413 processManifest(fd, fd.name, mf); 414 } else { 415 throw new RuntimeException("Compat: Fragments without a name must reside in an OSGi bundle"); 416 } 417 } 418 419 protected FragmentDescriptor getDirectoryFragment(File directory) throws IOException { 420 FragmentDescriptor fd; 421 File file = new File(directory.getAbsolutePath() + '/' + FRAGMENT_FILE); 422 if (file.isFile()) { 423 URL url; 424 try { 425 url = file.toURI().toURL(); 426 } catch (MalformedURLException e) { 427 throw new RuntimeException(e); 428 } 429 fd = (FragmentDescriptor) xmap.load(url); 430 } else { 431 return null; // don't need preprocessing 432 } 433 if (fd.name == null) { 434 // fallback on symbolic name 435 fd.name = getSymbolicName(directory); 436 } 437 if (fd.name == null) { 438 // fallback on artifact id 439 fd.name = getJarArtifactName(directory.getName()); 440 } 441 if (fd.version == 0) { // compat with versions < 5.4 442 processBundleForCompat(fd, directory); 443 } 444 return fd; 445 } 446 447 protected FragmentDescriptor getJARFragment(File file) throws IOException { 448 FragmentDescriptor fd = null; 449 try (JarFile jar = new JarFile(file)) { 450 ZipEntry ze = jar.getEntry(FRAGMENT_FILE); 451 if (ze != null) { 452 try (InputStream in = new BufferedInputStream(jar.getInputStream(ze))) { 453 fd = (FragmentDescriptor) xmap.load(in); 454 } 455 if (fd.name == null) { 456 // fallback on symbolic name 457 fd.name = getSymbolicName(file); 458 } 459 if (fd.name == null) { 460 // fallback on artifact id 461 fd.name = getJarArtifactName(file.getName()); 462 } 463 if (fd.version == 0) { // compat with versions < 5.4 464 processBundleForCompat(fd, file); 465 } 466 } 467 } 468 return fd; 469 } 470 471 protected void processManifest(FragmentDescriptor fd, String fileName, Manifest mf) { 472 Attributes attrs = mf.getMainAttributes(); 473 String id = attrs.getValue("Bundle-SymbolicName"); 474 int p = id.indexOf(';'); 475 if (p > -1) { // remove properties part if any 476 id = id.substring(0, p); 477 } 478 fd.name = id; 479 if (fd.requires != null && !fd.requires.isEmpty()) { 480 throw new RuntimeException( 481 "In compatibility mode you must not use <require> tags for OSGi bundles - use Require-Bundle manifest header instead. Bundle: " 482 + fileName); 483 } 484 // needed to control start-up order (which differs from 485 // Require-Bundle) 486 String requires = attrs.getValue("Nuxeo-Require"); 487 if (requires == null) { // if not specific requirement is met use 488 // Require-Bundle 489 requires = attrs.getValue("Require-Bundle"); 490 } 491 if (requires != null) { 492 String[] ids = StringUtils.split(requires, ',', true); 493 fd.requires = new ArrayList<>(ids.length); 494 for (int i = 0; i < ids.length; i++) { 495 String rid = ids[i]; 496 p = rid.indexOf(';'); 497 if (p > -1) { // remove properties part if any 498 ids[i] = rid.substring(0, p); 499 } 500 fd.requires.add(ids[i]); 501 } 502 } 503 504 String requiredBy = attrs.getValue("Nuxeo-RequiredBy"); 505 if (requiredBy != null) { 506 String[] ids = StringUtils.split(requiredBy, ',', true); 507 for (int i = 0; i < ids.length; i++) { 508 String rid = ids[i]; 509 p = rid.indexOf(';'); 510 if (p > -1) { // remove properties part if any 511 ids[i] = rid.substring(0, p); 512 } 513 } 514 fd.requiredBy = ids; 515 } 516 517 } 518 519 /** 520 * Reads a container fragment metadata file and returns the container descriptor. 521 */ 522 protected ContainerDescriptor getContainer(File home, File file) throws IOException { 523 URL url; 524 try { 525 url = file.toURI().toURL(); 526 } catch (MalformedURLException e) { 527 throw new RuntimeException(e); 528 } 529 ContainerDescriptor cd = (ContainerDescriptor) xmap.load(url); 530 if (cd != null) { 531 cd.directory = home; 532 if (cd.name == null) { 533 cd.name = home.getName(); 534 } 535 } 536 return cd; 537 } 538 539 protected ContainerDescriptor getDefaultContainer(File directory) throws IOException { 540 File file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE); 541 if (!file.isFile()) { 542 file = new File(directory.getAbsolutePath() + '/' + CONTAINER_FILE_COMPAT); 543 } 544 ContainerDescriptor cd = null; 545 if (file.isFile()) { 546 cd = getContainer(directory, file); 547 } 548 return cd; 549 } 550 551 public static String getRelativeChildPath(String parent, String child) { 552 // TODO optimize this method 553 // fix win32 case 554 if (parent.indexOf('\\') > -1) { 555 parent = parent.replace('\\', '/'); 556 } 557 if (child.indexOf('\\') > -1) { 558 child = child.replace('\\', '/'); 559 } // end fix win32 560 Path parentPath = new Path(parent); 561 Path childPath = new Path(child); 562 if (parentPath.isPrefixOf(childPath)) { 563 return childPath.removeFirstSegments(parentPath.segmentCount()).makeRelative().toString(); 564 } 565 return null; 566 } 567 568 /** 569 * Run preprocessing in the given home directory and using the given list of bundles. Bundles must be ordered by the 570 * caller to have same deployment order on all computers. 571 * <p> 572 * The metadata file is the metadat file to be used to configure the processor. If null the default location will be 573 * used (relative to home): {@link #CONTAINER_FILE}. 574 */ 575 public static void process(File home, File metadata, File[] files) throws IOException { 576 DeploymentPreprocessor processor = new DeploymentPreprocessor(home); 577 // initialize 578 processor.init(metadata, files); 579 // run preprocessor 580 processor.predeploy(); 581 } 582 583 public static void main(String[] args) throws IOException { 584 File root; 585 if (args.length > 0) { 586 root = new File(args[0]); 587 } else { 588 root = new File("."); 589 } 590 System.out.println("Preprocessing: " + root); 591 DeploymentPreprocessor processor = new DeploymentPreprocessor(root); 592 // initialize 593 processor.init(); 594 // and predeploy 595 processor.predeploy(); 596 System.out.println("Done."); 597 } 598 599}