001/* 002 * (C) Copyright 2006-2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Nuxeo - initial API and implementation 018 * bstefanescu, jcarsique 019 * Anahide Tchertchian 020 * 021 */ 022 023package org.nuxeo.common.utils; 024 025import java.io.File; 026import java.io.FileInputStream; 027import java.io.FileNotFoundException; 028import java.io.FileOutputStream; 029import java.io.FileWriter; 030import java.io.FilterWriter; 031import java.io.IOException; 032import java.io.InputStream; 033import java.io.OutputStream; 034import java.io.Writer; 035import java.util.ArrayList; 036import java.util.HashMap; 037import java.util.List; 038import java.util.Map; 039import java.util.Properties; 040import java.util.StringTokenizer; 041import java.util.regex.Matcher; 042import java.util.regex.Pattern; 043 044import org.apache.commons.io.Charsets; 045import org.apache.commons.io.FileUtils; 046import org.apache.commons.io.IOUtils; 047import org.apache.commons.logging.Log; 048import org.apache.commons.logging.LogFactory; 049 050import org.nuxeo.common.codec.Crypto; 051import org.nuxeo.common.codec.CryptoProperties; 052 053import freemarker.template.Configuration; 054import freemarker.template.Template; 055import freemarker.template.TemplateException; 056 057/** 058 * Text template processing. 059 * <p> 060 * Copy files or directories replacing parameters matching pattern '${[a-zA-Z_0-9\-\.]+}' with values from a 061 * {@link CryptoProperties}. 062 * <p> 063 * If the value of a variable is encrypted: 064 * 065 * <pre> 066 * setVariable("var", Crypto.encrypt(value.getBytes)) 067 * </pre> 068 * 069 * then "<code>${var}</code>" will be replaced with: 070 * <ul> 071 * <li>its decrypted value by default: "<code>value</code>"</li> 072 * <li>"<code>${var}</code>" after a call to "<code>setKeepEncryptedAsVar(true)}</code>" 073 * </ul> 074 * and "<code>${#var}</code>" will always be replaced with its decrypted value. 075 * <p> 076 * Since 5.7.2, variables can have a default value using syntax ${parameter:=defaultValue}. The default value will be 077 * used if parameter is null or unset. 078 * <p> 079 * Methods {@link #setTextParsingExtensions(String)} and {@link #setFreemarkerParsingExtensions(String)} allow to set 080 * the list of files being processed when using {@link #processDirectory(File, File)}, based on their extension; others 081 * being simply copied. 082 * 083 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a> 084 * @see CryptoProperties 085 * @see #setKeepEncryptedAsVar(boolean) 086 * @see #setFreemarkerParsingExtensions(String) 087 * @see #setTextParsingExtensions(String) 088 */ 089public class TextTemplate { 090 091 private static final Log log = LogFactory.getLog(TextTemplate.class); 092 093 private static final int MAX_RECURSION_LEVEL = 10; 094 095 private static final String PATTERN_GROUP_DECRYPT = "decrypt"; 096 097 private static final String PATTERN_GROUP_VAR = "var"; 098 099 private static final String PATTERN_GROUP_DEFAULT = "default"; 100 101 /** 102 * matches variables of the form "${[#]embeddedVar[:=defaultValue]}" but not those starting with "$${" 103 */ 104 private static final Pattern PATTERN = Pattern.compile("(?<!\\$)\\$\\{(?<" + PATTERN_GROUP_DECRYPT + ">#)?" // 105 + "(?<" + PATTERN_GROUP_VAR + ">[a-zA-Z_0-9\\-\\.]+)" // embeddedVar 106 + "(:=(?<" + PATTERN_GROUP_DEFAULT + ">.*))?\\}"); // defaultValue 107 108 private final CryptoProperties vars; 109 110 private Properties processedVars; 111 112 private boolean trim = false; 113 114 private List<String> plainTextExtensions; 115 116 private List<String> freemarkerExtensions = new ArrayList<>(); 117 118 private Configuration freemarkerConfiguration = null; 119 120 private Map<String, Object> freemarkerVars = null; 121 122 private boolean keepEncryptedAsVar; 123 124 public boolean isTrim() { 125 return trim; 126 } 127 128 /** 129 * Set to true in order to trim invisible characters (spaces) from values. 130 */ 131 public void setTrim(boolean trim) { 132 this.trim = trim; 133 } 134 135 public TextTemplate() { 136 vars = new CryptoProperties(); 137 } 138 139 /** 140 * {@link #TextTemplate(Properties)} provides an additional default values behavior 141 * 142 * @see #TextTemplate(Properties) 143 */ 144 public TextTemplate(Map<String, String> vars) { 145 this.vars = new CryptoProperties(); 146 this.vars.putAll(vars); 147 } 148 149 /** 150 * @param vars Properties containing keys and values for template processing 151 */ 152 public TextTemplate(Properties vars) { 153 if (vars instanceof CryptoProperties) { 154 this.vars = (CryptoProperties) vars; 155 } else { 156 this.vars = new CryptoProperties(vars); 157 } 158 } 159 160 public void setVariables(Map<String, String> vars) { 161 this.vars.putAll(vars); 162 freemarkerConfiguration = null; 163 } 164 165 /** 166 * If adding multiple variables, prefer use of {@link #setVariables(Map)} 167 */ 168 public void setVariable(String name, String value) { 169 vars.setProperty(name, value); 170 freemarkerConfiguration = null; 171 } 172 173 public String getVariable(String name) { 174 return vars.getProperty(name, keepEncryptedAsVar); 175 } 176 177 public Properties getVariables() { 178 return vars; 179 } 180 181 /** 182 * @deprecated Since 7.4. Use {@link #processText(CharSequence)} instead. 183 */ 184 @Deprecated 185 public String process(CharSequence text) { 186 return processText(text); 187 } 188 189 /** 190 * @deprecated Since 7.4. Use {@link #processText(InputStream)} instead. 191 */ 192 @Deprecated 193 public String process(InputStream in) throws IOException { 194 return processText(in); 195 } 196 197 /** 198 * @deprecated Since 7.4. Use {@link #processText(InputStream, OutputStream)} instead. 199 */ 200 @Deprecated 201 public void process(InputStream in, OutputStream out) throws IOException { 202 processText(in, out); 203 } 204 205 /** 206 * @param processText if true, text is processed for parameters replacement 207 * @deprecated Since 7.4. Use {@link #processText(InputStream, OutputStream)} (if {@code processText}) or 208 * {@link IOUtils#copy(InputStream, OutputStream)} 209 */ 210 @Deprecated 211 public void process(InputStream is, OutputStream os, boolean processText) throws IOException { 212 if (processText) { 213 String text = IOUtils.toString(is, Charsets.UTF_8); 214 text = processText(text); 215 os.write(text.getBytes()); 216 } else { 217 IOUtils.copy(is, os); 218 } 219 } 220 221 /** 222 * That method is not recursive. It processes the given text only once. 223 * 224 * @param props CryptoProperties containing the variable values 225 * @param text Text to process 226 * @return the processed text 227 * @since 7.4 228 */ 229 protected String processString(CryptoProperties props, String text) { 230 Matcher m = PATTERN.matcher(text); 231 StringBuffer sb = new StringBuffer(); 232 while (m.find()) { 233 // newVarsValue == ${[#]embeddedVar[:=default]} 234 String embeddedVar = m.group(PATTERN_GROUP_VAR); 235 String value = props.getProperty(embeddedVar, keepEncryptedAsVar); 236 if (value == null) { 237 value = m.group(PATTERN_GROUP_DEFAULT); 238 } 239 if (value != null) { 240 if (trim) { 241 value = value.trim(); 242 } 243 if (Crypto.isEncrypted(value)) { 244 if (keepEncryptedAsVar && m.group(PATTERN_GROUP_DECRYPT) == null) { 245 value = "${" + embeddedVar + "}"; 246 } else { 247 value = new String(vars.getCrypto().decrypt(value)); 248 } 249 } 250 251 // Allow use of backslash and dollars characters 252 value = Matcher.quoteReplacement(value); 253 m.appendReplacement(sb, value); 254 } 255 } 256 m.appendTail(sb); 257 return sb.toString(); 258 } 259 260 /** 261 * unescape variables 262 */ 263 protected Properties unescape(Properties props) { 264 for (Object key : props.keySet()) { 265 props.put(key, unescape((String) props.get(key))); 266 } 267 return props; 268 } 269 270 protected String unescape(String value) { 271 // unescape doubled $ characters, only if in front of a { 272 return value.replaceAll("\\$\\$\\{", "\\${"); 273 } 274 275 private void preprocessVars() { 276 processedVars = preprocessVars(vars); 277 } 278 279 public Properties preprocessVars(Properties unprocessedVars) { 280 CryptoProperties newVars = new CryptoProperties(unprocessedVars); 281 boolean doneProcessing = false; 282 int recursionLevel = 0; 283 while (!doneProcessing) { 284 doneProcessing = true; 285 for (String newVarsKey : newVars.stringPropertyNames()) { 286 String newVarsValue = newVars.getProperty(newVarsKey, keepEncryptedAsVar); 287 if (newVarsValue == null) { 288 continue; 289 } 290 if (Crypto.isEncrypted(newVarsValue)) { 291 // newVarsValue == {$[...]$...} 292 assert (keepEncryptedAsVar); 293 newVarsValue = "${" + newVarsKey + "}"; 294 newVars.put(newVarsKey, newVarsValue); 295 continue; 296 } 297 298 String replacementValue = processString(newVars, newVarsValue); 299 if (!replacementValue.equals(newVarsValue)) { 300 doneProcessing = false; 301 newVars.put(newVarsKey, replacementValue); 302 } 303 } 304 recursionLevel++; 305 // Avoid infinite replacement loops 306 if ((!doneProcessing) && (recursionLevel > MAX_RECURSION_LEVEL)) { 307 log.warn("Detected potential infinite loop when processing the following properties\n" + newVars); 308 break; 309 } 310 } 311 return unescape(newVars); 312 } 313 314 /** 315 * @deprecated Since 7.4. Use {@link #processText(String)} 316 */ 317 @Deprecated 318 public String processText(CharSequence text) { 319 return processText(text.toString()); 320 } 321 322 /** 323 * @since 7.4 324 */ 325 public String processText(String text) { 326 if (text == null) { 327 return null; 328 } 329 boolean doneProcessing = false; 330 int recursionLevel = 0; 331 while (!doneProcessing) { 332 doneProcessing = true; 333 String processedText = processString(vars, text); 334 if (!processedText.equals(text)) { 335 doneProcessing = false; 336 text = processedText; 337 } 338 recursionLevel++; 339 // Avoid infinite replacement loops 340 if ((!doneProcessing) && (recursionLevel > MAX_RECURSION_LEVEL)) { 341 log.warn("Detected potential infinite loop when processing the following text\n" + text); 342 break; 343 } 344 } 345 return unescape(text); 346 } 347 348 public String processText(InputStream in) throws IOException { 349 String text = IOUtils.toString(in, Charsets.UTF_8); 350 return processText(text); 351 } 352 353 public void processText(InputStream is, OutputStream os) throws IOException { 354 String text = IOUtils.toString(is, Charsets.UTF_8); 355 text = processText(text); 356 os.write(text.getBytes(Charsets.UTF_8)); 357 } 358 359 /** 360 * Initialize FreeMarker data model from Java properties. 361 * <p> 362 * Variables in the form "{@code foo.bar}" (String with dots) are transformed to "{@code foo[bar]}" (arrays).<br> 363 * So there will be conflicts if a variable name is equal to the prefix of another variable. For instance, " 364 * {@code foo.bar}" and "{@code foo.bar.qux}" will conflict.<br> 365 * When a conflict occurs, the conflicting variable is ignored and a warning is logged. The ignored variable will 366 * usually be the shortest one (without any contract on this behavior). 367 */ 368 @SuppressWarnings("unchecked") 369 public void initFreeMarker() { 370 freemarkerConfiguration = new Configuration(Configuration.getVersion()); 371 preprocessVars(); 372 freemarkerVars = new HashMap<>(); 373 Map<String, Object> currentMap; 374 String currentString; 375 KEYS: for (String key : processedVars.stringPropertyNames()) { 376 String value = processedVars.getProperty(key); 377 String[] keyparts = key.split("\\."); 378 currentMap = freemarkerVars; 379 currentString = ""; 380 for (int i = 0; i < (keyparts.length - 1); i++) { 381 currentString = currentString + ("".equals(currentString) ? "" : ".") + keyparts[i]; 382 if (!currentMap.containsKey(keyparts[i])) { 383 Map<String, Object> nextMap = new HashMap<>(); 384 currentMap.put(keyparts[i], nextMap); 385 currentMap = nextMap; 386 } else if (currentMap.get(keyparts[i]) instanceof Map<?, ?>) { 387 currentMap = (Map<String, Object>) currentMap.get(keyparts[i]); 388 } else { 389 // silently ignore known conflicts between Java properties and FreeMarker model 390 if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding") 391 && !key.startsWith("audit.elasticsearch")) { 392 log.warn(String.format("FreeMarker variables: ignored '%s' conflicting with '%s'", key, 393 currentString)); 394 } 395 continue KEYS; 396 } 397 } 398 if (!currentMap.containsKey(keyparts[keyparts.length - 1])) { 399 currentMap.put(keyparts[keyparts.length - 1], value); 400 } else if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding") 401 && !key.startsWith("audit.elasticsearch")) { 402 Map<String, Object> currentValue = (Map<String, Object>) currentMap.get(keyparts[keyparts.length - 1]); 403 log.warn(String.format("FreeMarker variables: ignored '%2$s' conflicting with '%2$s.%1$s'", 404 currentValue.keySet(), key)); 405 } 406 } 407 } 408 409 public void processFreemarker(File in, File out) throws IOException, TemplateException { 410 if (freemarkerConfiguration == null) { 411 initFreeMarker(); 412 } 413 freemarkerConfiguration.setDirectoryForTemplateLoading(in.getParentFile()); 414 Template nxtpl = freemarkerConfiguration.getTemplate(in.getName()); 415 try (Writer writer = new EscapeVariableFilter(new FileWriter(out))) { 416 nxtpl.process(freemarkerVars, writer); 417 } 418 } 419 420 protected static class EscapeVariableFilter extends FilterWriter { 421 422 protected static final int DOLLAR_SIGN = "$".codePointAt(0); 423 424 protected int last; 425 426 public EscapeVariableFilter(Writer out) { 427 super(out); 428 } 429 430 public @Override void write(int b) throws IOException { 431 if (b == DOLLAR_SIGN && last == DOLLAR_SIGN) { 432 return; 433 } 434 last = b; 435 super.write(b); 436 } 437 438 @Override 439 public void write(char[] cbuf, int off, int len) throws IOException { 440 for (int i = 0; i < len; ++i) { 441 write(cbuf[off + i]); 442 } 443 } 444 445 @Override 446 public void write(char[] cbuf) throws IOException { 447 write(cbuf, 0, cbuf.length); 448 } 449 450 } 451 452 /** 453 * Recursively process each file from "in" directory to "out" directory. 454 * 455 * @param in Directory to read files from 456 * @param out Directory to write files to 457 * @return copied files list 458 * @see TextTemplate#processText(InputStream, OutputStream) 459 * @see TextTemplate#processFreemarker(File, File) 460 */ 461 public List<String> processDirectory(File in, File out) throws FileNotFoundException, IOException, 462 TemplateException { 463 List<String> newFiles = new ArrayList<>(); 464 if (in.isFile()) { 465 if (out.isDirectory()) { 466 out = new File(out, in.getName()); 467 } 468 if (!out.getParentFile().exists()) { 469 out.getParentFile().mkdirs(); 470 } 471 472 boolean processAsText = false; 473 boolean processAsFreemarker = false; 474 // Check for each extension if it matches end of filename 475 String filename = in.getName().toLowerCase(); 476 for (String ext : freemarkerExtensions) { 477 if (filename.endsWith(ext)) { 478 processAsFreemarker = true; 479 out = new File(out.getCanonicalPath().replaceAll("\\.*" + Pattern.quote(ext) + "$", "")); 480 break; 481 } 482 } 483 if (!processAsFreemarker) { 484 for (String ext : plainTextExtensions) { 485 if (filename.endsWith(ext)) { 486 processAsText = true; 487 break; 488 } 489 } 490 } 491 492 // Backup existing file if not already done 493 if (out.exists()) { 494 File backup = new File(out.getPath() + ".bak"); 495 if (!backup.exists()) { 496 log.debug("Backup " + out); 497 FileUtils.copyFile(out, backup); 498 newFiles.add(backup.getPath()); 499 } 500 } else { 501 newFiles.add(out.getPath()); 502 } 503 try { 504 if (processAsFreemarker) { 505 log.debug("Process as FreeMarker " + in.getPath()); 506 processFreemarker(in, out); 507 } else if (processAsText) { 508 log.debug("Process as Text " + in.getPath()); 509 InputStream is = null; 510 OutputStream os = null; 511 try { 512 is = new FileInputStream(in); 513 os = new FileOutputStream(out); 514 processText(is, os); 515 } finally { 516 IOUtils.closeQuietly(is); 517 IOUtils.closeQuietly(os); 518 } 519 } else { 520 log.debug("Process as copy " + in.getPath()); 521 FileUtils.copyFile(in, out); 522 } 523 } catch (IOException | TemplateException e) { 524 log.error("Failure on " + in.getPath()); 525 throw e; 526 } 527 } else if (in.isDirectory()) { 528 if (!out.exists()) { 529 // allow renaming destination directory 530 out.mkdirs(); 531 } else if (!out.getName().equals(in.getName())) { 532 // allow copy over existing hierarchy 533 out = new File(out, in.getName()); 534 out.mkdir(); 535 } 536 for (File file : in.listFiles()) { 537 newFiles.addAll(processDirectory(file, out)); 538 } 539 } 540 return newFiles; 541 } 542 543 /** 544 * @param extensionsList comma-separated list of files extensions to parse 545 * @deprecated Since 7.4. Use {@link #setTextParsingExtensions(String)} instead. 546 * @see #setTextParsingExtensions(String) 547 * @see #setFreemarkerParsingExtensions(String) 548 */ 549 @Deprecated 550 public void setParsingExtensions(String extensionsList) { 551 setTextParsingExtensions(extensionsList); 552 } 553 554 /** 555 * @param extensionsList comma-separated list of files extensions to parse 556 */ 557 public void setTextParsingExtensions(String extensionsList) { 558 StringTokenizer st = new StringTokenizer(extensionsList, ","); 559 plainTextExtensions = new ArrayList<>(); 560 while (st.hasMoreTokens()) { 561 String extension = st.nextToken().toLowerCase(); 562 plainTextExtensions.add(extension); 563 } 564 } 565 566 public void setFreemarkerParsingExtensions(String extensionsList) { 567 StringTokenizer st = new StringTokenizer(extensionsList, ","); 568 freemarkerExtensions = new ArrayList<>(); 569 while (st.hasMoreTokens()) { 570 String extension = st.nextToken().toLowerCase(); 571 freemarkerExtensions.add(extension); 572 } 573 } 574 575 /** 576 * Whether to replace or not the variables which value is encrypted. 577 * 578 * @param keepEncryptedAsVar if {@code true}, the variables which value is encrypted won't be expanded 579 * @since 7.4 580 */ 581 public void setKeepEncryptedAsVar(boolean keepEncryptedAsVar) { 582 if (this.keepEncryptedAsVar != keepEncryptedAsVar) { 583 this.keepEncryptedAsVar = keepEncryptedAsVar; 584 freemarkerConfiguration = null; 585 } 586 } 587 588}