001/* 002 * (C) Copyright 2006-2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Nuxeo - initial API and implementation 018 * bstefanescu, jcarsique 019 * Anahide Tchertchian 020 * 021 */ 022 023package org.nuxeo.common.utils; 024 025import static java.nio.charset.StandardCharsets.UTF_8; 026 027import java.io.File; 028import java.io.FileInputStream; 029import java.io.FileNotFoundException; 030import java.io.FileOutputStream; 031import java.io.FileWriter; 032import java.io.FilterWriter; 033import java.io.IOException; 034import java.io.InputStream; 035import java.io.OutputStream; 036import java.io.Writer; 037import java.util.ArrayList; 038import java.util.HashMap; 039import java.util.List; 040import java.util.Map; 041import java.util.Properties; 042import java.util.StringTokenizer; 043import java.util.regex.Matcher; 044import java.util.regex.Pattern; 045 046import org.apache.commons.io.FileUtils; 047import org.apache.commons.io.IOUtils; 048import org.apache.commons.logging.Log; 049import org.apache.commons.logging.LogFactory; 050import org.nuxeo.common.codec.Crypto; 051import org.nuxeo.common.codec.CryptoProperties; 052 053import freemarker.template.Configuration; 054import freemarker.template.Template; 055import freemarker.template.TemplateException; 056 057/** 058 * Text template processing. 059 * <p> 060 * Copy files or directories replacing parameters matching pattern '${[a-zA-Z_0-9\-\.]+}' with values from a 061 * {@link CryptoProperties}. 062 * <p> 063 * If the value of a variable is encrypted: 064 * 065 * <pre> 066 * setVariable("var", Crypto.encrypt(value.getBytes)) 067 * </pre> 068 * 069 * then "<code>${var}</code>" will be replaced with: 070 * <ul> 071 * <li>its decrypted value by default: "<code>value</code>"</li> 072 * <li>"<code>${var}</code>" after a call to "<code>setKeepEncryptedAsVar(true)}</code>" 073 * </ul> 074 * and "<code>${#var}</code>" will always be replaced with its decrypted value. 075 * <p> 076 * Since 5.7.2, variables can have a default value using syntax ${parameter:=defaultValue}. The default value will be 077 * used if parameter is null or unset. 078 * <p> 079 * Methods {@link #setTextParsingExtensions(String)} and {@link #setFreemarkerParsingExtensions(String)} allow to set 080 * the list of files being processed when using {@link #processDirectory(File, File)}, based on their extension; others 081 * being simply copied. 082 * 083 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a> 084 * @see CryptoProperties 085 * @see #setKeepEncryptedAsVar(boolean) 086 * @see #setFreemarkerParsingExtensions(String) 087 * @see #setTextParsingExtensions(String) 088 */ 089public class TextTemplate { 090 091 private static final Log log = LogFactory.getLog(TextTemplate.class); 092 093 private static final int MAX_RECURSION_LEVEL = 10; 094 095 private static final String PATTERN_GROUP_DECRYPT = "decrypt"; 096 097 private static final String PATTERN_GROUP_VAR = "var"; 098 099 private static final String PATTERN_GROUP_DEFAULT = "default"; 100 101 /** 102 * matches variables of the form "${[#]embeddedVar[:=defaultValue]}" but not those starting with "$${" 103 */ 104 private static final Pattern PATTERN = Pattern.compile("(?<!\\$)\\$\\{(?<" + PATTERN_GROUP_DECRYPT + ">#)?" // 105 + "(?<" + PATTERN_GROUP_VAR + ">[a-zA-Z_0-9\\-\\.]+)" // embeddedVar 106 + "(:=(?<" + PATTERN_GROUP_DEFAULT + ">.*))?\\}"); // defaultValue 107 108 private final CryptoProperties vars; 109 110 private Properties processedVars; 111 112 private boolean trim = false; 113 114 private List<String> plainTextExtensions; 115 116 private List<String> freemarkerExtensions = new ArrayList<>(); 117 118 private Configuration freemarkerConfiguration = null; 119 120 private Map<String, Object> freemarkerVars = null; 121 122 private boolean keepEncryptedAsVar; 123 124 public boolean isTrim() { 125 return trim; 126 } 127 128 /** 129 * Set to true in order to trim invisible characters (spaces) from values. 130 */ 131 public void setTrim(boolean trim) { 132 this.trim = trim; 133 } 134 135 public TextTemplate() { 136 vars = new CryptoProperties(); 137 } 138 139 /** 140 * {@link #TextTemplate(Properties)} provides an additional default values behavior 141 * 142 * @see #TextTemplate(Properties) 143 */ 144 public TextTemplate(Map<String, String> vars) { 145 this.vars = new CryptoProperties(); 146 this.vars.putAll(vars); 147 } 148 149 /** 150 * @param vars Properties containing keys and values for template processing 151 */ 152 public TextTemplate(Properties vars) { 153 if (vars instanceof CryptoProperties) { 154 this.vars = (CryptoProperties) vars; 155 } else { 156 this.vars = new CryptoProperties(vars); 157 } 158 } 159 160 public void setVariables(Map<String, String> vars) { 161 this.vars.putAll(vars); 162 freemarkerConfiguration = null; 163 } 164 165 /** 166 * If adding multiple variables, prefer use of {@link #setVariables(Map)} 167 */ 168 public void setVariable(String name, String value) { 169 vars.setProperty(name, value); 170 freemarkerConfiguration = null; 171 } 172 173 public String getVariable(String name) { 174 return vars.getProperty(name, keepEncryptedAsVar); 175 } 176 177 public Properties getVariables() { 178 return vars; 179 } 180 181 /** 182 * That method is not recursive. It processes the given text only once. 183 * 184 * @param props CryptoProperties containing the variable values 185 * @param text Text to process 186 * @return the processed text 187 * @since 7.4 188 */ 189 protected String processString(CryptoProperties props, String text) { 190 Matcher m = PATTERN.matcher(text); 191 StringBuffer sb = new StringBuffer(); 192 while (m.find()) { 193 String embeddedVar = m.group(PATTERN_GROUP_VAR); 194 String value = props.getProperty(embeddedVar, keepEncryptedAsVar); 195 if (value == null) { 196 value = m.group(PATTERN_GROUP_DEFAULT); 197 } 198 if (value != null) { 199 if (trim) { 200 value = value.trim(); 201 } 202 if (Crypto.isEncrypted(value)) { 203 if (keepEncryptedAsVar && m.group(PATTERN_GROUP_DECRYPT) == null) { 204 value = "${" + embeddedVar + "}"; 205 } else { 206 value = new String(vars.getCrypto().decrypt(value)); 207 } 208 } 209 210 // Allow use of backslash and dollars characters 211 value = Matcher.quoteReplacement(value); 212 m.appendReplacement(sb, value); 213 } 214 } 215 m.appendTail(sb); 216 return sb.toString(); 217 } 218 219 /** 220 * unescape variables 221 */ 222 protected Properties unescape(Properties props) { 223 props.replaceAll((k, v) -> unescape((String) v)); 224 return props; 225 } 226 227 protected String unescape(String value) { 228 // unescape doubled $ characters, only if in front of a { 229 return value.replaceAll("\\$\\$\\{", "\\${"); 230 } 231 232 private void preprocessVars() { 233 processedVars = preprocessVars(vars); 234 } 235 236 public Properties preprocessVars(Properties unprocessedVars) { 237 CryptoProperties newVars = new CryptoProperties(unprocessedVars); 238 boolean doneProcessing = false; 239 int recursionLevel = 0; 240 while (!doneProcessing) { 241 doneProcessing = true; 242 for (String newVarsKey : newVars.stringPropertyNames()) { 243 String newVarsValue = newVars.getProperty(newVarsKey, keepEncryptedAsVar); 244 if (newVarsValue == null) { 245 continue; 246 } 247 if (Crypto.isEncrypted(newVarsValue)) { 248 // newVarsValue == {$[...]$...} 249 assert (keepEncryptedAsVar); 250 newVarsValue = "${" + newVarsKey + "}"; 251 newVars.put(newVarsKey, newVarsValue); 252 continue; 253 } 254 255 String replacementValue = processString(newVars, newVarsValue); 256 if (!replacementValue.equals(newVarsValue)) { 257 doneProcessing = false; 258 newVars.put(newVarsKey, replacementValue); 259 } 260 } 261 recursionLevel++; 262 // Avoid infinite replacement loops 263 if ((!doneProcessing) && (recursionLevel > MAX_RECURSION_LEVEL)) { 264 log.warn("Detected potential infinite loop when processing the following properties\n" + newVars); 265 break; 266 } 267 } 268 return unescape(newVars); 269 } 270 271 /** 272 * @since 7.4 273 */ 274 public String processText(String text) { 275 if (text == null) { 276 return null; 277 } 278 boolean doneProcessing = false; 279 int recursionLevel = 0; 280 while (!doneProcessing) { 281 doneProcessing = true; 282 String processedText = processString(vars, text); 283 if (!processedText.equals(text)) { 284 doneProcessing = false; 285 text = processedText; 286 } 287 recursionLevel++; 288 // Avoid infinite replacement loops 289 if ((!doneProcessing) && (recursionLevel > MAX_RECURSION_LEVEL)) { 290 log.warn("Detected potential infinite loop when processing the following text\n" + text); 291 break; 292 } 293 } 294 return unescape(text); 295 } 296 297 public String processText(InputStream in) throws IOException { 298 String text = IOUtils.toString(in, UTF_8); 299 return processText(text); 300 } 301 302 public void processText(InputStream is, OutputStream os) throws IOException { 303 String text = IOUtils.toString(is, UTF_8); 304 text = processText(text); 305 os.write(text.getBytes(UTF_8)); 306 } 307 308 /** 309 * Initialize FreeMarker data model from Java properties. 310 * <p> 311 * Variables in the form "{@code foo.bar}" (String with dots) are transformed to "{@code foo[bar]}" (arrays).<br> 312 * So there will be conflicts if a variable name is equal to the prefix of another variable. For instance, " 313 * {@code foo.bar}" and "{@code foo.bar.qux}" will conflict.<br> 314 * When a conflict occurs, the conflicting variable is ignored and a warning is logged. The ignored variable will 315 * usually be the shortest one (without any contract on this behavior). 316 */ 317 @SuppressWarnings("unchecked") 318 public void initFreeMarker() { 319 freemarkerConfiguration = new Configuration(Configuration.getVersion()); 320 preprocessVars(); 321 freemarkerVars = new HashMap<>(); 322 Map<String, Object> currentMap; 323 String currentString; 324 KEYS: for (String key : processedVars.stringPropertyNames()) { 325 String value = processedVars.getProperty(key); 326 String[] keyparts = key.split("\\."); 327 currentMap = freemarkerVars; 328 currentString = ""; 329 for (int i = 0; i < (keyparts.length - 1); i++) { 330 currentString = currentString + ("".equals(currentString) ? "" : ".") + keyparts[i]; 331 if (!currentMap.containsKey(keyparts[i])) { 332 Map<String, Object> nextMap = new HashMap<>(); 333 currentMap.put(keyparts[i], nextMap); 334 currentMap = nextMap; 335 } else if (currentMap.get(keyparts[i]) instanceof Map<?, ?>) { 336 currentMap = (Map<String, Object>) currentMap.get(keyparts[i]); 337 } else { 338 // silently ignore known conflicts between Java properties and FreeMarker model 339 if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding") 340 && !key.startsWith("audit.elasticsearch")) { 341 log.warn(String.format("FreeMarker variables: ignored '%s' conflicting with '%s'", key, 342 currentString)); 343 } 344 continue KEYS; 345 } 346 } 347 if (!currentMap.containsKey(keyparts[keyparts.length - 1])) { 348 currentMap.put(keyparts[keyparts.length - 1], value); 349 } else if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding") 350 && !key.startsWith("audit.elasticsearch")) { 351 Map<String, Object> currentValue = (Map<String, Object>) currentMap.get(keyparts[keyparts.length - 1]); 352 log.warn(String.format("FreeMarker variables: ignored '%2$s' conflicting with '%2$s.%1$s'", 353 currentValue.keySet(), key)); 354 } 355 } 356 } 357 358 public void processFreemarker(File in, File out) throws IOException, TemplateException { 359 if (freemarkerConfiguration == null) { 360 initFreeMarker(); 361 } 362 freemarkerConfiguration.setDirectoryForTemplateLoading(in.getParentFile()); 363 Template nxtpl = freemarkerConfiguration.getTemplate(in.getName()); 364 try (Writer writer = new EscapeVariableFilter(new FileWriter(out))) { 365 nxtpl.process(freemarkerVars, writer); 366 } 367 } 368 369 protected static class EscapeVariableFilter extends FilterWriter { 370 371 protected static final int DOLLAR_SIGN = "$".codePointAt(0); 372 373 protected int last; 374 375 public EscapeVariableFilter(Writer out) { 376 super(out); 377 } 378 379 public @Override void write(int b) throws IOException { 380 if (b == DOLLAR_SIGN && last == DOLLAR_SIGN) { 381 return; 382 } 383 last = b; 384 super.write(b); 385 } 386 387 @Override 388 public void write(char[] cbuf, int off, int len) throws IOException { 389 for (int i = 0; i < len; ++i) { 390 write(cbuf[off + i]); 391 } 392 } 393 394 @Override 395 public void write(char[] cbuf) throws IOException { 396 write(cbuf, 0, cbuf.length); 397 } 398 399 } 400 401 /** 402 * Recursively process each file from "in" directory to "out" directory. 403 * 404 * @param in Directory to read files from 405 * @param out Directory to write files to 406 * @return copied files list 407 * @see TextTemplate#processText(InputStream, OutputStream) 408 * @see TextTemplate#processFreemarker(File, File) 409 */ 410 public List<String> processDirectory(File in, File out) throws FileNotFoundException, IOException, 411 TemplateException { 412 List<String> newFiles = new ArrayList<>(); 413 if (in.isFile()) { 414 if (out.isDirectory()) { 415 out = new File(out, in.getName()); 416 } 417 if (!out.getParentFile().exists()) { 418 out.getParentFile().mkdirs(); 419 } 420 421 boolean processAsText = false; 422 boolean processAsFreemarker = false; 423 // Check for each extension if it matches end of filename 424 String filename = in.getName().toLowerCase(); 425 for (String ext : freemarkerExtensions) { 426 if (filename.endsWith(ext)) { 427 processAsFreemarker = true; 428 out = new File(out.getCanonicalPath().replaceAll("\\.*" + Pattern.quote(ext) + "$", "")); 429 if (filename.equals("." + ext.toLowerCase())) { 430 throw new IOException("Extension only as a filename is not allowed: " + in.getAbsolutePath()); 431 } 432 break; 433 } 434 } 435 if (!processAsFreemarker) { 436 for (String ext : plainTextExtensions) { 437 if (filename.endsWith(ext)) { 438 processAsText = true; 439 break; 440 } 441 } 442 } 443 444 // Backup existing file if not already done 445 if (out.exists()) { 446 File backup = new File(out.getPath() + ".bak"); 447 if (!backup.exists()) { 448 log.debug("Backup " + out); 449 FileUtils.copyFile(out, backup); 450 newFiles.add(backup.getPath()); 451 } 452 } else { 453 newFiles.add(out.getPath()); 454 } 455 try { 456 if (processAsFreemarker) { 457 log.debug("Process as FreeMarker " + in.getPath()); 458 processFreemarker(in, out); 459 } else if (processAsText) { 460 log.debug("Process as Text " + in.getPath()); 461 try (InputStream is = new FileInputStream(in); OutputStream os = new FileOutputStream(out)) { 462 processText(is, os); 463 } 464 } else { 465 log.debug("Process as copy " + in.getPath()); 466 FileUtils.copyFile(in, out); 467 } 468 } catch (IOException | TemplateException e) { 469 log.error("Failure on " + in.getPath()); 470 throw e; 471 } 472 } else if (in.isDirectory()) { 473 if (!out.exists()) { 474 // allow renaming destination directory 475 out.mkdirs(); 476 } else if (!out.getName().equals(in.getName())) { 477 // allow copy over existing hierarchy 478 out = new File(out, in.getName()); 479 out.mkdir(); 480 } 481 for (File file : in.listFiles()) { 482 newFiles.addAll(processDirectory(file, out)); 483 } 484 } 485 return newFiles; 486 } 487 488 /** 489 * @param extensionsList comma-separated list of files extensions to parse 490 */ 491 public void setTextParsingExtensions(String extensionsList) { 492 StringTokenizer st = new StringTokenizer(extensionsList, ","); 493 plainTextExtensions = new ArrayList<>(); 494 while (st.hasMoreTokens()) { 495 String extension = st.nextToken().toLowerCase(); 496 plainTextExtensions.add(extension); 497 } 498 } 499 500 public void setFreemarkerParsingExtensions(String extensionsList) { 501 StringTokenizer st = new StringTokenizer(extensionsList, ","); 502 freemarkerExtensions = new ArrayList<>(); 503 while (st.hasMoreTokens()) { 504 String extension = st.nextToken().toLowerCase(); 505 freemarkerExtensions.add(extension); 506 } 507 } 508 509 /** 510 * Whether to replace or not the variables which value is encrypted. 511 * 512 * @param keepEncryptedAsVar if {@code true}, the variables which value is encrypted won't be expanded 513 * @since 7.4 514 */ 515 public void setKeepEncryptedAsVar(boolean keepEncryptedAsVar) { 516 if (this.keepEncryptedAsVar != keepEncryptedAsVar) { 517 this.keepEncryptedAsVar = keepEncryptedAsVar; 518 freemarkerConfiguration = null; 519 } 520 } 521 522}