001/* 002 * (C) Copyright 2006-2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Nuxeo - initial API and implementation 018 * bstefanescu, jcarsique 019 * Anahide Tchertchian 020 * 021 */ 022 023package org.nuxeo.common.utils; 024 025import static java.nio.charset.StandardCharsets.UTF_8; 026 027import java.io.File; 028import java.io.FileInputStream; 029import java.io.FileNotFoundException; 030import java.io.FileOutputStream; 031import java.io.FileWriter; 032import java.io.FilterWriter; 033import java.io.IOException; 034import java.io.InputStream; 035import java.io.OutputStreamWriter; 036import java.io.Writer; 037import java.util.ArrayList; 038import java.util.HashMap; 039import java.util.List; 040import java.util.Map; 041import java.util.Properties; 042import java.util.StringTokenizer; 043import java.util.regex.Matcher; 044import java.util.regex.Pattern; 045 046import org.apache.commons.io.FileUtils; 047import org.apache.commons.io.IOUtils; 048import org.apache.commons.logging.Log; 049import org.apache.commons.logging.LogFactory; 050import org.nuxeo.common.codec.Crypto; 051import org.nuxeo.common.codec.CryptoProperties; 052 053import freemarker.template.Configuration; 054import freemarker.template.Template; 055import freemarker.template.TemplateException; 056 057/** 058 * Text template processing. 059 * <p> 060 * Copy files or directories replacing parameters matching pattern '${[a-zA-Z_0-9\-\.]+}' with values from a 061 * {@link CryptoProperties}. 062 * <p> 063 * If the value of a variable is encrypted: 064 * 065 * <pre> 066 * setVariable("var", Crypto.encrypt(value.getBytes)) 067 * </pre> 068 * 069 * then "<code>${var}</code>" will be replaced with: 070 * <ul> 071 * <li>its decrypted value by default: "<code>value</code>"</li> 072 * <li>"<code>${var}</code>" after a call to "<code>setKeepEncryptedAsVar(true)}</code>" 073 * </ul> 074 * and "<code>${#var}</code>" will always be replaced with its decrypted value. 075 * <p> 076 * Since 5.7.2, variables can have a default value using syntax ${parameter:=defaultValue}. The default value will be 077 * used if parameter is null or unset. 078 * <p> 079 * Methods {@link #setTextParsingExtensions(String)} and {@link #setFreemarkerParsingExtensions(String)} allow to set 080 * the list of files being processed when using {@link #processDirectory(File, File)}, based on their extension; others 081 * being simply copied. 082 * 083 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a> 084 * @see CryptoProperties 085 * @see #setKeepEncryptedAsVar(boolean) 086 * @see #setFreemarkerParsingExtensions(String) 087 * @see #setTextParsingExtensions(String) 088 */ 089public class TextTemplate { 090 091 private static final Log log = LogFactory.getLog(TextTemplate.class); 092 093 private static final int MAX_RECURSION_LEVEL = 10; 094 095 private static final String PATTERN_GROUP_DECRYPT = "decrypt"; 096 097 private static final String PATTERN_GROUP_VAR = "var"; 098 099 private static final String PATTERN_GROUP_DEFAULT = "default"; 100 101 /** 102 * matches variables of the form "${[#]embeddedVar[:=defaultValue]}" but not those starting with "$${" 103 */ 104 private static final Pattern PATTERN = Pattern.compile("(?<!\\$)\\$\\{(?<" + PATTERN_GROUP_DECRYPT + ">#)?" // 105 + "(?<" + PATTERN_GROUP_VAR + ">[a-zA-Z_0-9\\-\\.]+)" // embeddedVar 106 + "(:=(?<" + PATTERN_GROUP_DEFAULT + ">.*))?\\}"); // defaultValue 107 108 private final CryptoProperties vars; 109 110 private Properties processedVars; 111 112 private boolean trim = false; 113 114 private List<String> plainTextExtensions; 115 116 private List<String> freemarkerExtensions = new ArrayList<>(); 117 118 private Configuration freemarkerConfiguration = null; 119 120 private Map<String, Object> freemarkerVars = null; 121 122 private boolean keepEncryptedAsVar; 123 124 public boolean isTrim() { 125 return trim; 126 } 127 128 /** 129 * Set to true in order to trim invisible characters (spaces) from values. 130 */ 131 public void setTrim(boolean trim) { 132 this.trim = trim; 133 } 134 135 public TextTemplate() { 136 vars = new CryptoProperties(); 137 } 138 139 /** 140 * {@link #TextTemplate(Properties)} provides an additional default values behavior 141 * 142 * @see #TextTemplate(Properties) 143 */ 144 public TextTemplate(Map<String, String> vars) { 145 this.vars = new CryptoProperties(); 146 this.vars.putAll(vars); 147 } 148 149 /** 150 * @param vars Properties containing keys and values for template processing 151 */ 152 public TextTemplate(Properties vars) { 153 if (vars instanceof CryptoProperties) { 154 this.vars = (CryptoProperties) vars; 155 } else { 156 this.vars = new CryptoProperties(vars); 157 } 158 } 159 160 public void setVariables(Map<String, String> vars) { 161 this.vars.putAll(vars); 162 freemarkerConfiguration = null; 163 } 164 165 /** 166 * If adding multiple variables, prefer use of {@link #setVariables(Map)} 167 */ 168 public void setVariable(String name, String value) { 169 vars.setProperty(name, value); 170 freemarkerConfiguration = null; 171 } 172 173 public String getVariable(String name) { 174 return vars.getProperty(name, keepEncryptedAsVar); 175 } 176 177 public Properties getVariables() { 178 return vars; 179 } 180 181 /** 182 * That method is not recursive. It processes the given text only once. 183 * 184 * @param props CryptoProperties containing the variable values 185 * @param text Text to process 186 * @return the processed text 187 * @since 7.4 188 */ 189 protected String processString(CryptoProperties props, String text) { 190 Matcher m = PATTERN.matcher(text); 191 StringBuffer sb = new StringBuffer(); 192 while (m.find()) { 193 String embeddedVar = m.group(PATTERN_GROUP_VAR); 194 String value = props.getProperty(embeddedVar, keepEncryptedAsVar); 195 if (value == null) { 196 value = m.group(PATTERN_GROUP_DEFAULT); 197 } 198 if (value != null) { 199 if (trim) { 200 value = value.trim(); 201 } 202 if (Crypto.isEncrypted(value)) { 203 if (keepEncryptedAsVar && m.group(PATTERN_GROUP_DECRYPT) == null) { 204 value = "${" + embeddedVar + "}"; 205 } else { 206 value = new String(vars.getCrypto().decrypt(value)); 207 } 208 } 209 210 // Allow use of backslash and dollars characters 211 value = Matcher.quoteReplacement(value); 212 m.appendReplacement(sb, value); 213 } 214 } 215 m.appendTail(sb); 216 return sb.toString(); 217 } 218 219 /** 220 * unescape variables 221 */ 222 protected Properties unescape(Properties props) { 223 props.replaceAll((k, v) -> unescape((String) v)); 224 return props; 225 } 226 227 protected String unescape(String value) { 228 // unescape doubled $ characters, only if in front of a { 229 return value.replaceAll("\\$\\$\\{", "\\${"); 230 } 231 232 private void preprocessVars() { 233 processedVars = preprocessVars(vars); 234 } 235 236 public Properties preprocessVars(Properties unprocessedVars) { 237 CryptoProperties newVars = new CryptoProperties(unprocessedVars); 238 boolean doneProcessing = false; 239 int recursionLevel = 0; 240 while (!doneProcessing) { 241 doneProcessing = true; 242 for (String newVarsKey : newVars.stringPropertyNames()) { 243 String newVarsValue = newVars.getProperty(newVarsKey, keepEncryptedAsVar); 244 if (newVarsValue == null) { 245 continue; 246 } 247 if (Crypto.isEncrypted(newVarsValue)) { 248 // newVarsValue == {$[...]$...} 249 assert keepEncryptedAsVar; 250 newVarsValue = "${" + newVarsKey + "}"; 251 newVars.put(newVarsKey, newVarsValue); 252 continue; 253 } 254 255 String replacementValue = processString(newVars, newVarsValue); 256 if (!replacementValue.equals(newVarsValue)) { 257 doneProcessing = false; 258 newVars.put(newVarsKey, replacementValue); 259 } 260 } 261 recursionLevel++; 262 // Avoid infinite replacement loops 263 if (!doneProcessing && recursionLevel > MAX_RECURSION_LEVEL) { 264 log.warn("Detected potential infinite loop when processing the following properties\n" + newVars); 265 break; 266 } 267 } 268 return unescape(newVars); 269 } 270 271 /** 272 * @since 7.4 273 */ 274 public String processText(String text) { 275 if (text == null) { 276 return null; 277 } 278 boolean doneProcessing = false; 279 int recursionLevel = 0; 280 while (!doneProcessing) { 281 doneProcessing = true; 282 String processedText = processString(vars, text); 283 if (!processedText.equals(text)) { 284 doneProcessing = false; 285 text = processedText; 286 } 287 recursionLevel++; 288 // Avoid infinite replacement loops 289 if (!doneProcessing && recursionLevel > MAX_RECURSION_LEVEL) { 290 log.warn("Detected potential infinite loop when processing the following text\n" + text); 291 break; 292 } 293 } 294 return unescape(text); 295 } 296 297 public String processText(InputStream in) throws IOException { 298 String text = IOUtils.toString(in, UTF_8); 299 return processText(text); 300 } 301 302 public void processText(InputStream is, OutputStreamWriter os) throws IOException { 303 String text = IOUtils.toString(is, UTF_8); 304 text = processText(text); 305 os.write(text); 306 } 307 308 /** 309 * Initialize FreeMarker data model from Java properties. 310 * <p> 311 * Variables in the form "{@code foo.bar}" (String with dots) are transformed to "{@code foo[bar]}" (arrays).<br> 312 * So there will be conflicts if a variable name is equal to the prefix of another variable. For instance, " 313 * {@code foo.bar}" and "{@code foo.bar.qux}" will conflict.<br> 314 * When a conflict occurs, the conflicting variable is ignored and a warning is logged. The ignored variable will 315 * usually be the shortest one (without any contract on this behavior). 316 */ 317 @SuppressWarnings("unchecked") 318 public void initFreeMarker() { 319 freemarkerConfiguration = new Configuration(Configuration.VERSION_2_3_30); 320 preprocessVars(); 321 freemarkerVars = new HashMap<>(); 322 Map<String, Object> currentMap; 323 String currentString; 324 KEYS: for (String key : processedVars.stringPropertyNames()) { 325 String value = processedVars.getProperty(key); 326 if (value.startsWith("${") && value.endsWith("}")) { 327 // crypted variables have to be decrypted in freemarker vars 328 value = vars.getProperty(key, false); 329 } 330 String[] keyparts = key.split("\\."); 331 currentMap = freemarkerVars; 332 currentString = ""; 333 for (int i = 0; i < keyparts.length - 1; i++) { 334 currentString = currentString + ("".equals(currentString) ? "" : ".") + keyparts[i]; 335 if (!currentMap.containsKey(keyparts[i])) { 336 Map<String, Object> nextMap = new HashMap<>(); 337 currentMap.put(keyparts[i], nextMap); 338 currentMap = nextMap; 339 } else if (currentMap.get(keyparts[i]) instanceof Map<?, ?>) { 340 currentMap = (Map<String, Object>) currentMap.get(keyparts[i]); 341 } else { 342 // silently ignore known conflicts between Java properties and FreeMarker model 343 if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding") 344 && !key.startsWith("audit.elasticsearch")) { 345 log.warn(String.format("FreeMarker variables: ignored '%s' conflicting with '%s'", key, 346 currentString)); 347 } 348 continue KEYS; 349 } 350 } 351 if (!currentMap.containsKey(keyparts[keyparts.length - 1])) { 352 currentMap.put(keyparts[keyparts.length - 1], value); 353 } else if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding") 354 && !key.startsWith("audit.elasticsearch")) { 355 Map<String, Object> currentValue = (Map<String, Object>) currentMap.get(keyparts[keyparts.length - 1]); 356 log.warn(String.format("FreeMarker variables: ignored '%2$s' conflicting with '%2$s.%1$s'", 357 currentValue.keySet(), key)); 358 } 359 } 360 } 361 362 public void processFreemarker(File in, File out) throws IOException, TemplateException { 363 if (freemarkerConfiguration == null) { 364 initFreeMarker(); 365 } 366 freemarkerConfiguration.setDirectoryForTemplateLoading(in.getParentFile()); 367 Template nxtpl = freemarkerConfiguration.getTemplate(in.getName()); 368 try (Writer writer = new EscapeVariableFilter(new FileWriter(out))) { 369 nxtpl.process(freemarkerVars, writer); 370 } 371 } 372 373 protected static class EscapeVariableFilter extends FilterWriter { 374 375 protected static final int DOLLAR_SIGN = "$".codePointAt(0); 376 377 protected int last; 378 379 public EscapeVariableFilter(Writer out) { 380 super(out); 381 } 382 383 public @Override void write(int b) throws IOException { 384 if (b == DOLLAR_SIGN && last == DOLLAR_SIGN) { 385 return; 386 } 387 last = b; 388 super.write(b); 389 } 390 391 @Override 392 public void write(char[] cbuf, int off, int len) throws IOException { 393 for (int i = 0; i < len; ++i) { 394 write(cbuf[off + i]); 395 } 396 } 397 398 @Override 399 public void write(char[] cbuf) throws IOException { 400 write(cbuf, 0, cbuf.length); 401 } 402 403 } 404 405 /** 406 * Recursively process each file from "in" directory to "out" directory. 407 * 408 * @param in Directory to read files from 409 * @param out Directory to write files to 410 * @return copied files list 411 * @see TextTemplate#processText(InputStream, OutputStreamWriter) 412 * @see TextTemplate#processFreemarker(File, File) 413 */ 414 public List<String> processDirectory(File in, File out) throws FileNotFoundException, IOException, 415 TemplateException { 416 List<String> newFiles = new ArrayList<>(); 417 if (in.isFile()) { 418 if (out.isDirectory()) { 419 out = new File(out, in.getName()); 420 } 421 if (!out.getParentFile().exists()) { 422 out.getParentFile().mkdirs(); 423 } 424 425 boolean processAsText = false; 426 boolean processAsFreemarker = false; 427 // Check for each extension if it matches end of filename 428 String filename = in.getName().toLowerCase(); 429 for (String ext : freemarkerExtensions) { 430 if (filename.endsWith(ext)) { 431 processAsFreemarker = true; 432 out = new File(out.getCanonicalPath().replaceAll("\\.*" + Pattern.quote(ext) + "$", "")); 433 if (filename.equals("." + ext.toLowerCase())) { 434 throw new IOException("Extension only as a filename is not allowed: " + in.getAbsolutePath()); 435 } 436 break; 437 } 438 } 439 if (!processAsFreemarker) { 440 for (String ext : plainTextExtensions) { 441 if (filename.endsWith(ext)) { 442 processAsText = true; 443 break; 444 } 445 } 446 } 447 448 // Backup existing file if not already done 449 if (out.exists()) { 450 File backup = new File(out.getPath() + ".bak"); 451 if (!backup.exists()) { 452 log.debug("Backup " + out); 453 FileUtils.copyFile(out, backup); 454 newFiles.add(backup.getPath()); 455 } 456 } else { 457 newFiles.add(out.getPath()); 458 } 459 try { 460 if (processAsFreemarker) { 461 log.debug("Process as FreeMarker " + in.getPath()); 462 processFreemarker(in, out); 463 } else if (processAsText) { 464 log.debug("Process as Text " + in.getPath()); 465 try (InputStream is = new FileInputStream(in); 466 OutputStreamWriter os = new OutputStreamWriter(new FileOutputStream(out), "UTF-8")) { 467 processText(is, os); 468 } 469 } else { 470 log.debug("Process as copy " + in.getPath()); 471 FileUtils.copyFile(in, out); 472 } 473 } catch (IOException | TemplateException e) { 474 log.error("Failure on " + in.getPath()); 475 throw e; 476 } 477 } else if (in.isDirectory()) { 478 if (!out.exists()) { 479 // allow renaming destination directory 480 out.mkdirs(); 481 } else if (!out.getName().equals(in.getName())) { 482 // allow copy over existing hierarchy 483 out = new File(out, in.getName()); 484 out.mkdir(); 485 } 486 for (File file : in.listFiles()) { 487 newFiles.addAll(processDirectory(file, out)); 488 } 489 } 490 return newFiles; 491 } 492 493 /** 494 * @param extensionsList comma-separated list of files extensions to parse 495 */ 496 public void setTextParsingExtensions(String extensionsList) { 497 StringTokenizer st = new StringTokenizer(extensionsList, ","); 498 plainTextExtensions = new ArrayList<>(); 499 while (st.hasMoreTokens()) { 500 String extension = st.nextToken().toLowerCase(); 501 plainTextExtensions.add(extension); 502 } 503 } 504 505 public void setFreemarkerParsingExtensions(String extensionsList) { 506 StringTokenizer st = new StringTokenizer(extensionsList, ","); 507 freemarkerExtensions = new ArrayList<>(); 508 while (st.hasMoreTokens()) { 509 String extension = st.nextToken().toLowerCase(); 510 freemarkerExtensions.add(extension); 511 } 512 } 513 514 /** 515 * Whether to replace or not the variables which value is encrypted. 516 * 517 * @param keepEncryptedAsVar if {@code true}, the variables which value is encrypted won't be expanded 518 * @since 7.4 519 */ 520 public void setKeepEncryptedAsVar(boolean keepEncryptedAsVar) { 521 if (this.keepEncryptedAsVar != keepEncryptedAsVar) { 522 this.keepEncryptedAsVar = keepEncryptedAsVar; 523 freemarkerConfiguration = null; 524 } 525 } 526 527}