001/*
002 * (C) Copyright 2006-2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Nuxeo - initial API and implementation
018 *     bstefanescu, jcarsique
019 *     Anahide Tchertchian
020 *
021 */
022
023package org.nuxeo.common.utils;
024
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.FileNotFoundException;
028import java.io.FileOutputStream;
029import java.io.FileWriter;
030import java.io.FilterWriter;
031import java.io.IOException;
032import java.io.InputStream;
033import java.io.OutputStream;
034import java.io.Writer;
035import java.util.ArrayList;
036import java.util.HashMap;
037import java.util.List;
038import java.util.Map;
039import java.util.Properties;
040import java.util.StringTokenizer;
041import java.util.regex.Matcher;
042import java.util.regex.Pattern;
043
044import org.apache.commons.io.Charsets;
045import org.apache.commons.io.FileUtils;
046import org.apache.commons.io.IOUtils;
047import org.apache.commons.logging.Log;
048import org.apache.commons.logging.LogFactory;
049
050import org.nuxeo.common.codec.Crypto;
051import org.nuxeo.common.codec.CryptoProperties;
052
053import freemarker.template.Configuration;
054import freemarker.template.Template;
055import freemarker.template.TemplateException;
056
057/**
058 * Text template processing.
059 * <p>
060 * Copy files or directories replacing parameters matching pattern '${[a-zA-Z_0-9\-\.]+}' with values from a
061 * {@link CryptoProperties}.
062 * <p>
063 * If the value of a variable is encrypted:
064 *
065 * <pre>
066 * setVariable(&quot;var&quot;, Crypto.encrypt(value.getBytes))
067 * </pre>
068 *
069 * then "<code>${var}</code>" will be replaced with:
070 * <ul>
071 * <li>its decrypted value by default: "<code>value</code>"</li>
072 * <li>"<code>${var}</code>" after a call to "<code>setKeepEncryptedAsVar(true)}</code>"
073 * </ul>
074 * and "<code>${#var}</code>" will always be replaced with its decrypted value.
075 * <p>
076 * Since 5.7.2, variables can have a default value using syntax ${parameter:=defaultValue}. The default value will be
077 * used if parameter is null or unset.
078 * <p>
079 * Methods {@link #setTextParsingExtensions(String)} and {@link #setFreemarkerParsingExtensions(String)} allow to set
080 * the list of files being processed when using {@link #processDirectory(File, File)}, based on their extension; others
081 * being simply copied.
082 *
083 * @author <a href="mailto:bs@nuxeo.com">Bogdan Stefanescu</a>
084 * @see CryptoProperties
085 * @see #setKeepEncryptedAsVar(boolean)
086 * @see #setFreemarkerParsingExtensions(String)
087 * @see #setTextParsingExtensions(String)
088 */
089public class TextTemplate {
090
091    private static final Log log = LogFactory.getLog(TextTemplate.class);
092
093    private static final int MAX_RECURSION_LEVEL = 10;
094
095    private static final String PATTERN_GROUP_DECRYPT = "decrypt";
096
097    private static final String PATTERN_GROUP_VAR = "var";
098
099    private static final String PATTERN_GROUP_DEFAULT = "default";
100
101    /**
102     * matches variables of the form "${[#]embeddedVar[:=defaultValue]}" but not those starting with "$${"
103     */
104    private static final Pattern PATTERN = Pattern.compile("(?<!\\$)\\$\\{(?<" + PATTERN_GROUP_DECRYPT + ">#)?" //
105            + "(?<" + PATTERN_GROUP_VAR + ">[a-zA-Z_0-9\\-\\.]+)" // embeddedVar
106            + "(:=(?<" + PATTERN_GROUP_DEFAULT + ">.*))?\\}"); // defaultValue
107
108    private final CryptoProperties vars;
109
110    private Properties processedVars;
111
112    private boolean trim = false;
113
114    private List<String> plainTextExtensions;
115
116    private List<String> freemarkerExtensions = new ArrayList<>();
117
118    private Configuration freemarkerConfiguration = null;
119
120    private Map<String, Object> freemarkerVars = null;
121
122    private boolean keepEncryptedAsVar;
123
124    public boolean isTrim() {
125        return trim;
126    }
127
128    /**
129     * Set to true in order to trim invisible characters (spaces) from values.
130     */
131    public void setTrim(boolean trim) {
132        this.trim = trim;
133    }
134
135    public TextTemplate() {
136        vars = new CryptoProperties();
137    }
138
139    /**
140     * {@link #TextTemplate(Properties)} provides an additional default values behavior
141     *
142     * @see #TextTemplate(Properties)
143     */
144    public TextTemplate(Map<String, String> vars) {
145        this.vars = new CryptoProperties();
146        this.vars.putAll(vars);
147    }
148
149    /**
150     * @param vars Properties containing keys and values for template processing
151     */
152    public TextTemplate(Properties vars) {
153        if (vars instanceof CryptoProperties) {
154            this.vars = (CryptoProperties) vars;
155        } else {
156            this.vars = new CryptoProperties(vars);
157        }
158    }
159
160    public void setVariables(Map<String, String> vars) {
161        this.vars.putAll(vars);
162        freemarkerConfiguration = null;
163    }
164
165    /**
166     * If adding multiple variables, prefer use of {@link #setVariables(Map)}
167     */
168    public void setVariable(String name, String value) {
169        vars.setProperty(name, value);
170        freemarkerConfiguration = null;
171    }
172
173    public String getVariable(String name) {
174        return vars.getProperty(name, keepEncryptedAsVar);
175    }
176
177    public Properties getVariables() {
178        return vars;
179    }
180
181    /**
182     * @deprecated Since 7.4. Use {@link #processText(CharSequence)} instead.
183     */
184    @Deprecated
185    public String process(CharSequence text) {
186        return processText(text);
187    }
188
189    /**
190     * @deprecated Since 7.4. Use {@link #processText(InputStream)} instead.
191     */
192    @Deprecated
193    public String process(InputStream in) throws IOException {
194        return processText(in);
195    }
196
197    /**
198     * @deprecated Since 7.4. Use {@link #processText(InputStream, OutputStream)} instead.
199     */
200    @Deprecated
201    public void process(InputStream in, OutputStream out) throws IOException {
202        processText(in, out);
203    }
204
205    /**
206     * @param processText if true, text is processed for parameters replacement
207     * @deprecated Since 7.4. Use {@link #processText(InputStream, OutputStream)} (if {@code processText}) or
208     *             {@link IOUtils#copy(InputStream, OutputStream)}
209     */
210    @Deprecated
211    public void process(InputStream is, OutputStream os, boolean processText) throws IOException {
212        if (processText) {
213            String text = IOUtils.toString(is, Charsets.UTF_8);
214            text = processText(text);
215            os.write(text.getBytes());
216        } else {
217            IOUtils.copy(is, os);
218        }
219    }
220
221    /**
222     * That method is not recursive. It processes the given text only once.
223     *
224     * @param props CryptoProperties containing the variable values
225     * @param text Text to process
226     * @return the processed text
227     * @since 7.4
228     */
229    protected String processString(CryptoProperties props, String text) {
230        Matcher m = PATTERN.matcher(text);
231        StringBuffer sb = new StringBuffer();
232        while (m.find()) {
233            // newVarsValue == ${[#]embeddedVar[:=default]}
234            String embeddedVar = m.group(PATTERN_GROUP_VAR);
235            String value = props.getProperty(embeddedVar, keepEncryptedAsVar);
236            if (value == null) {
237                value = m.group(PATTERN_GROUP_DEFAULT);
238            }
239            if (value != null) {
240                if (trim) {
241                    value = value.trim();
242                }
243                if (Crypto.isEncrypted(value)) {
244                    if (keepEncryptedAsVar && m.group(PATTERN_GROUP_DECRYPT) == null) {
245                        value = "${" + embeddedVar + "}";
246                    } else {
247                        value = new String(vars.getCrypto().decrypt(value));
248                    }
249                }
250
251                // Allow use of backslash and dollars characters
252                value = Matcher.quoteReplacement(value);
253                m.appendReplacement(sb, value);
254            }
255        }
256        m.appendTail(sb);
257        return sb.toString();
258    }
259
260    /**
261     * unescape variables
262     */
263    protected Properties unescape(Properties props) {
264        for (Object key : props.keySet()) {
265            props.put(key, unescape((String) props.get(key)));
266        }
267        return props;
268    }
269
270    protected String unescape(String value) {
271        // unescape doubled $ characters, only if in front of a {
272        return value.replaceAll("\\$\\$\\{", "\\${");
273    }
274
275    private void preprocessVars() {
276        processedVars = preprocessVars(vars);
277    }
278
279    public Properties preprocessVars(Properties unprocessedVars) {
280        CryptoProperties newVars = new CryptoProperties(unprocessedVars);
281        boolean doneProcessing = false;
282        int recursionLevel = 0;
283        while (!doneProcessing) {
284            doneProcessing = true;
285            for (String newVarsKey : newVars.stringPropertyNames()) {
286                String newVarsValue = newVars.getProperty(newVarsKey, keepEncryptedAsVar);
287                if (newVarsValue == null) {
288                    continue;
289                }
290                if (Crypto.isEncrypted(newVarsValue)) {
291                    // newVarsValue == {$[...]$...}
292                    assert (keepEncryptedAsVar);
293                    newVarsValue = "${" + newVarsKey + "}";
294                    newVars.put(newVarsKey, newVarsValue);
295                    continue;
296                }
297
298                String replacementValue = processString(newVars, newVarsValue);
299                if (!replacementValue.equals(newVarsValue)) {
300                    doneProcessing = false;
301                    newVars.put(newVarsKey, replacementValue);
302                }
303            }
304            recursionLevel++;
305            // Avoid infinite replacement loops
306            if ((!doneProcessing) && (recursionLevel > MAX_RECURSION_LEVEL)) {
307                log.warn("Detected potential infinite loop when processing the following properties\n" + newVars);
308                break;
309            }
310        }
311        return unescape(newVars);
312    }
313
314    /**
315     * @deprecated Since 7.4. Use {@link #processText(String)}
316     */
317    @Deprecated
318    public String processText(CharSequence text) {
319        return processText(text.toString());
320    }
321
322    /**
323     * @since 7.4
324     */
325    public String processText(String text) {
326        if (text == null) {
327            return null;
328        }
329        boolean doneProcessing = false;
330        int recursionLevel = 0;
331        while (!doneProcessing) {
332            doneProcessing = true;
333            String processedText = processString(vars, text);
334            if (!processedText.equals(text)) {
335                doneProcessing = false;
336                text = processedText;
337            }
338            recursionLevel++;
339            // Avoid infinite replacement loops
340            if ((!doneProcessing) && (recursionLevel > MAX_RECURSION_LEVEL)) {
341                log.warn("Detected potential infinite loop when processing the following text\n" + text);
342                break;
343            }
344        }
345        return unescape(text);
346    }
347
348    public String processText(InputStream in) throws IOException {
349        String text = IOUtils.toString(in, Charsets.UTF_8);
350        return processText(text);
351    }
352
353    public void processText(InputStream is, OutputStream os) throws IOException {
354        String text = IOUtils.toString(is, Charsets.UTF_8);
355        text = processText(text);
356        os.write(text.getBytes(Charsets.UTF_8));
357    }
358
359    /**
360     * Initialize FreeMarker data model from Java properties.
361     * <p>
362     * Variables in the form "{@code foo.bar}" (String with dots) are transformed to "{@code foo[bar]}" (arrays).<br>
363     * So there will be conflicts if a variable name is equal to the prefix of another variable. For instance, "
364     * {@code foo.bar}" and "{@code foo.bar.qux}" will conflict.<br>
365     * When a conflict occurs, the conflicting variable is ignored and a warning is logged. The ignored variable will
366     * usually be the shortest one (without any contract on this behavior).
367     */
368    @SuppressWarnings("unchecked")
369    public void initFreeMarker() {
370        freemarkerConfiguration = new Configuration(Configuration.getVersion());
371        preprocessVars();
372        freemarkerVars = new HashMap<>();
373        Map<String, Object> currentMap;
374        String currentString;
375        KEYS: for (String key : processedVars.stringPropertyNames()) {
376            String value = processedVars.getProperty(key);
377            String[] keyparts = key.split("\\.");
378            currentMap = freemarkerVars;
379            currentString = "";
380            for (int i = 0; i < (keyparts.length - 1); i++) {
381                currentString = currentString + ("".equals(currentString) ? "" : ".") + keyparts[i];
382                if (!currentMap.containsKey(keyparts[i])) {
383                    Map<String, Object> nextMap = new HashMap<>();
384                    currentMap.put(keyparts[i], nextMap);
385                    currentMap = nextMap;
386                } else if (currentMap.get(keyparts[i]) instanceof Map<?, ?>) {
387                    currentMap = (Map<String, Object>) currentMap.get(keyparts[i]);
388                } else {
389                    // silently ignore known conflicts between Java properties and FreeMarker model
390                    if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding")
391                            && !key.startsWith("audit.elasticsearch")) {
392                        log.warn(String.format("FreeMarker variables: ignored '%s' conflicting with '%s'", key,
393                                currentString));
394                    }
395                    continue KEYS;
396                }
397            }
398            if (!currentMap.containsKey(keyparts[keyparts.length - 1])) {
399                currentMap.put(keyparts[keyparts.length - 1], value);
400            } else if (!key.startsWith("java.vendor") && !key.startsWith("file.encoding")
401                    && !key.startsWith("audit.elasticsearch")) {
402                Map<String, Object> currentValue = (Map<String, Object>) currentMap.get(keyparts[keyparts.length - 1]);
403                log.warn(String.format("FreeMarker variables: ignored '%2$s' conflicting with '%2$s.%1$s'",
404                        currentValue.keySet(), key));
405            }
406        }
407    }
408
409    public void processFreemarker(File in, File out) throws IOException, TemplateException {
410        if (freemarkerConfiguration == null) {
411            initFreeMarker();
412        }
413        freemarkerConfiguration.setDirectoryForTemplateLoading(in.getParentFile());
414        Template nxtpl = freemarkerConfiguration.getTemplate(in.getName());
415        try (Writer writer = new EscapeVariableFilter(new FileWriter(out))) {
416            nxtpl.process(freemarkerVars, writer);
417        }
418    }
419
420    protected static class EscapeVariableFilter extends FilterWriter {
421
422        protected static final int DOLLAR_SIGN = "$".codePointAt(0);
423
424        protected int last;
425
426        public EscapeVariableFilter(Writer out) {
427            super(out);
428        }
429
430        public @Override void write(int b) throws IOException {
431            if (b == DOLLAR_SIGN && last == DOLLAR_SIGN) {
432                return;
433            }
434            last = b;
435            super.write(b);
436        }
437
438        @Override
439        public void write(char[] cbuf, int off, int len) throws IOException {
440            for (int i = 0; i < len; ++i) {
441                write(cbuf[off + i]);
442            }
443        }
444
445        @Override
446        public void write(char[] cbuf) throws IOException {
447            write(cbuf, 0, cbuf.length);
448        }
449
450    }
451
452    /**
453     * Recursively process each file from "in" directory to "out" directory.
454     *
455     * @param in Directory to read files from
456     * @param out Directory to write files to
457     * @return copied files list
458     * @see TextTemplate#processText(InputStream, OutputStream)
459     * @see TextTemplate#processFreemarker(File, File)
460     */
461    public List<String> processDirectory(File in, File out) throws FileNotFoundException, IOException,
462            TemplateException {
463        List<String> newFiles = new ArrayList<>();
464        if (in.isFile()) {
465            if (out.isDirectory()) {
466                out = new File(out, in.getName());
467            }
468            if (!out.getParentFile().exists()) {
469                out.getParentFile().mkdirs();
470            }
471
472            boolean processAsText = false;
473            boolean processAsFreemarker = false;
474            // Check for each extension if it matches end of filename
475            String filename = in.getName().toLowerCase();
476            for (String ext : freemarkerExtensions) {
477                if (filename.endsWith(ext)) {
478                    processAsFreemarker = true;
479                    out = new File(out.getCanonicalPath().replaceAll("\\.*" + Pattern.quote(ext) + "$", ""));
480                    break;
481                }
482            }
483            if (!processAsFreemarker) {
484                for (String ext : plainTextExtensions) {
485                    if (filename.endsWith(ext)) {
486                        processAsText = true;
487                        break;
488                    }
489                }
490            }
491
492            // Backup existing file if not already done
493            if (out.exists()) {
494                File backup = new File(out.getPath() + ".bak");
495                if (!backup.exists()) {
496                    log.debug("Backup " + out);
497                    FileUtils.copyFile(out, backup);
498                    newFiles.add(backup.getPath());
499                }
500            } else {
501                newFiles.add(out.getPath());
502            }
503            try {
504                if (processAsFreemarker) {
505                    log.debug("Process as FreeMarker " + in.getPath());
506                    processFreemarker(in, out);
507                } else if (processAsText) {
508                    log.debug("Process as Text " + in.getPath());
509                    InputStream is = null;
510                    OutputStream os = null;
511                    try {
512                        is = new FileInputStream(in);
513                        os = new FileOutputStream(out);
514                        processText(is, os);
515                    } finally {
516                        IOUtils.closeQuietly(is);
517                        IOUtils.closeQuietly(os);
518                    }
519                } else {
520                    log.debug("Process as copy " + in.getPath());
521                    FileUtils.copyFile(in, out);
522                }
523            } catch (IOException | TemplateException e) {
524                log.error("Failure on " + in.getPath());
525                throw e;
526            }
527        } else if (in.isDirectory()) {
528            if (!out.exists()) {
529                // allow renaming destination directory
530                out.mkdirs();
531            } else if (!out.getName().equals(in.getName())) {
532                // allow copy over existing hierarchy
533                out = new File(out, in.getName());
534                out.mkdir();
535            }
536            for (File file : in.listFiles()) {
537                newFiles.addAll(processDirectory(file, out));
538            }
539        }
540        return newFiles;
541    }
542
543    /**
544     * @param extensionsList comma-separated list of files extensions to parse
545     * @deprecated Since 7.4. Use {@link #setTextParsingExtensions(String)} instead.
546     * @see #setTextParsingExtensions(String)
547     * @see #setFreemarkerParsingExtensions(String)
548     */
549    @Deprecated
550    public void setParsingExtensions(String extensionsList) {
551        setTextParsingExtensions(extensionsList);
552    }
553
554    /**
555     * @param extensionsList comma-separated list of files extensions to parse
556     */
557    public void setTextParsingExtensions(String extensionsList) {
558        StringTokenizer st = new StringTokenizer(extensionsList, ",");
559        plainTextExtensions = new ArrayList<>();
560        while (st.hasMoreTokens()) {
561            String extension = st.nextToken().toLowerCase();
562            plainTextExtensions.add(extension);
563        }
564    }
565
566    public void setFreemarkerParsingExtensions(String extensionsList) {
567        StringTokenizer st = new StringTokenizer(extensionsList, ",");
568        freemarkerExtensions = new ArrayList<>();
569        while (st.hasMoreTokens()) {
570            String extension = st.nextToken().toLowerCase();
571            freemarkerExtensions.add(extension);
572        }
573    }
574
575    /**
576     * Whether to replace or not the variables which value is encrypted.
577     *
578     * @param keepEncryptedAsVar if {@code true}, the variables which value is encrypted won't be expanded
579     * @since 7.4
580     */
581    public void setKeepEncryptedAsVar(boolean keepEncryptedAsVar) {
582        if (this.keepEncryptedAsVar != keepEncryptedAsVar) {
583            this.keepEncryptedAsVar = keepEncryptedAsVar;
584            freemarkerConfiguration = null;
585        }
586    }
587
588}