001/*
002 * (C) Copyright 2014-2020 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023
024import java.time.ZonedDateTime;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Date;
028import java.util.LinkedList;
029import java.util.List;
030import java.util.Set;
031import java.util.regex.Pattern;
032
033import org.bson.Document;
034import org.nuxeo.common.utils.DateUtils;
035import org.nuxeo.ecm.core.query.QueryParseException;
036import org.nuxeo.ecm.core.query.sql.NXQL;
037import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral;
038import org.nuxeo.ecm.core.query.sql.model.DateLiteral;
039import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral;
040import org.nuxeo.ecm.core.query.sql.model.Expression;
041import org.nuxeo.ecm.core.query.sql.model.Function;
042import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral;
043import org.nuxeo.ecm.core.query.sql.model.Literal;
044import org.nuxeo.ecm.core.query.sql.model.LiteralList;
045import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
046import org.nuxeo.ecm.core.query.sql.model.Operand;
047import org.nuxeo.ecm.core.query.sql.model.Operator;
048import org.nuxeo.ecm.core.query.sql.model.Reference;
049import org.nuxeo.ecm.core.query.sql.model.StringLiteral;
050import org.nuxeo.ecm.core.schema.types.ListType;
051import org.nuxeo.ecm.core.schema.types.Type;
052import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
053import org.nuxeo.ecm.core.schema.types.primitives.DateType;
054import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
055import org.nuxeo.ecm.core.storage.QueryOptimizer.PrefixInfo;
056import org.nuxeo.runtime.api.Framework;
057import org.nuxeo.runtime.mongodb.MongoDBOperators;
058import org.nuxeo.runtime.services.config.ConfigurationService;
059
060/**
061 * Abstract query builder for a MongoDB query from an {@link Expression}.
062 * <p>
063 * Must be customized by defining an implementation for the {@link #walkReference(String)} method.
064 *
065 * @since 5.9.4
066 */
067public abstract class MongoDBAbstractQueryBuilder {
068
069    public static final Long LONG_ZERO = Long.valueOf(0);
070
071    public static final Long LONG_ONE = Long.valueOf(1);
072
073    public static final Double ONE = Double.valueOf(1);
074
075    public static final Double MINUS_ONE = Double.valueOf(-1);
076
077    protected static final String DATE_CAST = "DATE";
078
079    protected static final String LIKE_ANCHORED_PROP = "nuxeo.mongodb.like.anchored";
080
081    protected final MongoDBConverter converter;
082
083    protected final Expression expression;
084
085    protected Document query;
086
087    /**
088     * Prefix to remove for $elemMatch (including final dot), or {@code null} if there's no current prefix to remove.
089     */
090    protected String elemMatchPrefix;
091
092    protected boolean likeAnchored;
093
094    public MongoDBAbstractQueryBuilder(MongoDBConverter converter, Expression expression) {
095        this.converter = converter;
096        this.expression = expression;
097        likeAnchored = Framework.getService(ConfigurationService.class).isBooleanTrue(LIKE_ANCHORED_PROP);
098    }
099
100    public void walk() {
101        if (expression instanceof MultiExpression && ((MultiExpression) expression).predicates.isEmpty()) {
102            // special-case empty query
103            query = new Document();
104        } else {
105            query = walkExpression(expression);
106        }
107    }
108
109    public Document getQuery() {
110        return query;
111    }
112
113    public Document walkExpression(Expression expr) {
114        Operator op = expr.operator;
115        Operand lvalue = expr.lvalue;
116        Operand rvalue = expr.rvalue;
117        Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null;
118        String name = ref != null ? ref.name : null;
119        String cast = ref != null ? ref.cast : null;
120        if (DATE_CAST.equals(cast)) {
121            checkDateLiteralForCast(op, rvalue, name);
122        }
123        if (op == Operator.SUM) {
124            throw new UnsupportedOperationException("SUM");
125        } else if (op == Operator.SUB) {
126            throw new UnsupportedOperationException("SUB");
127        } else if (op == Operator.MUL) {
128            throw new UnsupportedOperationException("MUL");
129        } else if (op == Operator.DIV) {
130            throw new UnsupportedOperationException("DIV");
131        } else if (op == Operator.LT) {
132            return walkLt(lvalue, rvalue);
133        } else if (op == Operator.GT) {
134            return walkGt(lvalue, rvalue);
135        } else if (op == Operator.EQ) {
136            return walkEq(lvalue, rvalue);
137        } else if (op == Operator.NOTEQ) {
138            return walkNotEq(lvalue, rvalue);
139        } else if (op == Operator.LTEQ) {
140            return walkLtEq(lvalue, rvalue);
141        } else if (op == Operator.GTEQ) {
142            return walkGtEq(lvalue, rvalue);
143        } else if (op == Operator.AND || op == Operator.OR) {
144            if (expr instanceof MultiExpression) {
145                return walkAndOrMultiExpression((MultiExpression) expr);
146            } else {
147                return walkAndOr(expr);
148            }
149        } else if (op == Operator.NOT) {
150            return walkNot(lvalue);
151        } else if (op == Operator.LIKE) {
152            return walkLike(lvalue, rvalue, true, false);
153        } else if (op == Operator.ILIKE) {
154            return walkLike(lvalue, rvalue, true, true);
155        } else if (op == Operator.NOTLIKE) {
156            return walkLike(lvalue, rvalue, false, false);
157        } else if (op == Operator.NOTILIKE) {
158            return walkLike(lvalue, rvalue, false, true);
159        } else if (op == Operator.IN) {
160            return walkIn(lvalue, rvalue, true);
161        } else if (op == Operator.NOTIN) {
162            return walkIn(lvalue, rvalue, false);
163        } else if (op == Operator.ISNULL) {
164            return walkIsNull(lvalue);
165        } else if (op == Operator.ISNOTNULL) {
166            return walkIsNotNull(lvalue);
167        } else if (op == Operator.BETWEEN) {
168            return walkBetween(lvalue, rvalue, true);
169        } else if (op == Operator.NOTBETWEEN) {
170            return walkBetween(lvalue, rvalue, false);
171        } else {
172            throw new QueryParseException("Unknown operator: " + op);
173        }
174    }
175
176    protected void checkDateLiteralForCast(Operator op, Operand value, String name) {
177        if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) {
178            LiteralList l = (LiteralList) value;
179            checkDateLiteralForCast(l.get(0), name);
180            checkDateLiteralForCast(l.get(1), name);
181        } else {
182            checkDateLiteralForCast(value, name);
183        }
184    }
185
186    protected void checkDateLiteralForCast(Operand value, String name) {
187        if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) {
188            throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name);
189        }
190    }
191
192    public Document walkNot(Operand value) {
193        Object val = walkOperand(null, value);
194        Object not = pushDownNot(val);
195        if (!(not instanceof Document)) {
196            throw new QueryParseException("Cannot do NOT on: " + val);
197        }
198        return (Document) not;
199    }
200
201    protected Object pushDownNot(Object object) {
202        if (!(object instanceof Document)) {
203            throw new QueryParseException("Cannot do NOT on: " + object);
204        }
205        Document ob = (Document) object;
206        Set<String> keySet = ob.keySet();
207        if (keySet.size() != 1) {
208            throw new QueryParseException("Cannot do NOT on: " + ob);
209        }
210        String key = keySet.iterator().next();
211        Object value = ob.get(key);
212        if (!key.startsWith("$")) {
213            if (value instanceof Document) {
214                // push down inside dbobject
215                return new Document(key, pushDownNot(value));
216            } else {
217                // k = v -> k != v
218                return new Document(key, new Document(MongoDBOperators.NE, value));
219            }
220        }
221        if (MongoDBOperators.NE.equals(key)) {
222            // NOT k != v -> k = v
223            return value;
224        }
225        if (MongoDBOperators.NOT.equals(key)) {
226            // NOT NOT v -> v
227            return value;
228        }
229        if (MongoDBOperators.AND.equals(key) || MongoDBOperators.OR.equals(key)) {
230            // boolean algebra
231            // NOT (v1 AND v2) -> NOT v1 OR NOT v2
232            // NOT (v1 OR v2) -> NOT v1 AND NOT v2
233            String op = MongoDBOperators.AND.equals(key) ? MongoDBOperators.OR : MongoDBOperators.AND;
234            List<Object> list = (List<Object>) value;
235            for (int i = 0; i < list.size(); i++) {
236                list.set(i, pushDownNot(list.get(i)));
237            }
238            return new Document(op, list);
239        }
240        if (MongoDBOperators.IN.equals(key) || MongoDBOperators.NIN.equals(key)) {
241            // boolean algebra
242            // IN <-> NIN
243            String op = MongoDBOperators.IN.equals(key) ? MongoDBOperators.NIN : MongoDBOperators.IN;
244            return new Document(op, value);
245        }
246        if (MongoDBOperators.LT.equals(key) || MongoDBOperators.GT.equals(key) || MongoDBOperators.LTE.equals(key)
247                || MongoDBOperators.GTE.equals(key)) {
248            // TODO use inverse operators?
249            return new Document(MongoDBOperators.NOT, ob);
250        }
251        throw new QueryParseException("Unknown operator for NOT: " + key);
252    }
253
254    protected Document newDocumentWithField(FieldInfo fieldInfo, Object value) {
255        return new Document(fieldInfo.queryField, value);
256    }
257
258    public Document walkIsNull(Operand value) {
259        FieldInfo fieldInfo = walkReference(value);
260        return newDocumentWithField(fieldInfo, null);
261    }
262
263    public Document walkIsNotNull(Operand value) {
264        FieldInfo fieldInfo = walkReference(value);
265        return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.NE, null));
266    }
267
268    public Document walkAndOrMultiExpression(MultiExpression expr) {
269        return walkAndOr(expr, expr.predicates);
270    }
271
272    public Document walkAndOr(Expression expr) {
273        return walkAndOr(expr, Arrays.asList(expr.lvalue, expr.rvalue));
274    }
275
276    protected static final Pattern SLASH_WILDCARD_SLASH = Pattern.compile("/\\*\\d+(/)?");
277
278    protected Document walkAndOr(Expression expr, List<? extends Operand> values) {
279        if (values.size() == 1) {
280            return (Document) walkOperand(null, values.get(0));
281        }
282        boolean and = expr.operator == Operator.AND;
283        String op = and ? MongoDBOperators.AND : MongoDBOperators.OR;
284        // PrefixInfo was computed by the QueryOptimizer for common AND predicates
285        PrefixInfo info = (PrefixInfo) expr.getInfo();
286        if (info == null || info.count < 2 || !and) {
287            List<Object> list = walkOperandList(values);
288            return new Document(op, list);
289        }
290
291        // we have a common prefix for all underlying references, extract it into an $elemMatch node
292
293        // info.prefix is the DBS common prefix, ex: foo/bar/*1; ecm:acp/*1/acl/*1
294        // compute MongoDB prefix: foo.bar.; ecm:acp.acl.
295        String prefix = SLASH_WILDCARD_SLASH.matcher(info.prefix).replaceAll(".");
296        // remove current prefix and trailing . for actual field match
297        String fieldBase = stripElemMatchPrefix(prefix.substring(0, prefix.length() - 1));
298
299        String previousElemMatchPrefix = elemMatchPrefix;
300        elemMatchPrefix = prefix;
301        List<Object> list = walkOperandList(values);
302        elemMatchPrefix = previousElemMatchPrefix;
303
304        return new Document(fieldBase, new Document(MongoDBOperators.ELEM_MATCH, new Document(op, list)));
305    }
306
307    protected String stripElemMatchPrefix(String field) {
308        if (elemMatchPrefix != null && field.startsWith(elemMatchPrefix)) {
309            field = field.substring(elemMatchPrefix.length());
310        }
311        return field;
312    }
313
314    public Document walkEq(Operand lvalue, Operand rvalue) {
315        FieldInfo fieldInfo = walkReference(lvalue);
316        return walkEq(fieldInfo, rvalue);
317    }
318
319    public Document walkEq(FieldInfo fieldInfo, Operand rvalue) {
320        Object right = walkOperand(fieldInfo, rvalue);
321        return newDocumentWithField(fieldInfo, right);
322    }
323
324    public Document walkNotEq(Operand lvalue, Operand rvalue) {
325        FieldInfo fieldInfo = walkReference(lvalue);
326        return walkNotEq(fieldInfo, rvalue);
327    }
328
329    public Document walkNotEq(FieldInfo fieldInfo, Operand rvalue) {
330        Object right = walkOperand(fieldInfo, rvalue);
331        return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.NE, right));
332    }
333
334    public Document walkLt(Operand lvalue, Operand rvalue) {
335        FieldInfo fieldInfo = walkReference(lvalue);
336        Object right = walkOperand(fieldInfo, rvalue);
337        return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.LT, right));
338    }
339
340    public Document walkGt(Operand lvalue, Operand rvalue) {
341        FieldInfo fieldInfo = walkReference(lvalue);
342        Object right = walkOperand(fieldInfo, rvalue);
343        return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.GT, right));
344    }
345
346    public Document walkLtEq(Operand lvalue, Operand rvalue) {
347        FieldInfo fieldInfo = walkReference(lvalue);
348        Object right = walkOperand(fieldInfo, rvalue);
349        return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.LTE, right));
350    }
351
352    public Document walkGtEq(Operand lvalue, Operand rvalue) {
353        FieldInfo fieldInfo = walkReference(lvalue);
354        Object right = walkOperand(fieldInfo, rvalue);
355        return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.GTE, right));
356    }
357
358    public Document walkBetween(Operand lvalue, Operand rvalue, boolean positive) {
359        LiteralList l = (LiteralList) rvalue;
360        FieldInfo fieldInfo = walkReference(lvalue);
361        Object left = walkOperand(fieldInfo, l.get(0));
362        Object right = walkOperand(fieldInfo, l.get(1));
363        if (positive) {
364            Document range = new Document();
365            range.put(MongoDBOperators.GTE, left);
366            range.put(MongoDBOperators.LTE, right);
367            return newDocumentWithField(fieldInfo, range);
368        } else {
369            Document a = newDocumentWithField(fieldInfo, new Document(MongoDBOperators.LT, left));
370            Document b = newDocumentWithField(fieldInfo, new Document(MongoDBOperators.GT, right));
371            return new Document(MongoDBOperators.OR, Arrays.asList(a, b));
372        }
373    }
374
375    public Document walkIn(Operand lvalue, Operand rvalue, boolean positive) {
376        FieldInfo fieldInfo = walkReference(lvalue);
377        return walkIn(fieldInfo, rvalue, positive);
378    }
379
380    public Document walkIn(FieldInfo fieldInfo, Operand rvalue, boolean positive) {
381        Object right = walkOperand(fieldInfo, rvalue);
382        if (!(right instanceof List)) {
383            throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue);
384        }
385        // TODO check list fields
386        List<Object> list = (List<Object>) right;
387        return newDocumentWithField(fieldInfo,
388                new Document(positive ? MongoDBOperators.IN : MongoDBOperators.NIN, list));
389    }
390
391    public Document walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) {
392        FieldInfo fieldInfo = walkReference(lvalue);
393        if (!(rvalue instanceof StringLiteral)) {
394            throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue);
395        }
396        // TODO check list fields
397        String like = (String) walkStringLiteral(fieldInfo, (StringLiteral) rvalue);
398        String regex = ExpressionEvaluator.likeToRegex(like);
399        // MongoDB native matches are unanchored: optimize the regex for faster matches
400        if (regex.startsWith(".*")) {
401            regex = regex.substring(2);
402        } else if (likeAnchored) {
403            regex = "^" + regex;
404        }
405        if (regex.endsWith(".*")) {
406            regex = regex.substring(0, regex.length() - 2); // better range index use
407        } else if (likeAnchored) {
408            regex = regex + "$";
409        }
410
411        int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0;
412        Pattern pattern = Pattern.compile(regex, flags);
413        Object value;
414        if (positive) {
415            value = pattern;
416        } else {
417            value = new Document(MongoDBOperators.NOT, pattern);
418        }
419        return newDocumentWithField(fieldInfo, value);
420    }
421
422    public Object walkOperand(FieldInfo fieldInfo, Operand op) {
423        if (op instanceof Literal) {
424            return walkLiteral(fieldInfo, (Literal) op);
425        } else if (op instanceof LiteralList) {
426            return walkLiteralList(fieldInfo, (LiteralList) op);
427        } else if (op instanceof Function) {
428            return walkFunction((Function) op);
429        } else if (op instanceof Expression) {
430            return walkExpression((Expression) op);
431        } else if (op instanceof Reference) {
432            return walkReference((Reference) op);
433        } else {
434            throw new QueryParseException("Unknown operand: " + op);
435        }
436    }
437
438    public Object walkLiteral(FieldInfo fieldInfo, Literal lit) {
439        if (lit instanceof BooleanLiteral) {
440            return walkBooleanLiteral(fieldInfo, (BooleanLiteral) lit);
441        } else if (lit instanceof DateLiteral) {
442            return walkDateLiteral(fieldInfo, (DateLiteral) lit);
443        } else if (lit instanceof DoubleLiteral) {
444            return walkDoubleLiteral(fieldInfo, (DoubleLiteral) lit);
445        } else if (lit instanceof IntegerLiteral) {
446            return walkIntegerLiteral(fieldInfo, (IntegerLiteral) lit);
447        } else if (lit instanceof StringLiteral) {
448            return walkStringLiteral(fieldInfo, (StringLiteral) lit);
449        } else {
450            throw new QueryParseException("Unknown literal: " + lit);
451        }
452    }
453
454    public Object walkBooleanLiteral(FieldInfo fieldInfo, BooleanLiteral lit) {
455        return Boolean.valueOf(lit.value);
456    }
457
458    public Date walkDateLiteral(FieldInfo fieldInfo, DateLiteral lit) {
459        return DateUtils.toDate(lit.value); // TODO onlyDate
460    }
461
462    public Double walkDoubleLiteral(FieldInfo fieldInfo, DoubleLiteral lit) {
463        return Double.valueOf(lit.value);
464    }
465
466    public Object walkIntegerLiteral(FieldInfo fieldInfo, IntegerLiteral lit) {
467        long value = lit.value;
468        if (fieldInfo != null && fieldInfo.isBoolean()) {
469            // convert 0 / 1 to actual booleans
470            Boolean b;
471            if (value == 0) {
472                b = FALSE;
473            } else if (value == 1) {
474                b = TRUE;
475            } else {
476                throw new QueryParseException("Invalid boolean: " + value);
477            }
478            return converter.serializableToBson(fieldInfo.key, b);
479        }
480        return Long.valueOf(value);
481    }
482
483    public Object walkStringLiteral(FieldInfo fieldInfo, StringLiteral lit) {
484        String value = lit.value;
485        if (fieldInfo != null) {
486            return converter.serializableToBson(fieldInfo.key, value);
487        }
488        return value;
489    }
490
491    public List<Object> walkLiteralList(FieldInfo fieldInfo, LiteralList litList) {
492        List<Object> list = new ArrayList<>(litList.size());
493        for (Literal lit : litList) {
494            list.add(walkLiteral(fieldInfo, lit));
495        }
496        return list;
497    }
498
499    protected List<Object> walkOperandList(List<? extends Operand> values) {
500        List<Object> list = new LinkedList<>();
501        for (Operand value : values) {
502            list.add(walkOperand(null, value));
503        }
504        return list;
505    }
506
507    public Object walkFunction(Function func) {
508        String name = func.name;
509        if (NXQL.NOW_FUNCTION.equalsIgnoreCase(name)) {
510            String periodAndDurationText;
511            if (func.args == null || func.args.size() != 1) {
512                periodAndDurationText = null;
513            } else {
514                periodAndDurationText = ((StringLiteral) func.args.get(0)).value;
515            }
516            ZonedDateTime dateTime;
517            try {
518                dateTime = NXQL.nowPlusPeriodAndDuration(periodAndDurationText);
519            } catch (IllegalArgumentException e) {
520                throw new QueryParseException(e);
521            }
522            DateLiteral dateLiteral = new DateLiteral(dateTime);
523            return walkDateLiteral(null, dateLiteral);
524        } else {
525            throw new QueryParseException("Function not supported: " + func);
526        }
527    }
528
529    protected FieldInfo walkReference(Operand value) {
530        if (!(value instanceof Reference)) {
531            throw new QueryParseException("Invalid query, left hand side must be a property: " + value);
532        }
533        return walkReference((Reference) value);
534    }
535
536    public static class FieldInfo {
537
538        /** NXQL property. */
539        public final String prop;
540
541        /**
542         * DBS key.
543         *
544         * @since 11.1
545         */
546        public final String key;
547
548        /** MongoDB field for query. foo/0/bar -&gt; foo.0.bar; foo / * / bar -&gt; foo.bar */
549        public final String queryField;
550
551        /** MongoDB field for projection. */
552        public final String projectionField;
553
554        public final Type type;
555
556        public FieldInfo(String prop, String key, String queryField, String projectionField, Type type) {
557            this.prop = prop;
558            this.key = key;
559            this.queryField = queryField;
560            this.projectionField = projectionField;
561            this.type = type;
562        }
563
564        public boolean isBoolean() {
565            return type instanceof BooleanType;
566        }
567    }
568
569    /**
570     * Returns the MongoDB field for this reference.
571     */
572    public FieldInfo walkReference(Reference ref) {
573        FieldInfo fieldInfo = walkReference(ref.name);
574        if (DATE_CAST.equals(ref.cast)) {
575            Type type = fieldInfo.type;
576            if (!(type instanceof DateType
577                    || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) {
578                throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name);
579            }
580            // fieldInfo.isDateCast = true;
581        }
582        return fieldInfo;
583    }
584
585    /**
586     * Walks a reference, and returns field info about it.
587     */
588    protected abstract FieldInfo walkReference(String name);
589
590}