001/*
002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023
024import java.util.ArrayList;
025import java.util.Arrays;
026import java.util.Date;
027import java.util.HashMap;
028import java.util.LinkedList;
029import java.util.List;
030import java.util.Map;
031import java.util.Set;
032import java.util.regex.Pattern;
033
034import org.bson.Document;
035import org.nuxeo.ecm.core.query.QueryParseException;
036import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral;
037import org.nuxeo.ecm.core.query.sql.model.DateLiteral;
038import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral;
039import org.nuxeo.ecm.core.query.sql.model.Expression;
040import org.nuxeo.ecm.core.query.sql.model.Function;
041import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral;
042import org.nuxeo.ecm.core.query.sql.model.Literal;
043import org.nuxeo.ecm.core.query.sql.model.LiteralList;
044import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
045import org.nuxeo.ecm.core.query.sql.model.Operand;
046import org.nuxeo.ecm.core.query.sql.model.Operator;
047import org.nuxeo.ecm.core.query.sql.model.Reference;
048import org.nuxeo.ecm.core.query.sql.model.StringLiteral;
049import org.nuxeo.ecm.core.schema.types.ListType;
050import org.nuxeo.ecm.core.schema.types.Type;
051import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
052import org.nuxeo.ecm.core.schema.types.primitives.DateType;
053import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
054import org.nuxeo.ecm.core.storage.QueryOptimizer.PrefixInfo;
055import org.nuxeo.runtime.api.Framework;
056import org.nuxeo.runtime.services.config.ConfigurationService;
057
058import com.mongodb.QueryOperators;
059
060/**
061 * Abstract query builder for a MongoDB query from an {@link Expression}.
062 * <p>
063 * Must be customized by defining an implementation for the {@link #walkReference(String)} method.
064 *
065 * @since 5.9.4
066 */
067public abstract class MongoDBAbstractQueryBuilder {
068
069    public static final Long LONG_ZERO = Long.valueOf(0);
070
071    public static final Long LONG_ONE = Long.valueOf(1);
072
073    public static final Double ONE = Double.valueOf(1);
074
075    public static final Double MINUS_ONE = Double.valueOf(-1);
076
077    protected static final String DATE_CAST = "DATE";
078
079    protected static final String LIKE_ANCHORED_PROP = "nuxeo.mongodb.like.anchored";
080
081    protected final MongoDBConverter converter;
082
083    protected final Expression expression;
084
085    protected Document query;
086
087    /**
088     * Prefix to remove for $elemMatch (including final dot), or {@code null} if there's no current prefix to remove.
089     */
090    protected String elemMatchPrefix;
091
092    protected boolean likeAnchored;
093
094    public MongoDBAbstractQueryBuilder(MongoDBConverter converter, Expression expression) {
095        this.converter = converter;
096        this.expression = expression;
097        likeAnchored = !Framework.getService(ConfigurationService.class).isBooleanPropertyFalse(LIKE_ANCHORED_PROP);
098    }
099
100    public void walk() {
101        if (expression instanceof MultiExpression && ((MultiExpression) expression).predicates.isEmpty()) {
102            // special-case empty query
103            query = new Document();
104        } else {
105            query = walkExpression(expression);
106        }
107    }
108
109    public Document getQuery() {
110        return query;
111    }
112
113    public Document walkExpression(Expression expr) {
114        Operator op = expr.operator;
115        Operand lvalue = expr.lvalue;
116        Operand rvalue = expr.rvalue;
117        Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null;
118        String name = ref != null ? ref.name : null;
119        String cast = ref != null ? ref.cast : null;
120        if (DATE_CAST.equals(cast)) {
121            checkDateLiteralForCast(op, rvalue, name);
122        }
123        if (op == Operator.SUM) {
124            throw new UnsupportedOperationException("SUM");
125        } else if (op == Operator.SUB) {
126            throw new UnsupportedOperationException("SUB");
127        } else if (op == Operator.MUL) {
128            throw new UnsupportedOperationException("MUL");
129        } else if (op == Operator.DIV) {
130            throw new UnsupportedOperationException("DIV");
131        } else if (op == Operator.LT) {
132            return walkLt(lvalue, rvalue);
133        } else if (op == Operator.GT) {
134            return walkGt(lvalue, rvalue);
135        } else if (op == Operator.EQ) {
136            return walkEq(lvalue, rvalue);
137        } else if (op == Operator.NOTEQ) {
138            return walkNotEq(lvalue, rvalue);
139        } else if (op == Operator.LTEQ) {
140            return walkLtEq(lvalue, rvalue);
141        } else if (op == Operator.GTEQ) {
142            return walkGtEq(lvalue, rvalue);
143        } else if (op == Operator.AND || op == Operator.OR) {
144            if (expr instanceof MultiExpression) {
145                return walkAndOrMultiExpression((MultiExpression) expr);
146            } else {
147                return walkAndOr(expr);
148            }
149        } else if (op == Operator.NOT) {
150            return walkNot(lvalue);
151        } else if (op == Operator.LIKE) {
152            return walkLike(lvalue, rvalue, true, false);
153        } else if (op == Operator.ILIKE) {
154            return walkLike(lvalue, rvalue, true, true);
155        } else if (op == Operator.NOTLIKE) {
156            return walkLike(lvalue, rvalue, false, false);
157        } else if (op == Operator.NOTILIKE) {
158            return walkLike(lvalue, rvalue, false, true);
159        } else if (op == Operator.IN) {
160            return walkIn(lvalue, rvalue, true);
161        } else if (op == Operator.NOTIN) {
162            return walkIn(lvalue, rvalue, false);
163        } else if (op == Operator.ISNULL) {
164            return walkIsNull(lvalue);
165        } else if (op == Operator.ISNOTNULL) {
166            return walkIsNotNull(lvalue);
167        } else if (op == Operator.BETWEEN) {
168            return walkBetween(lvalue, rvalue, true);
169        } else if (op == Operator.NOTBETWEEN) {
170            return walkBetween(lvalue, rvalue, false);
171        } else {
172            throw new QueryParseException("Unknown operator: " + op);
173        }
174    }
175
176    protected void checkDateLiteralForCast(Operator op, Operand value, String name) {
177        if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) {
178            LiteralList l = (LiteralList) value;
179            checkDateLiteralForCast(l.get(0), name);
180            checkDateLiteralForCast(l.get(1), name);
181        } else {
182            checkDateLiteralForCast(value, name);
183        }
184    }
185
186    protected void checkDateLiteralForCast(Operand value, String name) {
187        if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) {
188            throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name);
189        }
190    }
191
192    public Document walkNot(Operand value) {
193        Object val = walkOperand(value);
194        Object not = pushDownNot(val);
195        if (!(not instanceof Document)) {
196            throw new QueryParseException("Cannot do NOT on: " + val);
197        }
198        return (Document) not;
199    }
200
201    protected Object pushDownNot(Object object) {
202        if (!(object instanceof Document)) {
203            throw new QueryParseException("Cannot do NOT on: " + object);
204        }
205        Document ob = (Document) object;
206        Set<String> keySet = ob.keySet();
207        if (keySet.size() != 1) {
208            throw new QueryParseException("Cannot do NOT on: " + ob);
209        }
210        String key = keySet.iterator().next();
211        Object value = ob.get(key);
212        if (!key.startsWith("$")) {
213            if (value instanceof Document) {
214                // push down inside dbobject
215                return new Document(key, pushDownNot(value));
216            } else {
217                // k = v -> k != v
218                return new Document(key, new Document(QueryOperators.NE, value));
219            }
220        }
221        if (QueryOperators.NE.equals(key)) {
222            // NOT k != v -> k = v
223            return value;
224        }
225        if (QueryOperators.NOT.equals(key)) {
226            // NOT NOT v -> v
227            return value;
228        }
229        if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) {
230            // boolean algebra
231            // NOT (v1 AND v2) -> NOT v1 OR NOT v2
232            // NOT (v1 OR v2) -> NOT v1 AND NOT v2
233            String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND;
234            List<Object> list = (List<Object>) value;
235            for (int i = 0; i < list.size(); i++) {
236                list.set(i, pushDownNot(list.get(i)));
237            }
238            return new Document(op, list);
239        }
240        if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) {
241            // boolean algebra
242            // IN <-> NIN
243            String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN;
244            return new Document(op, value);
245        }
246        if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key)
247                || QueryOperators.GTE.equals(key)) {
248            // TODO use inverse operators?
249            return new Document(QueryOperators.NOT, ob);
250        }
251        throw new QueryParseException("Unknown operator for NOT: " + key);
252    }
253
254    protected Document newDocumentWithField(FieldInfo fieldInfo, Object value) {
255        return new Document(fieldInfo.queryField, value);
256    }
257
258    public Document walkIsNull(Operand value) {
259        FieldInfo fieldInfo = walkReference(value);
260        return newDocumentWithField(fieldInfo, null);
261    }
262
263    public Document walkIsNotNull(Operand value) {
264        FieldInfo fieldInfo = walkReference(value);
265        return newDocumentWithField(fieldInfo, new Document(QueryOperators.NE, null));
266    }
267
268    public Document walkAndOrMultiExpression(MultiExpression expr) {
269        return walkAndOr(expr, expr.predicates);
270    }
271
272    public Document walkAndOr(Expression expr) {
273        return walkAndOr(expr, Arrays.asList(expr.lvalue, expr.rvalue));
274    }
275
276    protected static final Pattern SLASH_WILDCARD_SLASH = Pattern.compile("/\\*\\d+(/)?");
277
278    protected Document walkAndOr(Expression expr, List<? extends Operand> values) {
279        if (values.size() == 1) {
280            return (Document) walkOperand(values.get(0));
281        }
282        boolean and = expr.operator == Operator.AND;
283        String op = and ? QueryOperators.AND : QueryOperators.OR;
284        // PrefixInfo was computed by the QueryOptimizer for common AND predicates
285        PrefixInfo info = (PrefixInfo) expr.getInfo();
286        if (info == null || info.count < 2 || !and) {
287            List<Object> list = walkOperandList(values);
288            return new Document(op, list);
289        }
290
291        // we have a common prefix for all underlying references, extract it into an $elemMatch node
292
293        // info.prefix is the DBS common prefix, ex: foo/bar/*1; ecm:acp/*1/acl/*1
294        // compute MongoDB prefix: foo.bar.; ecm:acp.acl.
295        String prefix = SLASH_WILDCARD_SLASH.matcher(info.prefix).replaceAll(".");
296        // remove current prefix and trailing . for actual field match
297        String fieldBase = stripElemMatchPrefix(prefix.substring(0, prefix.length() - 1));
298
299        String previousElemMatchPrefix = elemMatchPrefix;
300        elemMatchPrefix = prefix;
301        List<Object> list = walkOperandList(values);
302        elemMatchPrefix = previousElemMatchPrefix;
303
304        return new Document(fieldBase, new Document(QueryOperators.ELEM_MATCH, new Document(op, list)));
305    }
306
307    protected String stripElemMatchPrefix(String field) {
308        if (elemMatchPrefix != null && field.startsWith(elemMatchPrefix)) {
309            field = field.substring(elemMatchPrefix.length());
310        }
311        return field;
312    }
313
314    protected Object checkBoolean(FieldInfo fieldInfo, Object right) {
315        if (fieldInfo.isBoolean()) {
316            // convert 0 / 1 to actual booleans
317            if (right instanceof Long) {
318                if (LONG_ZERO.equals(right)) {
319                    right = fieldInfo.isTrueOrNullBoolean ? null : FALSE;
320                } else if (LONG_ONE.equals(right)) {
321                    right = TRUE;
322                } else {
323                    throw new QueryParseException("Invalid boolean: " + right);
324                }
325            }
326        }
327        return right;
328    }
329
330    public Document walkEq(Operand lvalue, Operand rvalue) {
331        FieldInfo fieldInfo = walkReference(lvalue);
332        Object right = walkOperand(rvalue);
333        right = checkBoolean(fieldInfo, right);
334        // TODO check list fields
335        return newDocumentWithField(fieldInfo, right);
336    }
337
338    public Document walkNotEq(Operand lvalue, Operand rvalue) {
339        FieldInfo fieldInfo = walkReference(lvalue);
340        Object right = walkOperand(rvalue);
341        right = checkBoolean(fieldInfo, right);
342        // TODO check list fields
343        return newDocumentWithField(fieldInfo, new Document(QueryOperators.NE, right));
344    }
345
346    public Document walkLt(Operand lvalue, Operand rvalue) {
347        FieldInfo fieldInfo = walkReference(lvalue);
348        Object right = walkOperand(rvalue);
349        return newDocumentWithField(fieldInfo, new Document(QueryOperators.LT, right));
350    }
351
352    public Document walkGt(Operand lvalue, Operand rvalue) {
353        FieldInfo fieldInfo = walkReference(lvalue);
354        Object right = walkOperand(rvalue);
355        return newDocumentWithField(fieldInfo, new Document(QueryOperators.GT, right));
356    }
357
358    public Document walkLtEq(Operand lvalue, Operand rvalue) {
359        FieldInfo fieldInfo = walkReference(lvalue);
360        Object right = walkOperand(rvalue);
361        return newDocumentWithField(fieldInfo, new Document(QueryOperators.LTE, right));
362    }
363
364    public Document walkGtEq(Operand lvalue, Operand rvalue) {
365        FieldInfo fieldInfo = walkReference(lvalue);
366        Object right = walkOperand(rvalue);
367        return newDocumentWithField(fieldInfo, new Document(QueryOperators.GTE, right));
368    }
369
370    public Document walkBetween(Operand lvalue, Operand rvalue, boolean positive) {
371        LiteralList l = (LiteralList) rvalue;
372        FieldInfo fieldInfo = walkReference(lvalue);
373        Object left = walkOperand(l.get(0));
374        Object right = walkOperand(l.get(1));
375        if (positive) {
376            Document range = new Document();
377            range.put(QueryOperators.GTE, left);
378            range.put(QueryOperators.LTE, right);
379            return newDocumentWithField(fieldInfo, range);
380        } else {
381            Document a = newDocumentWithField(fieldInfo, new Document(QueryOperators.LT, left));
382            Document b = newDocumentWithField(fieldInfo, new Document(QueryOperators.GT, right));
383            return new Document(QueryOperators.OR, Arrays.asList(a, b));
384        }
385    }
386
387    public Document walkIn(Operand lvalue, Operand rvalue, boolean positive) {
388        FieldInfo fieldInfo = walkReference(lvalue);
389        Object right = walkOperand(rvalue);
390        if (!(right instanceof List)) {
391            throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue);
392        }
393        // TODO check list fields
394        List<Object> list = (List<Object>) right;
395        return newDocumentWithField(fieldInfo, new Document(positive ? QueryOperators.IN : QueryOperators.NIN, list));
396    }
397
398    public Document walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) {
399        FieldInfo fieldInfo = walkReference(lvalue);
400        if (!(rvalue instanceof StringLiteral)) {
401            throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue);
402        }
403        // TODO check list fields
404        String like = walkStringLiteral((StringLiteral) rvalue);
405        String regex = ExpressionEvaluator.likeToRegex(like);
406        // MongoDB native matches are unanchored: optimize the regex for faster matches
407        if (regex.startsWith(".*")) {
408            regex = regex.substring(2);
409        } else if (likeAnchored) {
410            regex = "^" + regex;
411        }
412        if (regex.endsWith(".*")) {
413            regex = regex.substring(0, regex.length() - 2); // better range index use
414        } else if (likeAnchored) {
415            regex = regex + "$";
416        }
417
418        int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0;
419        Pattern pattern = Pattern.compile(regex, flags);
420        Object value;
421        if (positive) {
422            value = pattern;
423        } else {
424            value = new Document(QueryOperators.NOT, pattern);
425        }
426        return newDocumentWithField(fieldInfo, value);
427    }
428
429    public Object walkOperand(Operand op) {
430        if (op instanceof Literal) {
431            return walkLiteral((Literal) op);
432        } else if (op instanceof LiteralList) {
433            return walkLiteralList((LiteralList) op);
434        } else if (op instanceof Function) {
435            return walkFunction((Function) op);
436        } else if (op instanceof Expression) {
437            return walkExpression((Expression) op);
438        } else if (op instanceof Reference) {
439            return walkReference((Reference) op);
440        } else {
441            throw new QueryParseException("Unknown operand: " + op);
442        }
443    }
444
445    public Object walkLiteral(Literal lit) {
446        if (lit instanceof BooleanLiteral) {
447            return walkBooleanLiteral((BooleanLiteral) lit);
448        } else if (lit instanceof DateLiteral) {
449            return walkDateLiteral((DateLiteral) lit);
450        } else if (lit instanceof DoubleLiteral) {
451            return walkDoubleLiteral((DoubleLiteral) lit);
452        } else if (lit instanceof IntegerLiteral) {
453            return walkIntegerLiteral((IntegerLiteral) lit);
454        } else if (lit instanceof StringLiteral) {
455            return walkStringLiteral((StringLiteral) lit);
456        } else {
457            throw new QueryParseException("Unknown literal: " + lit);
458        }
459    }
460
461    public Object walkBooleanLiteral(BooleanLiteral lit) {
462        return Boolean.valueOf(lit.value);
463    }
464
465    public Date walkDateLiteral(DateLiteral lit) {
466        return lit.value.toDate(); // TODO onlyDate
467    }
468
469    public Double walkDoubleLiteral(DoubleLiteral lit) {
470        return Double.valueOf(lit.value);
471    }
472
473    public Long walkIntegerLiteral(IntegerLiteral lit) {
474        return Long.valueOf(lit.value);
475    }
476
477    public String walkStringLiteral(StringLiteral lit) {
478        return lit.value;
479    }
480
481    public List<Object> walkLiteralList(LiteralList litList) {
482        List<Object> list = new ArrayList<>(litList.size());
483        for (Literal lit : litList) {
484            list.add(walkLiteral(lit));
485        }
486        return list;
487    }
488
489    protected List<Object> walkOperandList(List<? extends Operand> values) {
490        List<Object> list = new LinkedList<>();
491        for (Operand value : values) {
492            list.add(walkOperand(value));
493        }
494        return list;
495    }
496
497    public Object walkFunction(Function func) {
498        throw new UnsupportedOperationException(func.name);
499    }
500
501    protected FieldInfo walkReference(Operand value) {
502        if (!(value instanceof Reference)) {
503            throw new QueryParseException("Invalid query, left hand side must be a property: " + value);
504        }
505        return walkReference((Reference) value);
506    }
507
508    public static class FieldInfo {
509
510        /** NXQL property. */
511        public final String prop;
512
513        /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */
514        public final String queryField;
515
516        /** MongoDB field for projection. */
517        public final String projectionField;
518
519        public final Type type;
520
521        /**
522         * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly.
523         */
524        public final boolean isTrueOrNullBoolean;
525
526        public FieldInfo(String prop, String queryField, String projectionField, Type type,
527                boolean isTrueOrNullBoolean) {
528            this.prop = prop;
529            this.queryField = queryField;
530            this.projectionField = projectionField;
531            this.type = type;
532            this.isTrueOrNullBoolean = isTrueOrNullBoolean;
533        }
534
535        public boolean isBoolean() {
536            return type instanceof BooleanType;
537        }
538    }
539
540    /**
541     * Returns the MongoDB field for this reference.
542     */
543    public FieldInfo walkReference(Reference ref) {
544        FieldInfo fieldInfo = walkReference(ref.name);
545        if (DATE_CAST.equals(ref.cast)) {
546            Type type = fieldInfo.type;
547            if (!(type instanceof DateType
548                    || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) {
549                throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name);
550            }
551            // fieldInfo.isDateCast = true;
552        }
553        return fieldInfo;
554    }
555
556    /**
557     * Walks a reference, and returns field info about it.
558     */
559    protected abstract FieldInfo walkReference(String name);
560
561}