001/*
002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.FACETED_TAG;
024import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.FACETED_TAG_LABEL;
025import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL_NAME;
027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SCORE;
029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.PROP_MAJOR_VERSION;
030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.PROP_MINOR_VERSION;
031import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.PROP_UID_MAJOR_VERSION;
032import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.PROP_UID_MINOR_VERSION;
033import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID;
034import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_META;
035import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_TEXT_SCORE;
036
037import java.util.ArrayList;
038import java.util.Arrays;
039import java.util.Collections;
040import java.util.Date;
041import java.util.HashMap;
042import java.util.HashSet;
043import java.util.LinkedList;
044import java.util.List;
045import java.util.Map;
046import java.util.Set;
047import java.util.concurrent.atomic.AtomicInteger;
048import java.util.regex.Pattern;
049
050import org.apache.commons.lang.StringUtils;
051import org.apache.commons.lang.math.NumberUtils;
052import org.bson.Document;
053import org.nuxeo.ecm.core.query.QueryParseException;
054import org.nuxeo.ecm.core.query.sql.NXQL;
055import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral;
056import org.nuxeo.ecm.core.query.sql.model.DateLiteral;
057import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral;
058import org.nuxeo.ecm.core.query.sql.model.Expression;
059import org.nuxeo.ecm.core.query.sql.model.Function;
060import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral;
061import org.nuxeo.ecm.core.query.sql.model.Literal;
062import org.nuxeo.ecm.core.query.sql.model.LiteralList;
063import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
064import org.nuxeo.ecm.core.query.sql.model.Operand;
065import org.nuxeo.ecm.core.query.sql.model.Operator;
066import org.nuxeo.ecm.core.query.sql.model.OrderByClause;
067import org.nuxeo.ecm.core.query.sql.model.OrderByExpr;
068import org.nuxeo.ecm.core.query.sql.model.Reference;
069import org.nuxeo.ecm.core.query.sql.model.SelectClause;
070import org.nuxeo.ecm.core.query.sql.model.StringLiteral;
071import org.nuxeo.ecm.core.schema.DocumentType;
072import org.nuxeo.ecm.core.schema.SchemaManager;
073import org.nuxeo.ecm.core.schema.types.ComplexType;
074import org.nuxeo.ecm.core.schema.types.Field;
075import org.nuxeo.ecm.core.schema.types.ListType;
076import org.nuxeo.ecm.core.schema.types.Schema;
077import org.nuxeo.ecm.core.schema.types.Type;
078import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
079import org.nuxeo.ecm.core.schema.types.primitives.DateType;
080import org.nuxeo.ecm.core.schema.types.primitives.StringType;
081import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
082import org.nuxeo.ecm.core.storage.ExpressionEvaluator.PathResolver;
083import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer;
084import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.FulltextQuery;
085import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.Op;
086import org.nuxeo.ecm.core.storage.QueryOptimizer.PrefixInfo;
087import org.nuxeo.ecm.core.storage.dbs.DBSDocument;
088import org.nuxeo.ecm.core.storage.dbs.DBSSession;
089import org.nuxeo.runtime.api.Framework;
090
091import com.mongodb.QueryOperators;
092
093/**
094 * Query builder for a MongoDB query from an {@link Expression}.
095 *
096 * @since 5.9.4
097 */
098public class MongoDBQueryBuilder {
099
100    public static final Long LONG_ZERO = Long.valueOf(0);
101
102    public static final Long LONG_ONE = Long.valueOf(1);
103
104    public static final Double ONE = Double.valueOf(1);
105
106    public static final Double MINUS_ONE = Double.valueOf(-1);
107
108    protected static final String DATE_CAST = "DATE";
109
110    protected final AtomicInteger counter = new AtomicInteger();
111
112    protected final SchemaManager schemaManager;
113
114    protected final MongoDBConverter converter;
115
116    protected final String idKey;
117
118    protected List<String> documentTypes;
119
120    protected final Expression expression;
121
122    protected final SelectClause selectClause;
123
124    protected final OrderByClause orderByClause;
125
126    protected final PathResolver pathResolver;
127
128    public boolean hasFulltext;
129
130    public boolean sortOnFulltextScore;
131
132    protected Document query;
133
134    protected Document orderBy;
135
136    protected Document projection;
137
138    protected Map<String, String> propertyKeys;
139
140    boolean projectionHasWildcard;
141
142    private boolean fulltextSearchDisabled;
143
144    /**
145     * Prefix to remove for $elemMatch (including final dot), or {@code null} if there's no current prefix to remove.
146     */
147    protected String elemMatchPrefix;
148
149    public MongoDBQueryBuilder(MongoDBRepository repository, Expression expression, SelectClause selectClause,
150            OrderByClause orderByClause, PathResolver pathResolver, boolean fulltextSearchDisabled) {
151        schemaManager = Framework.getService(SchemaManager.class);
152        converter = repository.converter;
153        idKey = repository.idKey;
154        this.expression = expression;
155        this.selectClause = selectClause;
156        this.orderByClause = orderByClause;
157        this.pathResolver = pathResolver;
158        this.fulltextSearchDisabled = fulltextSearchDisabled;
159        this.propertyKeys = new HashMap<>();
160    }
161
162    public void walk() {
163        query = walkExpression(expression); // computes hasFulltext
164        walkOrderBy(); // computes sortOnFulltextScore
165        walkProjection(); // needs hasFulltext and sortOnFulltextScore
166    }
167
168    public Document getQuery() {
169        return query;
170    }
171
172    public Document getOrderBy() {
173        return orderBy;
174    }
175
176    public Document getProjection() {
177        return projection;
178    }
179
180    public boolean hasProjectionWildcard() {
181        return projectionHasWildcard;
182    }
183
184    protected void walkOrderBy() {
185        sortOnFulltextScore = false;
186        if (orderByClause == null) {
187            orderBy = null;
188        } else {
189            orderBy = new Document();
190            for (OrderByExpr ob : orderByClause.elements) {
191                Reference ref = ob.reference;
192                boolean desc = ob.isDescending;
193                String field = walkReference(ref).queryField;
194                if (!orderBy.containsKey(field)) {
195                    Object value;
196                    if (KEY_FULLTEXT_SCORE.equals(field)) {
197                        if (!desc) {
198                            throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " ascending");
199                        }
200                        sortOnFulltextScore = true;
201                        value = new Document(MONGODB_META, MONGODB_TEXT_SCORE);
202                    } else {
203                        value = desc ? MINUS_ONE : ONE;
204                    }
205                    orderBy.put(field, value);
206                }
207            }
208            if (sortOnFulltextScore && orderBy.size() > 1) {
209                throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " and other criteria");
210            }
211        }
212    }
213
214    protected void walkProjection() {
215        projection = new Document();
216        boolean projectionOnFulltextScore = false;
217        for (Operand op : selectClause.getSelectList().values()) {
218            if (!(op instanceof Reference)) {
219                throw new QueryParseException("Projection not supported: " + op);
220            }
221            FieldInfo fieldInfo = walkReference((Reference) op);
222            String propertyField = fieldInfo.prop;
223            if (!propertyField.equals(NXQL.ECM_UUID) &&
224                !propertyField.equals(fieldInfo.projectionField) &&
225                !propertyField.contains("/")) {
226                propertyKeys.put(fieldInfo.projectionField, propertyField);
227            }
228            projection.put(fieldInfo.projectionField, ONE);
229            if (propertyField.contains("*")) {
230                projectionHasWildcard = true;
231            }
232            if (fieldInfo.projectionField.equals(KEY_FULLTEXT_SCORE)) {
233                projectionOnFulltextScore = true;
234            }
235        }
236        if (projectionOnFulltextScore || sortOnFulltextScore) {
237            if (!hasFulltext) {
238                throw new QueryParseException(NXQL.ECM_FULLTEXT_SCORE + " cannot be used without " + NXQL.ECM_FULLTEXT);
239            }
240            projection.put(KEY_FULLTEXT_SCORE, new Document(MONGODB_META, MONGODB_TEXT_SCORE));
241        }
242    }
243
244    public Document walkExpression(Expression expr) {
245        Operator op = expr.operator;
246        Operand lvalue = expr.lvalue;
247        Operand rvalue = expr.rvalue;
248        Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null;
249        String name = ref != null ? ref.name : null;
250        String cast = ref != null ? ref.cast : null;
251        if (DATE_CAST.equals(cast)) {
252            checkDateLiteralForCast(op, rvalue, name);
253        }
254        if (op == Operator.STARTSWITH) {
255            return walkStartsWith(lvalue, rvalue);
256        } else if (NXQL.ECM_PATH.equals(name)) {
257            return walkEcmPath(op, rvalue);
258        } else if (NXQL.ECM_ANCESTORID.equals(name)) {
259            return walkAncestorId(op, rvalue);
260        } else if (name != null && name.startsWith(NXQL.ECM_FULLTEXT) && !NXQL.ECM_FULLTEXT_JOBID.equals(name)) {
261            return walkEcmFulltext(name, op, rvalue);
262        } else if (op == Operator.SUM) {
263            throw new UnsupportedOperationException("SUM");
264        } else if (op == Operator.SUB) {
265            throw new UnsupportedOperationException("SUB");
266        } else if (op == Operator.MUL) {
267            throw new UnsupportedOperationException("MUL");
268        } else if (op == Operator.DIV) {
269            throw new UnsupportedOperationException("DIV");
270        } else if (op == Operator.LT) {
271            return walkLt(lvalue, rvalue);
272        } else if (op == Operator.GT) {
273            return walkGt(lvalue, rvalue);
274        } else if (op == Operator.EQ) {
275            return walkEq(lvalue, rvalue);
276        } else if (op == Operator.NOTEQ) {
277            return walkNotEq(lvalue, rvalue);
278        } else if (op == Operator.LTEQ) {
279            return walkLtEq(lvalue, rvalue);
280        } else if (op == Operator.GTEQ) {
281            return walkGtEq(lvalue, rvalue);
282        } else if (op == Operator.AND) {
283            if (expr instanceof MultiExpression) {
284                return walkAndMultiExpression((MultiExpression) expr);
285            } else {
286                return walkAnd(expr);
287            }
288        } else if (op == Operator.NOT) {
289            return walkNot(lvalue);
290        } else if (op == Operator.OR) {
291            return walkOr(lvalue, rvalue);
292        } else if (op == Operator.LIKE) {
293            return walkLike(lvalue, rvalue, true, false);
294        } else if (op == Operator.ILIKE) {
295            return walkLike(lvalue, rvalue, true, true);
296        } else if (op == Operator.NOTLIKE) {
297            return walkLike(lvalue, rvalue, false, false);
298        } else if (op == Operator.NOTILIKE) {
299            return walkLike(lvalue, rvalue, false, true);
300        } else if (op == Operator.IN) {
301            return walkIn(lvalue, rvalue, true);
302        } else if (op == Operator.NOTIN) {
303            return walkIn(lvalue, rvalue, false);
304        } else if (op == Operator.ISNULL) {
305            return walkIsNull(lvalue);
306        } else if (op == Operator.ISNOTNULL) {
307            return walkIsNotNull(lvalue);
308        } else if (op == Operator.BETWEEN) {
309            return walkBetween(lvalue, rvalue, true);
310        } else if (op == Operator.NOTBETWEEN) {
311            return walkBetween(lvalue, rvalue, false);
312        } else {
313            throw new QueryParseException("Unknown operator: " + op);
314        }
315    }
316
317    protected void checkDateLiteralForCast(Operator op, Operand value, String name) {
318        if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) {
319            LiteralList l = (LiteralList) value;
320            checkDateLiteralForCast(l.get(0), name);
321            checkDateLiteralForCast(l.get(1), name);
322        } else {
323            checkDateLiteralForCast(value, name);
324        }
325    }
326
327    protected void checkDateLiteralForCast(Operand value, String name) {
328        if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) {
329            throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name);
330        }
331    }
332
333    protected Document walkEcmPath(Operator op, Operand rvalue) {
334        if (op != Operator.EQ && op != Operator.NOTEQ) {
335            throw new QueryParseException(NXQL.ECM_PATH + " requires = or <> operator");
336        }
337        if (!(rvalue instanceof StringLiteral)) {
338            throw new QueryParseException(NXQL.ECM_PATH + " requires literal path as right argument");
339        }
340        String path = ((StringLiteral) rvalue).value;
341        if (path.length() > 1 && path.endsWith("/")) {
342            path = path.substring(0, path.length() - 1);
343        }
344        String id = pathResolver.getIdForPath(path);
345        if (id == null) {
346            // no such path
347            // TODO XXX do better
348            return new Document(MONGODB_ID, "__nosuchid__");
349        }
350        if (op == Operator.EQ) {
351            return new Document(idKey, id);
352        } else {
353            return new Document(idKey, new Document(QueryOperators.NE, id));
354        }
355    }
356
357    protected Document walkAncestorId(Operator op, Operand rvalue) {
358        if (op != Operator.EQ && op != Operator.NOTEQ) {
359            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires = or <> operator");
360        }
361        if (!(rvalue instanceof StringLiteral)) {
362            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires literal id as right argument");
363        }
364        String ancestorId = ((StringLiteral) rvalue).value;
365        if (op == Operator.EQ) {
366            return new Document(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
367        } else {
368            return new Document(DBSDocument.KEY_ANCESTOR_IDS, new Document(QueryOperators.NE, ancestorId));
369        }
370    }
371
372    protected Document walkEcmFulltext(String name, Operator op, Operand rvalue) {
373        if (op != Operator.EQ && op != Operator.LIKE) {
374            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires = or LIKE operator");
375        }
376        if (!(rvalue instanceof StringLiteral)) {
377            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires literal string as right argument");
378        }
379        if (fulltextSearchDisabled) {
380            throw new QueryParseException("Fulltext search disabled by configuration");
381        }
382        String fulltextQuery = ((StringLiteral) rvalue).value;
383        if (name.equals(NXQL.ECM_FULLTEXT)) {
384            // standard fulltext query
385            hasFulltext = true;
386            String ft = getMongoDBFulltextQuery(fulltextQuery);
387            if (ft == null) {
388                // empty query, matches nothing
389                return new Document(MONGODB_ID, "__nosuchid__");
390            }
391            Document textSearch = new Document();
392            textSearch.put(QueryOperators.SEARCH, ft);
393            // TODO language?
394            return new Document(QueryOperators.TEXT, textSearch);
395        } else {
396            // secondary index match with explicit field
397            // do a regexp on the field
398            if (name.charAt(NXQL.ECM_FULLTEXT.length()) != '.') {
399                throw new QueryParseException(name + " has incorrect syntax" + " for a secondary fulltext index");
400            }
401            String prop = name.substring(NXQL.ECM_FULLTEXT.length() + 1);
402            String ft = fulltextQuery.replace(" ", "%");
403            rvalue = new StringLiteral(ft);
404            return walkLike(new Reference(prop), rvalue, true, true);
405        }
406    }
407
408    // public static for tests
409    public static String getMongoDBFulltextQuery(String query) {
410        FulltextQuery ft = FulltextQueryAnalyzer.analyzeFulltextQuery(query);
411        if (ft == null) {
412            return null;
413        }
414        // translate into MongoDB syntax
415        return translateFulltext(ft, false);
416    }
417
418    /**
419     * Transforms the NXQL fulltext syntax into MongoDB syntax.
420     * <p>
421     * The MongoDB fulltext query syntax is badly documented, but is actually the following:
422     * <ul>
423     * <li>a term is a word,
424     * <li>a phrase is a set of spaced-separated words enclosed in double quotes,
425     * <li>negation is done by prepending a -,
426     * <li>the query is a space-separated set of terms, negated terms, phrases, or negated phrases.
427     * <li>all the words of non-negated phrases are also added to the terms.
428     * </ul>
429     * <p>
430     * The matching algorithm is (excluding stemming and stop words):
431     * <ul>
432     * <li>filter out documents with the negative terms, the negative phrases, or missing the phrases,
433     * <li>then if any term is present in the document then it's a match.
434     * </ul>
435     */
436    protected static String translateFulltext(FulltextQuery ft, boolean and) {
437        List<String> buf = new ArrayList<>();
438        translateFulltext(ft, buf, and);
439        return StringUtils.join(buf, ' ');
440    }
441
442    protected static void translateFulltext(FulltextQuery ft, List<String> buf, boolean and) {
443        if (ft.op == Op.OR) {
444            for (FulltextQuery term : ft.terms) {
445                // don't quote words for OR
446                translateFulltext(term, buf, false);
447            }
448        } else if (ft.op == Op.AND) {
449            for (FulltextQuery term : ft.terms) {
450                // quote words for AND
451                translateFulltext(term, buf, true);
452            }
453        } else {
454            String neg;
455            if (ft.op == Op.NOTWORD) {
456                neg = "-";
457            } else { // Op.WORD
458                neg = "";
459            }
460            String word = ft.word.toLowerCase();
461            if (ft.isPhrase() || and) {
462                buf.add(neg + '"' + word + '"');
463            } else {
464                buf.add(neg + word);
465            }
466        }
467    }
468
469    public Document walkNot(Operand value) {
470        Object val = walkOperand(value);
471        Object not = pushDownNot(val);
472        if (!(not instanceof Document)) {
473            throw new QueryParseException("Cannot do NOT on: " + val);
474        }
475        return (Document) not;
476    }
477
478    protected Object pushDownNot(Object object) {
479        if (!(object instanceof Document)) {
480            throw new QueryParseException("Cannot do NOT on: " + object);
481        }
482        Document ob = (Document) object;
483        Set<String> keySet = ob.keySet();
484        if (keySet.size() != 1) {
485            throw new QueryParseException("Cannot do NOT on: " + ob);
486        }
487        String key = keySet.iterator().next();
488        Object value = ob.get(key);
489        if (!key.startsWith("$")) {
490            if (value instanceof Document) {
491                // push down inside dbobject
492                return new Document(key, pushDownNot(value));
493            } else {
494                // k = v -> k != v
495                return new Document(key, new Document(QueryOperators.NE, value));
496            }
497        }
498        if (QueryOperators.NE.equals(key)) {
499            // NOT k != v -> k = v
500            return value;
501        }
502        if (QueryOperators.NOT.equals(key)) {
503            // NOT NOT v -> v
504            return value;
505        }
506        if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) {
507            // boolean algebra
508            // NOT (v1 AND v2) -> NOT v1 OR NOT v2
509            // NOT (v1 OR v2) -> NOT v1 AND NOT v2
510            String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND;
511            List<Object> list = (List<Object>) value;
512            for (int i = 0; i < list.size(); i++) {
513                list.set(i, pushDownNot(list.get(i)));
514            }
515            return new Document(op, list);
516        }
517        if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) {
518            // boolean algebra
519            // IN <-> NIN
520            String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN;
521            return new Document(op, value);
522        }
523        if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key)
524                || QueryOperators.GTE.equals(key)) {
525            // TODO use inverse operators?
526            return new Document(QueryOperators.NOT, ob);
527        }
528        throw new QueryParseException("Unknown operator for NOT: " + key);
529    }
530
531    protected Document newDocumentWithField(FieldInfo fieldInfo, Object value) {
532        return new Document(fieldInfo.queryField, value);
533    }
534
535    public Document walkIsNull(Operand value) {
536        FieldInfo fieldInfo = walkReference(value);
537        return newDocumentWithField(fieldInfo, null);
538    }
539
540    public Document walkIsNotNull(Operand value) {
541        FieldInfo fieldInfo = walkReference(value);
542        return newDocumentWithField(fieldInfo, new Document(QueryOperators.NE, null));
543    }
544
545    public Document walkAndMultiExpression(MultiExpression expr) {
546        return walkAnd(expr, expr.values);
547    }
548
549    public Document walkAnd(Expression expr) {
550        return walkAnd(expr, Arrays.asList(expr.lvalue, expr.rvalue));
551    }
552
553    protected static final Pattern SLASH_WILDCARD_SLASH = Pattern.compile("/\\*\\d+(/)?");
554
555    protected Document walkAnd(Expression expr, List<Operand> values) {
556        if (values.size() == 1) {
557            return (Document) walkOperand(values.get(0));
558        }
559        // PrefixInfo was computed by the QueryOptimizer
560        PrefixInfo info = (PrefixInfo) expr.getInfo();
561        if (info == null || info.count < 2) {
562            List<Object> list = walkOperandList(values);
563            return new Document(QueryOperators.AND, list);
564        }
565
566        // we have a common prefix for all underlying references, extract it into an $elemMatch node
567
568        // info.prefix is the DBS common prefix, ex: foo/bar/*1; ecm:acp/*1/acl/*1
569        // compute MongoDB prefix: foo.bar.; ecm:acp.acl.
570        String prefix = SLASH_WILDCARD_SLASH.matcher(info.prefix).replaceAll(".");
571        // remove current prefix and trailing . for actual field match
572        String fieldBase = stripElemMatchPrefix(prefix.substring(0, prefix.length() - 1));
573
574        String previousElemMatchPrefix = elemMatchPrefix;
575        elemMatchPrefix = prefix;
576        List<Object> list = walkOperandList(values);
577        elemMatchPrefix = previousElemMatchPrefix;
578
579        return new Document(fieldBase, new Document(QueryOperators.ELEM_MATCH, new Document(QueryOperators.AND, list)));
580    }
581
582    protected String stripElemMatchPrefix(String field) {
583        if (elemMatchPrefix != null && field.startsWith(elemMatchPrefix)) {
584            field = field.substring(elemMatchPrefix.length());
585        }
586        return field;
587    }
588
589    public Document walkOr(Operand lvalue, Operand rvalue) {
590        Object left = walkOperand(lvalue);
591        Object right = walkOperand(rvalue);
592        List<Object> list = new ArrayList<>(Arrays.asList(left, right));
593        return new Document(QueryOperators.OR, list);
594    }
595
596    protected Object checkBoolean(FieldInfo fieldInfo, Object right) {
597        if (fieldInfo.isBoolean()) {
598            // convert 0 / 1 to actual booleans
599            if (right instanceof Long) {
600                if (LONG_ZERO.equals(right)) {
601                    right = fieldInfo.isTrueOrNullBoolean ? null : FALSE;
602                } else if (LONG_ONE.equals(right)) {
603                    right = TRUE;
604                } else {
605                    throw new QueryParseException("Invalid boolean: " + right);
606                }
607            }
608        }
609        return right;
610    }
611
612    public Document walkEq(Operand lvalue, Operand rvalue) {
613        FieldInfo fieldInfo = walkReference(lvalue);
614        Object right = walkOperand(rvalue);
615        if (isMixinTypes(fieldInfo)) {
616            if (!(right instanceof String)) {
617                throw new QueryParseException("Invalid EQ rhs: " + rvalue);
618            }
619            return walkMixinTypes(Collections.singletonList((String) right), true);
620        }
621        right = checkBoolean(fieldInfo, right);
622        // TODO check list fields
623        return newDocumentWithField(fieldInfo, right);
624    }
625
626    public Document walkNotEq(Operand lvalue, Operand rvalue) {
627        FieldInfo fieldInfo = walkReference(lvalue);
628        Object right = walkOperand(rvalue);
629        if (isMixinTypes(fieldInfo)) {
630            if (!(right instanceof String)) {
631                throw new QueryParseException("Invalid NE rhs: " + rvalue);
632            }
633            return walkMixinTypes(Collections.singletonList((String) right), false);
634        }
635        right = checkBoolean(fieldInfo, right);
636        // TODO check list fields
637        return newDocumentWithField(fieldInfo, new Document(QueryOperators.NE, right));
638    }
639
640    public Document walkLt(Operand lvalue, Operand rvalue) {
641        FieldInfo fieldInfo = walkReference(lvalue);
642        Object right = walkOperand(rvalue);
643        return newDocumentWithField(fieldInfo, new Document(QueryOperators.LT, right));
644    }
645
646    public Document walkGt(Operand lvalue, Operand rvalue) {
647        FieldInfo fieldInfo = walkReference(lvalue);
648        Object right = walkOperand(rvalue);
649        return newDocumentWithField(fieldInfo, new Document(QueryOperators.GT, right));
650    }
651
652    public Document walkLtEq(Operand lvalue, Operand rvalue) {
653        FieldInfo fieldInfo = walkReference(lvalue);
654        Object right = walkOperand(rvalue);
655        return newDocumentWithField(fieldInfo, new Document(QueryOperators.LTE, right));
656    }
657
658    public Document walkGtEq(Operand lvalue, Operand rvalue) {
659        FieldInfo fieldInfo = walkReference(lvalue);
660        Object right = walkOperand(rvalue);
661        return newDocumentWithField(fieldInfo, new Document(QueryOperators.GTE, right));
662    }
663
664    public Document walkBetween(Operand lvalue, Operand rvalue, boolean positive) {
665        LiteralList l = (LiteralList) rvalue;
666        FieldInfo fieldInfo = walkReference(lvalue);
667        Object left = walkOperand(l.get(0));
668        Object right = walkOperand(l.get(1));
669        if (positive) {
670            Document range = new Document();
671            range.put(QueryOperators.GTE, left);
672            range.put(QueryOperators.LTE, right);
673            return newDocumentWithField(fieldInfo, range);
674        } else {
675            Document a = newDocumentWithField(fieldInfo, new Document(QueryOperators.LT, left));
676            Document b = newDocumentWithField(fieldInfo, new Document(QueryOperators.GT, right));
677            return new Document(QueryOperators.OR, Arrays.asList(a, b));
678        }
679    }
680
681    public Document walkIn(Operand lvalue, Operand rvalue, boolean positive) {
682        FieldInfo fieldInfo = walkReference(lvalue);
683        Object right = walkOperand(rvalue);
684        if (!(right instanceof List)) {
685            throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue);
686        }
687        if (isMixinTypes(fieldInfo)) {
688            return walkMixinTypes((List<String>) right, positive);
689        }
690        // TODO check list fields
691        List<Object> list = (List<Object>) right;
692        return newDocumentWithField(fieldInfo, new Document(positive ? QueryOperators.IN : QueryOperators.NIN, list));
693    }
694
695    public Document walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) {
696        FieldInfo fieldInfo = walkReference(lvalue);
697        if (!(rvalue instanceof StringLiteral)) {
698            throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue);
699        }
700        // TODO check list fields
701        String like = walkStringLiteral((StringLiteral) rvalue);
702        String regex = ExpressionEvaluator.likeToRegex(like);
703
704        int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0;
705        Pattern pattern = Pattern.compile(regex, flags);
706        Object value;
707        if (positive) {
708            value = pattern;
709        } else {
710            value = new Document(QueryOperators.NOT, pattern);
711        }
712        return newDocumentWithField(fieldInfo, value);
713    }
714
715    public Object walkOperand(Operand op) {
716        if (op instanceof Literal) {
717            return walkLiteral((Literal) op);
718        } else if (op instanceof LiteralList) {
719            return walkLiteralList((LiteralList) op);
720        } else if (op instanceof Function) {
721            return walkFunction((Function) op);
722        } else if (op instanceof Expression) {
723            return walkExpression((Expression) op);
724        } else if (op instanceof Reference) {
725            return walkReference((Reference) op);
726        } else {
727            throw new QueryParseException("Unknown operand: " + op);
728        }
729    }
730
731    public Object walkLiteral(Literal lit) {
732        if (lit instanceof BooleanLiteral) {
733            return walkBooleanLiteral((BooleanLiteral) lit);
734        } else if (lit instanceof DateLiteral) {
735            return walkDateLiteral((DateLiteral) lit);
736        } else if (lit instanceof DoubleLiteral) {
737            return walkDoubleLiteral((DoubleLiteral) lit);
738        } else if (lit instanceof IntegerLiteral) {
739            return walkIntegerLiteral((IntegerLiteral) lit);
740        } else if (lit instanceof StringLiteral) {
741            return walkStringLiteral((StringLiteral) lit);
742        } else {
743            throw new QueryParseException("Unknown literal: " + lit);
744        }
745    }
746
747    public Object walkBooleanLiteral(BooleanLiteral lit) {
748        return Boolean.valueOf(lit.value);
749    }
750
751    public Date walkDateLiteral(DateLiteral lit) {
752        return lit.value.toDate(); // TODO onlyDate
753    }
754
755    public Double walkDoubleLiteral(DoubleLiteral lit) {
756        return Double.valueOf(lit.value);
757    }
758
759    public Long walkIntegerLiteral(IntegerLiteral lit) {
760        return Long.valueOf(lit.value);
761    }
762
763    public String walkStringLiteral(StringLiteral lit) {
764        return lit.value;
765    }
766
767    public List<Object> walkLiteralList(LiteralList litList) {
768        List<Object> list = new ArrayList<>(litList.size());
769        for (Literal lit : litList) {
770            list.add(walkLiteral(lit));
771        }
772        return list;
773    }
774
775    protected List<Object> walkOperandList(List<Operand> values) {
776        List<Object> list = new LinkedList<>();
777        for (Operand value : values) {
778            list.add(walkOperand(value));
779        }
780        return list;
781    }
782
783    public Object walkFunction(Function func) {
784        throw new UnsupportedOperationException(func.name);
785    }
786
787    public Document walkStartsWith(Operand lvalue, Operand rvalue) {
788        if (!(lvalue instanceof Reference)) {
789            throw new QueryParseException("Invalid STARTSWITH query, left hand side must be a property: " + lvalue);
790        }
791        String name = ((Reference) lvalue).name;
792        if (!(rvalue instanceof StringLiteral)) {
793            throw new QueryParseException(
794                    "Invalid STARTSWITH query, right hand side must be a literal path: " + rvalue);
795        }
796        String path = ((StringLiteral) rvalue).value;
797        if (path.length() > 1 && path.endsWith("/")) {
798            path = path.substring(0, path.length() - 1);
799        }
800
801        if (NXQL.ECM_PATH.equals(name)) {
802            return walkStartsWithPath(path);
803        } else {
804            return walkStartsWithNonPath(lvalue, path);
805        }
806    }
807
808    protected Document walkStartsWithPath(String path) {
809        // resolve path
810        String ancestorId = pathResolver.getIdForPath(path);
811        if (ancestorId == null) {
812            // no such path
813            // TODO XXX do better
814            return new Document(MONGODB_ID, "__nosuchid__");
815        }
816        return new Document(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
817    }
818
819    protected Document walkStartsWithNonPath(Operand lvalue, String path) {
820        FieldInfo fieldInfo = walkReference(lvalue);
821        Document eq = newDocumentWithField(fieldInfo, path);
822        // escape except alphanumeric and others not needing escaping
823        String regex = path.replaceAll("([^a-zA-Z0-9 /])", "\\\\$1");
824        Pattern pattern = Pattern.compile(regex + "/.*");
825        Document like = newDocumentWithField(fieldInfo, pattern);
826        return new Document(QueryOperators.OR, Arrays.asList(eq, like));
827    }
828
829    protected FieldInfo walkReference(Operand value) {
830        if (!(value instanceof Reference)) {
831            throw new QueryParseException("Invalid query, left hand side must be a property: " + value);
832        }
833        return walkReference((Reference) value);
834    }
835
836    // non-canonical index syntax, for replaceAll
837    protected final static Pattern NON_CANON_INDEX = Pattern.compile("[^/\\[\\]]+" // name
838            + "\\[(\\d+|\\*|\\*\\d+)\\]" // index in brackets
839    );
840
841    /**
842     * Canonicalizes a Nuxeo-xpath.
843     * <p>
844     * Replaces {@code a/foo[123]/b} with {@code a/123/b}
845     * <p>
846     * A star or a star followed by digits can be used instead of just the digits as well.
847     *
848     * @param xpath the xpath
849     * @return the canonicalized xpath.
850     */
851    public static String canonicalXPath(String xpath) {
852        while (xpath.length() > 0 && xpath.charAt(0) == '/') {
853            xpath = xpath.substring(1);
854        }
855        if (xpath.indexOf('[') == -1) {
856            return xpath;
857        } else {
858            return NON_CANON_INDEX.matcher(xpath).replaceAll("$1");
859        }
860    }
861
862    protected static class FieldInfo {
863
864        /** NXQL property. */
865        protected final String prop;
866
867        /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */
868        protected final String queryField;
869
870        /** MongoDB field for projection. */
871        protected final String projectionField;
872
873        protected final Type type;
874
875        /**
876         * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly.
877         */
878        protected final boolean isTrueOrNullBoolean;
879
880        protected FieldInfo(String prop, String queryField, String projectionField, Type type,
881                boolean isTrueOrNullBoolean) {
882            this.prop = prop;
883            this.queryField = queryField;
884            this.projectionField = projectionField;
885            this.type = type;
886            this.isTrueOrNullBoolean = isTrueOrNullBoolean;
887        }
888
889        protected boolean isBoolean() {
890            return type instanceof BooleanType;
891        }
892    }
893
894    /**
895     * Returns the MongoDB field for this reference.
896     */
897    public FieldInfo walkReference(Reference ref) {
898        FieldInfo fieldInfo = walkReference(ref.name);
899        if (DATE_CAST.equals(ref.cast)) {
900            Type type = fieldInfo.type;
901            if (!(type instanceof DateType
902                    || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) {
903                throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name);
904            }
905            // fieldInfo.isDateCast = true;
906        }
907        return fieldInfo;
908    }
909
910    protected FieldInfo walkReference(String name) {
911        String prop = canonicalXPath(name);
912        String[] parts = prop.split("/");
913        if (prop.startsWith(NXQL.ECM_PREFIX)) {
914            if (prop.startsWith(NXQL.ECM_ACL + "/")) {
915                return parseACP(prop, parts);
916            }
917            if (prop.startsWith(NXQL.ECM_TAG)) {
918                String queryField = FACETED_TAG + "." + FACETED_TAG_LABEL;
919                queryField = stripElemMatchPrefix(queryField);
920                return new FieldInfo(prop, queryField, queryField, StringType.INSTANCE, true);
921            }
922            // simple field
923            String field = DBSSession.convToInternal(prop);
924            Type type = DBSSession.getType(field);
925            String queryField = converter.keyToBson(field);
926            queryField = stripElemMatchPrefix(queryField);
927            return new FieldInfo(prop, queryField, field, type, true);
928        } else {
929            String first = parts[0];
930            Field field = schemaManager.getField(first);
931            if (field == null) {
932                if (first.indexOf(':') > -1) {
933                    throw new QueryParseException("No such property: " + name);
934                }
935                // check without prefix
936                // TODO precompute this in SchemaManagerImpl
937                for (Schema schema : schemaManager.getSchemas()) {
938                    if (!StringUtils.isBlank(schema.getNamespace().prefix)) {
939                        // schema with prefix, do not consider as candidate
940                        continue;
941                    }
942                    if (schema != null) {
943                        field = schema.getField(first);
944                        if (field != null) {
945                            break;
946                        }
947                    }
948                }
949                if (field == null) {
950                    throw new QueryParseException("No such property: " + name);
951                }
952            }
953            Type type = field.getType();
954            if (PROP_UID_MAJOR_VERSION.equals(prop) || PROP_UID_MINOR_VERSION.equals(prop)
955                 || PROP_MAJOR_VERSION.equals(prop) || PROP_MINOR_VERSION.equals(prop)) {
956                String fieldName = DBSSession.convToInternal(prop);
957                return new FieldInfo(prop, fieldName, fieldName, type, true);
958            }
959
960            // canonical name
961            parts[0] = field.getName().getPrefixedName();
962            // are there wildcards or list indexes?
963            List<String> queryFieldParts = new LinkedList<>(); // field for query
964            List<String> projectionFieldParts = new LinkedList<>(); // field for projection
965            boolean firstPart = true;
966            for (String part : parts) {
967                if (NumberUtils.isDigits(part)) {
968                    // explicit list index
969                    queryFieldParts.add(part);
970                    type = ((ListType) type).getFieldType();
971                } else if (!part.startsWith("*")) {
972                    // complex sub-property
973                    queryFieldParts.add(part);
974                    projectionFieldParts.add(part);
975                    if (!firstPart) {
976                        // we already computed the type of the first part
977                        field = ((ComplexType) type).getField(part);
978                        if (field == null) {
979                            throw new QueryParseException("No such property: " + name);
980                        }
981                        type = field.getType();
982                    }
983                } else {
984                    // wildcard
985                    type = ((ListType) type).getFieldType();
986                }
987                firstPart = false;
988            }
989            String queryField = StringUtils.join(queryFieldParts, '.');
990            String projectionField = StringUtils.join(projectionFieldParts, '.');
991            queryField = stripElemMatchPrefix(queryField);
992            return new FieldInfo(prop, queryField, projectionField, type, false);
993        }
994    }
995
996    protected FieldInfo parseACP(String prop, String[] parts) {
997        if (parts.length != 3) {
998            throw new QueryParseException("No such property: " + prop);
999        }
1000        String wildcard = parts[1];
1001        if (NumberUtils.isDigits(wildcard)) {
1002            throw new QueryParseException("Cannot use explicit index in ACLs: " + prop);
1003        }
1004        String last = parts[2];
1005        String queryField;
1006        if (NXQL.ECM_ACL_NAME.equals(last)) {
1007            queryField = KEY_ACP + "." + KEY_ACL_NAME;
1008        } else {
1009            String fieldLast = DBSSession.convToInternalAce(last);
1010            if (fieldLast == null) {
1011                throw new QueryParseException("No such property: " + prop);
1012            }
1013            queryField = KEY_ACP + "." + KEY_ACL + "." + fieldLast;
1014        }
1015        Type type = DBSSession.getType(last);
1016        queryField = stripElemMatchPrefix(queryField);
1017        return new FieldInfo(prop, queryField, queryField, type, false);
1018    }
1019
1020    protected boolean isMixinTypes(FieldInfo fieldInfo) {
1021        return fieldInfo.queryField.equals(DBSDocument.KEY_MIXIN_TYPES);
1022    }
1023
1024    protected Set<String> getMixinDocumentTypes(String mixin) {
1025        Set<String> types = schemaManager.getDocumentTypeNamesForFacet(mixin);
1026        return types == null ? Collections.emptySet() : types;
1027    }
1028
1029    protected List<String> getDocumentTypes() {
1030        // TODO precompute in SchemaManager
1031        if (documentTypes == null) {
1032            documentTypes = new ArrayList<>();
1033            for (DocumentType docType : schemaManager.getDocumentTypes()) {
1034                documentTypes.add(docType.getName());
1035            }
1036        }
1037        return documentTypes;
1038    }
1039
1040    protected boolean isNeverPerInstanceMixin(String mixin) {
1041        return schemaManager.getNoPerDocumentQueryFacets().contains(mixin);
1042    }
1043
1044    /**
1045     * Matches the mixin types against a list of values.
1046     * <p>
1047     * Used for:
1048     * <ul>
1049     * <li>ecm:mixinTypes = 'Foo'
1050     * <li>ecm:mixinTypes != 'Foo'
1051     * <li>ecm:mixinTypes IN ('Foo', 'Bar')
1052     * <li>ecm:mixinTypes NOT IN ('Foo', 'Bar')
1053     * </ul>
1054     * <p>
1055     * ecm:mixinTypes IN ('Foo', 'Bar')
1056     *
1057     * <pre>
1058     * { "$or" : [ { "ecm:primaryType" : { "$in" : [ ... types with Foo or Bar ...]}} ,
1059     *             { "ecm:mixinTypes" : { "$in" : [ "Foo" , "Bar]}}]}
1060     * </pre>
1061     *
1062     * ecm:mixinTypes NOT IN ('Foo', 'Bar')
1063     * <p>
1064     *
1065     * <pre>
1066     * { "$and" : [ { "ecm:primaryType" : { "$in" : [ ... types without Foo nor Bar ...]}} ,
1067     *              { "ecm:mixinTypes" : { "$nin" : [ "Foo" , "Bar]}}]}
1068     * </pre>
1069     */
1070    public Document walkMixinTypes(List<String> mixins, boolean include) {
1071        /*
1072         * Primary types that match.
1073         */
1074        Set<String> matchPrimaryTypes;
1075        if (include) {
1076            matchPrimaryTypes = new HashSet<>();
1077            for (String mixin : mixins) {
1078                matchPrimaryTypes.addAll(getMixinDocumentTypes(mixin));
1079            }
1080        } else {
1081            matchPrimaryTypes = new HashSet<>(getDocumentTypes());
1082            for (String mixin : mixins) {
1083                matchPrimaryTypes.removeAll(getMixinDocumentTypes(mixin));
1084            }
1085        }
1086        /*
1087         * Instance mixins that match.
1088         */
1089        Set<String> matchMixinTypes = new HashSet<>();
1090        for (String mixin : mixins) {
1091            if (!isNeverPerInstanceMixin(mixin)) {
1092                matchMixinTypes.add(mixin);
1093            }
1094        }
1095        /*
1096         * MongoDB query generation.
1097         */
1098        // match on primary type
1099        Document p = new Document(DBSDocument.KEY_PRIMARY_TYPE,
1100                new Document(QueryOperators.IN, matchPrimaryTypes));
1101        // match on mixin types
1102        // $in/$nin with an array matches if any/no element of the array matches
1103        String innin = include ? QueryOperators.IN : QueryOperators.NIN;
1104        Document m = new Document(DBSDocument.KEY_MIXIN_TYPES, new Document(innin, matchMixinTypes));
1105        // and/or between those
1106        String op = include ? QueryOperators.OR : QueryOperators.AND;
1107        return new Document(op, Arrays.asList(p, m));
1108    }
1109
1110}