001/*
002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
024import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL_NAME;
025import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SCORE;
027import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID;
028import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_META;
029import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_TEXT_SCORE;
030
031import java.util.ArrayList;
032import java.util.Arrays;
033import java.util.Collections;
034import java.util.Date;
035import java.util.HashMap;
036import java.util.HashSet;
037import java.util.Iterator;
038import java.util.LinkedHashMap;
039import java.util.LinkedList;
040import java.util.List;
041import java.util.Map;
042import java.util.Map.Entry;
043import java.util.Set;
044import java.util.concurrent.atomic.AtomicInteger;
045import java.util.regex.Matcher;
046import java.util.regex.Pattern;
047
048import org.apache.commons.lang.StringUtils;
049import org.apache.commons.lang.math.NumberUtils;
050import org.nuxeo.ecm.core.query.QueryParseException;
051import org.nuxeo.ecm.core.query.sql.NXQL;
052import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral;
053import org.nuxeo.ecm.core.query.sql.model.DateLiteral;
054import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral;
055import org.nuxeo.ecm.core.query.sql.model.Expression;
056import org.nuxeo.ecm.core.query.sql.model.Function;
057import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral;
058import org.nuxeo.ecm.core.query.sql.model.Literal;
059import org.nuxeo.ecm.core.query.sql.model.LiteralList;
060import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
061import org.nuxeo.ecm.core.query.sql.model.Operand;
062import org.nuxeo.ecm.core.query.sql.model.Operator;
063import org.nuxeo.ecm.core.query.sql.model.OrderByClause;
064import org.nuxeo.ecm.core.query.sql.model.OrderByExpr;
065import org.nuxeo.ecm.core.query.sql.model.Reference;
066import org.nuxeo.ecm.core.query.sql.model.SelectClause;
067import org.nuxeo.ecm.core.query.sql.model.StringLiteral;
068import org.nuxeo.ecm.core.schema.DocumentType;
069import org.nuxeo.ecm.core.schema.SchemaManager;
070import org.nuxeo.ecm.core.schema.types.ComplexType;
071import org.nuxeo.ecm.core.schema.types.Field;
072import org.nuxeo.ecm.core.schema.types.ListType;
073import org.nuxeo.ecm.core.schema.types.Schema;
074import org.nuxeo.ecm.core.schema.types.Type;
075import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
076import org.nuxeo.ecm.core.schema.types.primitives.DateType;
077import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
078import org.nuxeo.ecm.core.storage.ExpressionEvaluator.PathResolver;
079import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer;
080import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.FulltextQuery;
081import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.Op;
082import org.nuxeo.ecm.core.storage.dbs.DBSDocument;
083import org.nuxeo.ecm.core.storage.dbs.DBSSession;
084import org.nuxeo.runtime.api.Framework;
085
086import com.mongodb.BasicDBObject;
087import com.mongodb.DBObject;
088import com.mongodb.QueryOperators;
089
090/**
091 * Query builder for a MongoDB query from an {@link Expression}.
092 *
093 * @since 5.9.4
094 */
095public class MongoDBQueryBuilder {
096
097    private static final Long ZERO = Long.valueOf(0);
098
099    private static final Long ONE = Long.valueOf(1);
100
101    private static final Long MINUS_ONE = Long.valueOf(-1);
102
103    protected static final String DATE_CAST = "DATE";
104
105    protected final AtomicInteger counter = new AtomicInteger();
106
107    protected final SchemaManager schemaManager;
108
109    protected final MongoDBConverter converter;
110
111    protected final String idKey;
112
113    protected List<String> documentTypes;
114
115    protected final Expression expression;
116
117    protected final SelectClause selectClause;
118
119    protected final OrderByClause orderByClause;
120
121    protected final PathResolver pathResolver;
122
123    public boolean hasFulltext;
124
125    public boolean sortOnFulltextScore;
126
127    protected DBObject query;
128
129    protected DBObject orderBy;
130
131    protected DBObject projection;
132
133    boolean projectionHasWildcard;
134
135    private boolean fulltextSearchDisabled;
136
137    public MongoDBQueryBuilder(MongoDBRepository repository, Expression expression, SelectClause selectClause,
138            OrderByClause orderByClause, PathResolver pathResolver, boolean fulltextSearchDisabled) {
139        schemaManager = Framework.getLocalService(SchemaManager.class);
140        converter = repository.converter;
141        idKey = repository.idKey;
142        this.expression = expression;
143        this.selectClause = selectClause;
144        this.orderByClause = orderByClause;
145        this.pathResolver = pathResolver;
146        this.fulltextSearchDisabled = fulltextSearchDisabled;
147    }
148
149    public void walk() {
150        query = walkExpression(expression); // computes hasFulltext
151        walkOrderBy(); // computes sortOnFulltextScore
152        walkProjection(); // needs hasFulltext and sortOnFulltextScore
153    }
154
155    public DBObject getQuery() {
156        return query;
157    }
158
159    public DBObject getOrderBy() {
160        return orderBy;
161    }
162
163    public DBObject getProjection() {
164        return projection;
165    }
166
167    public boolean hasProjectionWildcard() {
168        return projectionHasWildcard;
169    }
170
171    protected void walkOrderBy() {
172        sortOnFulltextScore = false;
173        if (orderByClause == null) {
174            orderBy = null;
175        } else {
176            orderBy = new BasicDBObject();
177            for (OrderByExpr ob : orderByClause.elements) {
178                Reference ref = ob.reference;
179                boolean desc = ob.isDescending;
180                String field = walkReference(ref).queryField;
181                if (!orderBy.containsField(field)) {
182                    Object value;
183                    if (KEY_FULLTEXT_SCORE.equals(field)) {
184                        if (!desc) {
185                            throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " ascending");
186                        }
187                        sortOnFulltextScore = true;
188                        value = new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE);
189                    } else {
190                        value = desc ? MINUS_ONE : ONE;
191                    }
192                    orderBy.put(field, value);
193                }
194            }
195            if (sortOnFulltextScore && ((BasicDBObject) orderBy).size() > 1) {
196                throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " and other criteria");
197            }
198        }
199    }
200
201    protected void walkProjection() {
202        projection = new BasicDBObject();
203        boolean projectionOnFulltextScore = false;
204        for (Operand op : selectClause.getSelectList().values()) {
205            if (!(op instanceof Reference)) {
206                throw new QueryParseException("Projection not supported: " + op);
207            }
208            FieldInfo fieldInfo = walkReference((Reference) op);
209            projection.put(fieldInfo.projectionField, ONE);
210            if (fieldInfo.hasWildcard) {
211                projectionHasWildcard = true;
212            }
213            if (fieldInfo.projectionField.equals(KEY_FULLTEXT_SCORE)) {
214                projectionOnFulltextScore = true;
215            }
216        }
217        if (projectionOnFulltextScore || sortOnFulltextScore) {
218            if (!hasFulltext) {
219                throw new QueryParseException(NXQL.ECM_FULLTEXT_SCORE + " cannot be used without " + NXQL.ECM_FULLTEXT);
220            }
221            projection.put(KEY_FULLTEXT_SCORE, new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE));
222        }
223    }
224
225    public DBObject walkExpression(Expression expr) {
226        Operator op = expr.operator;
227        Operand lvalue = expr.lvalue;
228        Operand rvalue = expr.rvalue;
229        Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null;
230        String name = ref != null ? ref.name : null;
231        String cast = ref != null ? ref.cast : null;
232        if (DATE_CAST.equals(cast)) {
233            checkDateLiteralForCast(op, rvalue, name);
234        }
235        if (op == Operator.STARTSWITH) {
236            return walkStartsWith(lvalue, rvalue);
237        } else if (NXQL.ECM_PATH.equals(name)) {
238            return walkEcmPath(op, rvalue);
239        } else if (NXQL.ECM_ANCESTORID.equals(name)) {
240            return walkAncestorId(op, rvalue);
241        } else if (name != null && name.startsWith(NXQL.ECM_FULLTEXT) && !NXQL.ECM_FULLTEXT_JOBID.equals(name)) {
242            return walkEcmFulltext(name, op, rvalue);
243        } else if (op == Operator.SUM) {
244            throw new UnsupportedOperationException("SUM");
245        } else if (op == Operator.SUB) {
246            throw new UnsupportedOperationException("SUB");
247        } else if (op == Operator.MUL) {
248            throw new UnsupportedOperationException("MUL");
249        } else if (op == Operator.DIV) {
250            throw new UnsupportedOperationException("DIV");
251        } else if (op == Operator.LT) {
252            return walkLt(lvalue, rvalue);
253        } else if (op == Operator.GT) {
254            return walkGt(lvalue, rvalue);
255        } else if (op == Operator.EQ) {
256            return walkEq(lvalue, rvalue);
257        } else if (op == Operator.NOTEQ) {
258            return walkNotEq(lvalue, rvalue);
259        } else if (op == Operator.LTEQ) {
260            return walkLtEq(lvalue, rvalue);
261        } else if (op == Operator.GTEQ) {
262            return walkGtEq(lvalue, rvalue);
263        } else if (op == Operator.AND) {
264            if (expr instanceof MultiExpression) {
265                return walkMultiExpression((MultiExpression) expr);
266            } else {
267                return walkAnd(lvalue, rvalue);
268            }
269        } else if (op == Operator.NOT) {
270            return walkNot(lvalue);
271        } else if (op == Operator.OR) {
272            return walkOr(lvalue, rvalue);
273        } else if (op == Operator.LIKE) {
274            return walkLike(lvalue, rvalue, true, false);
275        } else if (op == Operator.ILIKE) {
276            return walkLike(lvalue, rvalue, true, true);
277        } else if (op == Operator.NOTLIKE) {
278            return walkLike(lvalue, rvalue, false, false);
279        } else if (op == Operator.NOTILIKE) {
280            return walkLike(lvalue, rvalue, false, true);
281        } else if (op == Operator.IN) {
282            return walkIn(lvalue, rvalue, true);
283        } else if (op == Operator.NOTIN) {
284            return walkIn(lvalue, rvalue, false);
285        } else if (op == Operator.ISNULL) {
286            return walkIsNull(lvalue);
287        } else if (op == Operator.ISNOTNULL) {
288            return walkIsNotNull(lvalue);
289        } else if (op == Operator.BETWEEN) {
290            return walkBetween(lvalue, rvalue, true);
291        } else if (op == Operator.NOTBETWEEN) {
292            return walkBetween(lvalue, rvalue, false);
293        } else {
294            throw new QueryParseException("Unknown operator: " + op);
295        }
296    }
297
298    protected void checkDateLiteralForCast(Operator op, Operand value, String name) {
299        if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) {
300            LiteralList l = (LiteralList) value;
301            checkDateLiteralForCast(l.get(0), name);
302            checkDateLiteralForCast(l.get(1), name);
303        } else {
304            checkDateLiteralForCast(value, name);
305        }
306    }
307
308    protected void checkDateLiteralForCast(Operand value, String name) {
309        if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) {
310            throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name);
311        }
312    }
313
314    protected DBObject walkEcmPath(Operator op, Operand rvalue) {
315        if (op != Operator.EQ && op != Operator.NOTEQ) {
316            throw new QueryParseException(NXQL.ECM_PATH + " requires = or <> operator");
317        }
318        if (!(rvalue instanceof StringLiteral)) {
319            throw new QueryParseException(NXQL.ECM_PATH + " requires literal path as right argument");
320        }
321        String path = ((StringLiteral) rvalue).value;
322        if (path.length() > 1 && path.endsWith("/")) {
323            path = path.substring(0, path.length() - 1);
324        }
325        String id = pathResolver.getIdForPath(path);
326        if (id == null) {
327            // no such path
328            // TODO XXX do better
329            return new BasicDBObject(MONGODB_ID, "__nosuchid__");
330        }
331        if (op == Operator.EQ) {
332            return new BasicDBObject(idKey, id);
333        } else {
334            return new BasicDBObject(idKey, new BasicDBObject(QueryOperators.NE, id));
335        }
336    }
337
338    protected DBObject walkAncestorId(Operator op, Operand rvalue) {
339        if (op != Operator.EQ && op != Operator.NOTEQ) {
340            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires = or <> operator");
341        }
342        if (!(rvalue instanceof StringLiteral)) {
343            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires literal id as right argument");
344        }
345        String ancestorId = ((StringLiteral) rvalue).value;
346        if (op == Operator.EQ) {
347            return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
348        } else {
349            return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, new BasicDBObject(QueryOperators.NE, ancestorId));
350        }
351    }
352
353    protected DBObject walkEcmFulltext(String name, Operator op, Operand rvalue) {
354        if (op != Operator.EQ && op != Operator.LIKE) {
355            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires = or LIKE operator");
356        }
357        if (!(rvalue instanceof StringLiteral)) {
358            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires literal string as right argument");
359        }
360        if (fulltextSearchDisabled) {
361            throw new QueryParseException("Fulltext search disabled by configuration");
362        }
363        String fulltextQuery = ((StringLiteral) rvalue).value;
364        if (name.equals(NXQL.ECM_FULLTEXT)) {
365            // standard fulltext query
366            hasFulltext = true;
367            String ft = getMongoDBFulltextQuery(fulltextQuery);
368            if (ft == null) {
369                // empty query, matches nothing
370                return new BasicDBObject(MONGODB_ID, "__nosuchid__");
371            }
372            DBObject textSearch = new BasicDBObject();
373            textSearch.put(QueryOperators.SEARCH, ft);
374            // TODO language?
375            return new BasicDBObject(QueryOperators.TEXT, textSearch);
376        } else {
377            // secondary index match with explicit field
378            // do a regexp on the field
379            if (name.charAt(NXQL.ECM_FULLTEXT.length()) != '.') {
380                throw new QueryParseException(name + " has incorrect syntax" + " for a secondary fulltext index");
381            }
382            String prop = name.substring(NXQL.ECM_FULLTEXT.length() + 1);
383            String ft = fulltextQuery.replace(" ", "%");
384            rvalue = new StringLiteral(ft);
385            return walkLike(new Reference(prop), rvalue, true, true);
386        }
387    }
388
389    // public static for tests
390    public static String getMongoDBFulltextQuery(String query) {
391        FulltextQuery ft = FulltextQueryAnalyzer.analyzeFulltextQuery(query);
392        if (ft == null) {
393            return null;
394        }
395        // translate into MongoDB syntax
396        return translateFulltext(ft, false);
397    }
398
399    /**
400     * Transforms the NXQL fulltext syntax into MongoDB syntax.
401     * <p>
402     * The MongoDB fulltext query syntax is badly documented, but is actually the following:
403     * <ul>
404     * <li>a term is a word,
405     * <li>a phrase is a set of spaced-separated words enclosed in double quotes,
406     * <li>negation is done by prepending a -,
407     * <li>the query is a space-separated set of terms, negated terms, phrases, or negated phrases.
408     * <li>all the words of non-negated phrases are also added to the terms.
409     * </ul>
410     * <p>
411     * The matching algorithm is (excluding stemming and stop words):
412     * <ul>
413     * <li>filter out documents with the negative terms, the negative phrases, or missing the phrases,
414     * <li>then if any term is present in the document then it's a match.
415     * </ul>
416     */
417    protected static String translateFulltext(FulltextQuery ft, boolean and) {
418        List<String> buf = new ArrayList<>();
419        translateFulltext(ft, buf, and);
420        return StringUtils.join(buf, ' ');
421    }
422
423    protected static void translateFulltext(FulltextQuery ft, List<String> buf, boolean and) {
424        if (ft.op == Op.OR) {
425            for (FulltextQuery term : ft.terms) {
426                // don't quote words for OR
427                translateFulltext(term, buf, false);
428            }
429        } else if (ft.op == Op.AND) {
430            for (FulltextQuery term : ft.terms) {
431                // quote words for AND
432                translateFulltext(term, buf, true);
433            }
434        } else {
435            String neg;
436            if (ft.op == Op.NOTWORD) {
437                neg = "-";
438            } else { // Op.WORD
439                neg = "";
440            }
441            String word = ft.word.toLowerCase();
442            if (ft.isPhrase() || and) {
443                buf.add(neg + '"' + word + '"');
444            } else {
445                buf.add(neg + word);
446            }
447        }
448    }
449
450    public DBObject walkNot(Operand value) {
451        Object val = walkOperand(value);
452        Object not = pushDownNot(val);
453        if (!(not instanceof DBObject)) {
454            throw new QueryParseException("Cannot do NOT on: " + val);
455        }
456        return (DBObject) not;
457    }
458
459    protected Object pushDownNot(Object object) {
460        if (!(object instanceof DBObject)) {
461            throw new QueryParseException("Cannot do NOT on: " + object);
462        }
463        DBObject ob = (DBObject) object;
464        Set<String> keySet = ob.keySet();
465        if (keySet.size() != 1) {
466            throw new QueryParseException("Cannot do NOT on: " + ob);
467        }
468        String key = keySet.iterator().next();
469        Object value = ob.get(key);
470        if (!key.startsWith("$")) {
471            if (value instanceof DBObject) {
472                // push down inside dbobject
473                return new BasicDBObject(key, pushDownNot(value));
474            } else {
475                // k = v -> k != v
476                return new BasicDBObject(key, new BasicDBObject(QueryOperators.NE, value));
477            }
478        }
479        if (QueryOperators.NE.equals(key)) {
480            // NOT k != v -> k = v
481            return value;
482        }
483        if (QueryOperators.NOT.equals(key)) {
484            // NOT NOT v -> v
485            return value;
486        }
487        if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) {
488            // boolean algebra
489            // NOT (v1 AND v2) -> NOT v1 OR NOT v2
490            // NOT (v1 OR v2) -> NOT v1 AND NOT v2
491            String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND;
492            List<Object> list = (List<Object>) value;
493            for (int i = 0; i < list.size(); i++) {
494                list.set(i, pushDownNot(list.get(i)));
495            }
496            return new BasicDBObject(op, list);
497        }
498        if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) {
499            // boolean algebra
500            // IN <-> NIN
501            String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN;
502            return new BasicDBObject(op, value);
503        }
504        if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key)
505                || QueryOperators.GTE.equals(key)) {
506            // TODO use inverse operators?
507            return new BasicDBObject(QueryOperators.NOT, ob);
508        }
509        throw new QueryParseException("Unknown operator for NOT: " + key);
510    }
511
512    public DBObject walkIsNull(Operand value) {
513        FieldInfo fieldInfo = walkReference(value);
514        return new FieldInfoDBObject(fieldInfo, null);
515    }
516
517    public DBObject walkIsNotNull(Operand value) {
518        FieldInfo fieldInfo = walkReference(value);
519        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, null));
520    }
521
522    public DBObject walkMultiExpression(MultiExpression expr) {
523        return walkAnd(expr.values);
524    }
525
526    public DBObject walkAnd(Operand lvalue, Operand rvalue) {
527        return walkAnd(Arrays.asList(lvalue, rvalue));
528    }
529
530    protected DBObject walkAnd(List<Operand> values) {
531        List<Object> list = walkOperandList(values);
532        // check wildcards in the operands, extract common prefixes to use $elemMatch
533        Map<String, List<FieldInfoDBObject>> propBaseKeyToDBOs = new LinkedHashMap<>();
534        Map<String, String> propBaseKeyToFieldBase = new HashMap<>();
535        for (Iterator<Object> it = list.iterator(); it.hasNext();) {
536            Object ob = it.next();
537            if (ob instanceof FieldInfoDBObject) {
538                FieldInfoDBObject fidbo = (FieldInfoDBObject) ob;
539                FieldInfo fieldInfo = fidbo.fieldInfo;
540                if (fieldInfo.hasWildcard) {
541                    if (fieldInfo.fieldSuffix != null && fieldInfo.fieldSuffix.contains("*")) {
542                        // a double wildcard of the form foo/*/bar/* is not a problem if bar is an array
543                        // TODO prevent deep complex multiple wildcards
544                        // throw new QueryParseException("Cannot use two wildcards: " + fieldInfo.prop);
545                    }
546                    // generate a key unique per correlation for this element match
547                    String wildcardNumber = fieldInfo.fieldWildcard;
548                    if (wildcardNumber.isEmpty()) {
549                        // negative to not collide with regular correlated wildcards
550                        wildcardNumber = String.valueOf(-counter.incrementAndGet());
551                    }
552                    String propBaseKey = fieldInfo.fieldPrefix + "/*" + wildcardNumber;
553                    // store object for this key
554                    List<FieldInfoDBObject> dbos = propBaseKeyToDBOs.get(propBaseKey);
555                    if (dbos == null) {
556                        propBaseKeyToDBOs.put(propBaseKey, dbos = new LinkedList<>());
557                    }
558                    dbos.add(fidbo);
559                    // remember for which field base this is
560                    String fieldBase = fieldInfo.fieldPrefix.replace("/", ".");
561                    propBaseKeyToFieldBase.put(propBaseKey, fieldBase);
562                    // remove from list, will be re-added later through propBaseKeyToDBOs
563                    it.remove();
564                }
565            }
566        }
567        // generate $elemMatch items for correlated queries
568        for (Entry<String, List<FieldInfoDBObject>> es : propBaseKeyToDBOs.entrySet()) {
569            String propBaseKey = es.getKey();
570            List<FieldInfoDBObject> fidbos = es.getValue();
571            if (fidbos.size() == 1) {
572                // regular uncorrelated match
573                list.addAll(fidbos);
574            } else {
575                DBObject elemMatch = new BasicDBObject();
576                for (FieldInfoDBObject fidbo : fidbos) {
577                    // truncate field name to just the suffix
578                    FieldInfo fieldInfo = fidbo.fieldInfo;
579                    Object value = fidbo.get(fieldInfo.queryField);
580                    String fieldSuffix = fieldInfo.fieldSuffix.replace("/", ".");
581                    if (elemMatch.containsField(fieldSuffix)) {
582                        // ecm:acl/*1/principal = 'bob' AND ecm:acl/*1/principal = 'steve'
583                        // cannot match
584                        // TODO do better
585                        value = "__NOSUCHVALUE__";
586                    }
587                    elemMatch.put(fieldSuffix, value);
588                }
589                String fieldBase = propBaseKeyToFieldBase.get(propBaseKey);
590                BasicDBObject dbo = new BasicDBObject(fieldBase,
591                        new BasicDBObject(QueryOperators.ELEM_MATCH, elemMatch));
592                list.add(dbo);
593            }
594        }
595        if (list.size() == 1) {
596            return (DBObject) list.get(0);
597        } else {
598            return new BasicDBObject(QueryOperators.AND, list);
599        }
600    }
601
602    public DBObject walkOr(Operand lvalue, Operand rvalue) {
603        Object left = walkOperand(lvalue);
604        Object right = walkOperand(rvalue);
605        List<Object> list = new ArrayList<>(Arrays.asList(left, right));
606        return new BasicDBObject(QueryOperators.OR, list);
607    }
608
609    protected Object checkBoolean(FieldInfo fieldInfo, Object right) {
610        if (fieldInfo.isBoolean()) {
611            // convert 0 / 1 to actual booleans
612            if (right instanceof Long) {
613                if (ZERO.equals(right)) {
614                    right = fieldInfo.isTrueOrNullBoolean ? null : FALSE;
615                } else if (ONE.equals(right)) {
616                    right = TRUE;
617                } else {
618                    throw new QueryParseException("Invalid boolean: " + right);
619                }
620            }
621        }
622        return right;
623    }
624
625    public DBObject walkEq(Operand lvalue, Operand rvalue) {
626        FieldInfo fieldInfo = walkReference(lvalue);
627        Object right = walkOperand(rvalue);
628        if (isMixinTypes(fieldInfo)) {
629            if (!(right instanceof String)) {
630                throw new QueryParseException("Invalid EQ rhs: " + rvalue);
631            }
632            return walkMixinTypes(Collections.singletonList((String) right), true);
633        }
634        right = checkBoolean(fieldInfo, right);
635        // TODO check list fields
636        return new FieldInfoDBObject(fieldInfo, right);
637    }
638
639    public DBObject walkNotEq(Operand lvalue, Operand rvalue) {
640        FieldInfo fieldInfo = walkReference(lvalue);
641        Object right = walkOperand(rvalue);
642        if (isMixinTypes(fieldInfo)) {
643            if (!(right instanceof String)) {
644                throw new QueryParseException("Invalid NE rhs: " + rvalue);
645            }
646            return walkMixinTypes(Collections.singletonList((String) right), false);
647        }
648        right = checkBoolean(fieldInfo, right);
649        // TODO check list fields
650        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, right));
651    }
652
653    public DBObject walkLt(Operand lvalue, Operand rvalue) {
654        FieldInfo fieldInfo = walkReference(lvalue);
655        Object right = walkOperand(rvalue);
656        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, right));
657    }
658
659    public DBObject walkGt(Operand lvalue, Operand rvalue) {
660        FieldInfo fieldInfo = walkReference(lvalue);
661        Object right = walkOperand(rvalue);
662        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right));
663    }
664
665    public DBObject walkLtEq(Operand lvalue, Operand rvalue) {
666        FieldInfo fieldInfo = walkReference(lvalue);
667        Object right = walkOperand(rvalue);
668        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LTE, right));
669    }
670
671    public DBObject walkGtEq(Operand lvalue, Operand rvalue) {
672        FieldInfo fieldInfo = walkReference(lvalue);
673        Object right = walkOperand(rvalue);
674        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GTE, right));
675    }
676
677    public DBObject walkBetween(Operand lvalue, Operand rvalue, boolean positive) {
678        LiteralList l = (LiteralList) rvalue;
679        FieldInfo fieldInfo = walkReference(lvalue);
680        Object left = walkOperand(l.get(0));
681        Object right = walkOperand(l.get(1));
682        if (positive) {
683            DBObject range = new BasicDBObject();
684            range.put(QueryOperators.GTE, left);
685            range.put(QueryOperators.LTE, right);
686            return new FieldInfoDBObject(fieldInfo, range);
687        } else {
688            DBObject a = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, left));
689            DBObject b = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right));
690            return new BasicDBObject(QueryOperators.OR, Arrays.asList(a, b));
691        }
692    }
693
694    public DBObject walkIn(Operand lvalue, Operand rvalue, boolean positive) {
695        FieldInfo fieldInfo = walkReference(lvalue);
696        Object right = walkOperand(rvalue);
697        if (!(right instanceof List)) {
698            throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue);
699        }
700        if (isMixinTypes(fieldInfo)) {
701            return walkMixinTypes((List<String>) right, positive);
702        }
703        // TODO check list fields
704        List<Object> list = (List<Object>) right;
705        return new FieldInfoDBObject(fieldInfo,
706                new BasicDBObject(positive ? QueryOperators.IN : QueryOperators.NIN, list));
707    }
708
709    public DBObject walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) {
710        FieldInfo fieldInfo = walkReference(lvalue);
711        if (!(rvalue instanceof StringLiteral)) {
712            throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue);
713        }
714        // TODO check list fields
715        String like = walkStringLiteral((StringLiteral) rvalue);
716        String regex = ExpressionEvaluator.likeToRegex(like);
717
718        int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0;
719        Pattern pattern = Pattern.compile(regex, flags);
720        Object value;
721        if (positive) {
722            value = pattern;
723        } else {
724            value = new BasicDBObject(QueryOperators.NOT, pattern);
725        }
726        return new FieldInfoDBObject(fieldInfo, value);
727    }
728
729    public Object walkOperand(Operand op) {
730        if (op instanceof Literal) {
731            return walkLiteral((Literal) op);
732        } else if (op instanceof LiteralList) {
733            return walkLiteralList((LiteralList) op);
734        } else if (op instanceof Function) {
735            return walkFunction((Function) op);
736        } else if (op instanceof Expression) {
737            return walkExpression((Expression) op);
738        } else if (op instanceof Reference) {
739            return walkReference((Reference) op);
740        } else {
741            throw new QueryParseException("Unknown operand: " + op);
742        }
743    }
744
745    public Object walkLiteral(Literal lit) {
746        if (lit instanceof BooleanLiteral) {
747            return walkBooleanLiteral((BooleanLiteral) lit);
748        } else if (lit instanceof DateLiteral) {
749            return walkDateLiteral((DateLiteral) lit);
750        } else if (lit instanceof DoubleLiteral) {
751            return walkDoubleLiteral((DoubleLiteral) lit);
752        } else if (lit instanceof IntegerLiteral) {
753            return walkIntegerLiteral((IntegerLiteral) lit);
754        } else if (lit instanceof StringLiteral) {
755            return walkStringLiteral((StringLiteral) lit);
756        } else {
757            throw new QueryParseException("Unknown literal: " + lit);
758        }
759    }
760
761    public Object walkBooleanLiteral(BooleanLiteral lit) {
762        return Boolean.valueOf(lit.value);
763    }
764
765    public Date walkDateLiteral(DateLiteral lit) {
766        return lit.value.toDate(); // TODO onlyDate
767    }
768
769    public Double walkDoubleLiteral(DoubleLiteral lit) {
770        return Double.valueOf(lit.value);
771    }
772
773    public Long walkIntegerLiteral(IntegerLiteral lit) {
774        return Long.valueOf(lit.value);
775    }
776
777    public String walkStringLiteral(StringLiteral lit) {
778        return lit.value;
779    }
780
781    public List<Object> walkLiteralList(LiteralList litList) {
782        List<Object> list = new ArrayList<>(litList.size());
783        for (Literal lit : litList) {
784            list.add(walkLiteral(lit));
785        }
786        return list;
787    }
788
789    protected List<Object> walkOperandList(List<Operand> values) {
790        List<Object> list = new LinkedList<>();
791        for (Operand value : values) {
792            list.add(walkOperand(value));
793        }
794        return list;
795    }
796
797    public Object walkFunction(Function func) {
798        throw new UnsupportedOperationException(func.name);
799    }
800
801    public DBObject walkStartsWith(Operand lvalue, Operand rvalue) {
802        if (!(lvalue instanceof Reference)) {
803            throw new QueryParseException("Invalid STARTSWITH query, left hand side must be a property: " + lvalue);
804        }
805        String name = ((Reference) lvalue).name;
806        if (!(rvalue instanceof StringLiteral)) {
807            throw new QueryParseException(
808                    "Invalid STARTSWITH query, right hand side must be a literal path: " + rvalue);
809        }
810        String path = ((StringLiteral) rvalue).value;
811        if (path.length() > 1 && path.endsWith("/")) {
812            path = path.substring(0, path.length() - 1);
813        }
814
815        if (NXQL.ECM_PATH.equals(name)) {
816            return walkStartsWithPath(path);
817        } else {
818            return walkStartsWithNonPath(lvalue, path);
819        }
820    }
821
822    protected DBObject walkStartsWithPath(String path) {
823        // resolve path
824        String ancestorId = pathResolver.getIdForPath(path);
825        if (ancestorId == null) {
826            // no such path
827            // TODO XXX do better
828            return new BasicDBObject(MONGODB_ID, "__nosuchid__");
829        }
830        return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
831    }
832
833    protected DBObject walkStartsWithNonPath(Operand lvalue, String path) {
834        FieldInfo fieldInfo = walkReference(lvalue);
835        DBObject eq = new FieldInfoDBObject(fieldInfo, path);
836        // escape except alphanumeric and others not needing escaping
837        String regex = path.replaceAll("([^a-zA-Z0-9 /])", "\\\\$1");
838        Pattern pattern = Pattern.compile(regex + "/.*");
839        DBObject like = new FieldInfoDBObject(fieldInfo, pattern);
840        return new BasicDBObject(QueryOperators.OR, Arrays.asList(eq, like));
841    }
842
843    protected FieldInfo walkReference(Operand value) {
844        if (!(value instanceof Reference)) {
845            throw new QueryParseException("Invalid query, left hand side must be a property: " + value);
846        }
847        return walkReference((Reference) value);
848    }
849
850    // non-canonical index syntax, for replaceAll
851    protected final static Pattern NON_CANON_INDEX = Pattern.compile("[^/\\[\\]]+" // name
852            + "\\[(\\d+|\\*|\\*\\d+)\\]" // index in brackets
853    );
854
855    /**
856     * Canonicalizes a Nuxeo-xpath.
857     * <p>
858     * Replaces {@code a/foo[123]/b} with {@code a/123/b}
859     * <p>
860     * A star or a star followed by digits can be used instead of just the digits as well.
861     *
862     * @param xpath the xpath
863     * @return the canonicalized xpath.
864     */
865    public static String canonicalXPath(String xpath) {
866        while (xpath.length() > 0 && xpath.charAt(0) == '/') {
867            xpath = xpath.substring(1);
868        }
869        if (xpath.indexOf('[') == -1) {
870            return xpath;
871        } else {
872            return NON_CANON_INDEX.matcher(xpath).replaceAll("$1");
873        }
874    }
875
876    /** Splits foo.*.bar into foo, *, bar and split foo.*1.bar into foo, *1, bar with the last bar part optional */
877    protected final static Pattern WILDCARD_SPLIT = Pattern.compile("([^*]*)\\.\\*(\\d*)(?:\\.(.*))?");
878
879    protected static class FieldInfo {
880
881        /** NXQL property. */
882        protected final String prop;
883
884        /** MongoDB field including wildcards (not used as-is). */
885        protected final String fullField;
886
887        /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */
888        protected final String queryField;
889
890        /** MongoDB field for projection. */
891        protected final String projectionField;
892
893        protected final Type type;
894
895        /**
896         * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly.
897         */
898        protected final boolean isTrueOrNullBoolean;
899
900        protected final boolean hasWildcard;
901
902        /** Prefix before the wildcard. */
903        protected final String fieldPrefix;
904
905        /** Wildcard part after * */
906        protected final String fieldWildcard;
907
908        /** Part after wildcard, may be null. */
909        protected final String fieldSuffix;
910
911        protected FieldInfo(String prop, String fullField, String queryField, String projectionField, Type type,
912                boolean isTrueOrNullBoolean) {
913            this.prop = prop;
914            this.fullField = fullField;
915            this.queryField = queryField;
916            this.projectionField = projectionField;
917            this.type = type;
918            this.isTrueOrNullBoolean = isTrueOrNullBoolean;
919            Matcher m = WILDCARD_SPLIT.matcher(fullField);
920            if (m.matches()) {
921                hasWildcard = true;
922                fieldPrefix = m.group(1);
923                fieldWildcard = m.group(2);
924                fieldSuffix = m.group(3);
925            } else {
926                hasWildcard = false;
927                fieldPrefix = fieldWildcard = fieldSuffix = null;
928            }
929        }
930
931        protected boolean isBoolean() {
932            return type instanceof BooleanType;
933        }
934    }
935
936    protected static class FieldInfoDBObject extends BasicDBObject {
937
938        private static final long serialVersionUID = 1L;
939
940        protected FieldInfo fieldInfo;
941
942        public FieldInfoDBObject(FieldInfo fieldInfo, Object value) {
943            super(fieldInfo.queryField, value);
944            this.fieldInfo = fieldInfo;
945        }
946    }
947
948    /**
949     * Returns the MongoDB field for this reference.
950     */
951    public FieldInfo walkReference(Reference ref) {
952        FieldInfo fieldInfo = walkReference(ref.name);
953        if (DATE_CAST.equals(ref.cast)) {
954            Type type = fieldInfo.type;
955            if (!(type instanceof DateType
956                    || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) {
957                throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name);
958            }
959            // fieldInfo.isDateCast = true;
960        }
961        return fieldInfo;
962    }
963
964    protected FieldInfo walkReference(String name) {
965        String prop = canonicalXPath(name);
966        String[] parts = prop.split("/");
967        if (prop.startsWith(NXQL.ECM_PREFIX)) {
968            if (prop.startsWith(NXQL.ECM_ACL + "/")) {
969                return parseACP(prop, parts);
970            }
971            // simple field
972            String field = DBSSession.convToInternal(prop);
973            Type type = DBSSession.getType(field);
974            String queryField = converter.keyToBson(field);
975            return new FieldInfo(prop, field, queryField, field, type, true);
976        } else {
977            String first = parts[0];
978            Field field = schemaManager.getField(first);
979            if (field == null) {
980                if (first.indexOf(':') > -1) {
981                    throw new QueryParseException("No such property: " + name);
982                }
983                // check without prefix
984                // TODO precompute this in SchemaManagerImpl
985                for (Schema schema : schemaManager.getSchemas()) {
986                    if (!StringUtils.isBlank(schema.getNamespace().prefix)) {
987                        // schema with prefix, do not consider as candidate
988                        continue;
989                    }
990                    if (schema != null) {
991                        field = schema.getField(first);
992                        if (field != null) {
993                            break;
994                        }
995                    }
996                }
997                if (field == null) {
998                    throw new QueryParseException("No such property: " + name);
999                }
1000            }
1001            Type type = field.getType();
1002            // canonical name
1003            parts[0] = field.getName().getPrefixedName();
1004            // are there wildcards or list indexes?
1005            List<String> queryFieldParts = new LinkedList<>(); // field for query
1006            List<String> projectionFieldParts = new LinkedList<>(); // field for projection
1007            boolean firstPart = true;
1008            for (String part : parts) {
1009                if (NumberUtils.isDigits(part)) {
1010                    // explicit list index
1011                    queryFieldParts.add(part);
1012                    type = ((ListType) type).getFieldType();
1013                } else if (!part.startsWith("*")) {
1014                    // complex sub-property
1015                    queryFieldParts.add(part);
1016                    projectionFieldParts.add(part);
1017                    if (!firstPart) {
1018                        // we already computed the type of the first part
1019                        field = ((ComplexType) type).getField(part);
1020                        if (field == null) {
1021                            throw new QueryParseException("No such property: " + name);
1022                        }
1023                        type = field.getType();
1024                    }
1025                } else {
1026                    // wildcard
1027                    type = ((ListType) type).getFieldType();
1028                }
1029                firstPart = false;
1030            }
1031            String fullField = StringUtils.join(parts, '.');
1032            String queryField = StringUtils.join(queryFieldParts, '.');
1033            String projectionField = StringUtils.join(projectionFieldParts, '.');
1034            return new FieldInfo(prop, fullField, queryField, projectionField, type, false);
1035        }
1036    }
1037
1038    protected FieldInfo parseACP(String prop, String[] parts) {
1039        if (parts.length != 3) {
1040            throw new QueryParseException("No such property: " + prop);
1041        }
1042        String wildcard = parts[1];
1043        if (NumberUtils.isDigits(wildcard)) {
1044            throw new QueryParseException("Cannot use explicit index in ACLs: " + prop);
1045        }
1046        String last = parts[2];
1047        String fullField;
1048        String queryField;
1049        String projectionField;
1050        if (NXQL.ECM_ACL_NAME.equals(last)) {
1051            fullField = KEY_ACP + "." + KEY_ACL_NAME;
1052            queryField = KEY_ACP + "." + KEY_ACL_NAME;
1053            // TODO remember wildcard correlation
1054        } else {
1055            String fieldLast = DBSSession.convToInternalAce(last);
1056            if (fieldLast == null) {
1057                throw new QueryParseException("No such property: " + prop);
1058            }
1059            fullField = KEY_ACP + "." + KEY_ACL + "." + wildcard + "." + fieldLast;
1060            queryField = KEY_ACP + "." + KEY_ACL + "." + fieldLast;
1061        }
1062        Type type = DBSSession.getType(last);
1063        projectionField = queryField;
1064        return new FieldInfo(prop, fullField, queryField, projectionField, type, false);
1065    }
1066
1067    protected boolean isMixinTypes(FieldInfo fieldInfo) {
1068        return fieldInfo.queryField.equals(DBSDocument.KEY_MIXIN_TYPES);
1069    }
1070
1071    protected Set<String> getMixinDocumentTypes(String mixin) {
1072        Set<String> types = schemaManager.getDocumentTypeNamesForFacet(mixin);
1073        return types == null ? Collections.emptySet() : types;
1074    }
1075
1076    protected List<String> getDocumentTypes() {
1077        // TODO precompute in SchemaManager
1078        if (documentTypes == null) {
1079            documentTypes = new ArrayList<>();
1080            for (DocumentType docType : schemaManager.getDocumentTypes()) {
1081                documentTypes.add(docType.getName());
1082            }
1083        }
1084        return documentTypes;
1085    }
1086
1087    protected boolean isNeverPerInstanceMixin(String mixin) {
1088        return schemaManager.getNoPerDocumentQueryFacets().contains(mixin);
1089    }
1090
1091    /**
1092     * Matches the mixin types against a list of values.
1093     * <p>
1094     * Used for:
1095     * <ul>
1096     * <li>ecm:mixinTypes = 'Foo'
1097     * <li>ecm:mixinTypes != 'Foo'
1098     * <li>ecm:mixinTypes IN ('Foo', 'Bar')
1099     * <li>ecm:mixinTypes NOT IN ('Foo', 'Bar')
1100     * </ul>
1101     * <p>
1102     * ecm:mixinTypes IN ('Foo', 'Bar')
1103     *
1104     * <pre>
1105     * { "$or" : [ { "ecm:primaryType" : { "$in" : [ ... types with Foo or Bar ...]}} ,
1106     *             { "ecm:mixinTypes" : { "$in" : [ "Foo" , "Bar]}}]}
1107     * </pre>
1108     *
1109     * ecm:mixinTypes NOT IN ('Foo', 'Bar')
1110     * <p>
1111     *
1112     * <pre>
1113     * { "$and" : [ { "ecm:primaryType" : { "$in" : [ ... types without Foo nor Bar ...]}} ,
1114     *              { "ecm:mixinTypes" : { "$nin" : [ "Foo" , "Bar]}}]}
1115     * </pre>
1116     */
1117    public DBObject walkMixinTypes(List<String> mixins, boolean include) {
1118        /*
1119         * Primary types that match.
1120         */
1121        Set<String> matchPrimaryTypes;
1122        if (include) {
1123            matchPrimaryTypes = new HashSet<>();
1124            for (String mixin : mixins) {
1125                matchPrimaryTypes.addAll(getMixinDocumentTypes(mixin));
1126            }
1127        } else {
1128            matchPrimaryTypes = new HashSet<>(getDocumentTypes());
1129            for (String mixin : mixins) {
1130                matchPrimaryTypes.removeAll(getMixinDocumentTypes(mixin));
1131            }
1132        }
1133        /*
1134         * Instance mixins that match.
1135         */
1136        Set<String> matchMixinTypes = new HashSet<>();
1137        for (String mixin : mixins) {
1138            if (!isNeverPerInstanceMixin(mixin)) {
1139                matchMixinTypes.add(mixin);
1140            }
1141        }
1142        /*
1143         * MongoDB query generation.
1144         */
1145        // match on primary type
1146        DBObject p = new BasicDBObject(DBSDocument.KEY_PRIMARY_TYPE,
1147                new BasicDBObject(QueryOperators.IN, matchPrimaryTypes));
1148        // match on mixin types
1149        // $in/$nin with an array matches if any/no element of the array matches
1150        String innin = include ? QueryOperators.IN : QueryOperators.NIN;
1151        DBObject m = new BasicDBObject(DBSDocument.KEY_MIXIN_TYPES, new BasicDBObject(innin, matchMixinTypes));
1152        // and/or between those
1153        String op = include ? QueryOperators.OR : QueryOperators.AND;
1154        return new BasicDBObject(op, Arrays.asList(p, m));
1155    }
1156
1157}