Source code

001/*
002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_GRANT;
024import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
025import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL_NAME;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SCORE;
028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ID;
029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_NAME;
030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PARENT_ID;
031import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID;
032import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_META;
033import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_TEXT_SCORE;
034
035import java.util.ArrayList;
036import java.util.Arrays;
037import java.util.Collections;
038import java.util.Date;
039import java.util.HashMap;
040import java.util.HashSet;
041import java.util.Iterator;
042import java.util.LinkedHashMap;
043import java.util.LinkedList;
044import java.util.List;
045import java.util.Map;
046import java.util.Map.Entry;
047import java.util.Set;
048import java.util.concurrent.atomic.AtomicInteger;
049import java.util.regex.Matcher;
050import java.util.regex.Pattern;
051
052import org.apache.commons.lang.StringUtils;
053import org.apache.commons.lang.math.NumberUtils;
054import org.nuxeo.ecm.core.query.QueryParseException;
055import org.nuxeo.ecm.core.query.sql.NXQL;
056import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral;
057import org.nuxeo.ecm.core.query.sql.model.DateLiteral;
058import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral;
059import org.nuxeo.ecm.core.query.sql.model.Expression;
060import org.nuxeo.ecm.core.query.sql.model.Function;
061import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral;
062import org.nuxeo.ecm.core.query.sql.model.Literal;
063import org.nuxeo.ecm.core.query.sql.model.LiteralList;
064import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
065import org.nuxeo.ecm.core.query.sql.model.Operand;
066import org.nuxeo.ecm.core.query.sql.model.Operator;
067import org.nuxeo.ecm.core.query.sql.model.OrderByClause;
068import org.nuxeo.ecm.core.query.sql.model.OrderByExpr;
069import org.nuxeo.ecm.core.query.sql.model.Reference;
070import org.nuxeo.ecm.core.query.sql.model.SelectClause;
071import org.nuxeo.ecm.core.query.sql.model.SelectList;
072import org.nuxeo.ecm.core.query.sql.model.StringLiteral;
073import org.nuxeo.ecm.core.schema.DocumentType;
074import org.nuxeo.ecm.core.schema.SchemaManager;
075import org.nuxeo.ecm.core.schema.types.ComplexType;
076import org.nuxeo.ecm.core.schema.types.Field;
077import org.nuxeo.ecm.core.schema.types.ListType;
078import org.nuxeo.ecm.core.schema.types.Schema;
079import org.nuxeo.ecm.core.schema.types.Type;
080import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
081import org.nuxeo.ecm.core.schema.types.primitives.DateType;
082import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
083import org.nuxeo.ecm.core.storage.ExpressionEvaluator.PathResolver;
084import org.nuxeo.ecm.core.storage.dbs.DBSDocument;
085import org.nuxeo.ecm.core.storage.dbs.DBSSession;
086import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer;
087import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.FulltextQuery;
088import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.Op;
089import org.nuxeo.runtime.api.Framework;
090
091import com.mongodb.BasicDBObject;
092import com.mongodb.DBObject;
093import com.mongodb.QueryOperators;
094
095/**
096 * Query builder for a MongoDB query from an {@link Expression}.
097 *
098 * @since 5.9.4
099 */
100public class MongoDBQueryBuilder {
101
102    private static final Long ZERO = Long.valueOf(0);
103
104    private static final Long ONE = Long.valueOf(1);
105
106    private static final Long MINUS_ONE = Long.valueOf(-1);
107
108    protected static final String DATE_CAST = "DATE";
109
110    protected final AtomicInteger counter = new AtomicInteger();
111
112    protected final SchemaManager schemaManager;
113
114    protected List<String> documentTypes;
115
116    protected final Expression expression;
117
118    protected final SelectClause selectClause;
119
120    protected final OrderByClause orderByClause;
121
122    protected final PathResolver pathResolver;
123
124    public boolean hasFulltext;
125
126    public boolean sortOnFulltextScore;
127
128    protected DBObject query;
129
130    protected DBObject orderBy;
131
132    protected DBObject projection;
133
134    boolean projectionHasWildcard;
135
136    private boolean fulltextSearchDisabled;
137
138    public MongoDBQueryBuilder(Expression expression, SelectClause selectClause, OrderByClause orderByClause,
139            PathResolver pathResolver, boolean fulltextSearchDisabled) {
140        schemaManager = Framework.getLocalService(SchemaManager.class);
141        this.expression = expression;
142        this.selectClause = selectClause;
143        this.orderByClause = orderByClause;
144        this.pathResolver = pathResolver;
145        this.fulltextSearchDisabled = fulltextSearchDisabled;
146    }
147
148    public void walk() {
149        query = walkExpression(expression); // computes hasFulltext
150        walkOrderBy(); // computes sortOnFulltextScore
151        walkProjection(); // needs hasFulltext and sortOnFulltextScore
152    }
153
154    public DBObject getQuery() {
155        return query;
156    }
157
158    public DBObject getOrderBy() {
159        return orderBy;
160    }
161
162    public DBObject getProjection() {
163        return projection;
164    }
165
166    public boolean hasProjectionWildcard() {
167        return projectionHasWildcard;
168    }
169
170    protected void walkOrderBy() {
171        sortOnFulltextScore = false;
172        if (orderByClause == null) {
173            orderBy = null;
174        } else {
175            orderBy = new BasicDBObject();
176            for (OrderByExpr ob : orderByClause.elements) {
177                Reference ref = ob.reference;
178                boolean desc = ob.isDescending;
179                String field = walkReference(ref).queryField;
180                if (!orderBy.containsField(field)) {
181                    Object value;
182                    if (KEY_FULLTEXT_SCORE.equals(field)) {
183                        if (!desc) {
184                            throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " ascending");
185                        }
186                        sortOnFulltextScore = true;
187                        value = new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE);
188                    } else {
189                        value = desc ? MINUS_ONE : ONE;
190                    }
191                    orderBy.put(field, value);
192                }
193            }
194            if (sortOnFulltextScore && ((BasicDBObject) orderBy).size() > 1) {
195                throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " and other criteria");
196            }
197        }
198    }
199
200    protected void walkProjection() {
201        projection = new BasicDBObject();
202        projection.put(KEY_ID, ONE); // always useful
203        projection.put(KEY_NAME, ONE); // used in ORDER BY ecm:path
204        projection.put(KEY_PARENT_ID, ONE); // used in ORDER BY ecm:path
205        boolean projectionOnFulltextScore = false;
206        for (int i = 0; i < selectClause.elements.size(); i++) {
207            Operand op = selectClause.elements.get(i);
208            if (!(op instanceof Reference)) {
209                throw new QueryParseException("Projection not supported: " + op);
210            }
211            FieldInfo fieldInfo = walkReference((Reference) op);
212            projection.put(fieldInfo.projectionField, ONE);
213            if (fieldInfo.hasWildcard) {
214                projectionHasWildcard = true;
215            }
216            if (fieldInfo.projectionField.equals(KEY_FULLTEXT_SCORE)) {
217                projectionOnFulltextScore = true;
218            }
219        }
220        if (projectionOnFulltextScore || sortOnFulltextScore) {
221            if (!hasFulltext) {
222                throw new QueryParseException(NXQL.ECM_FULLTEXT_SCORE + " cannot be used without " + NXQL.ECM_FULLTEXT);
223            }
224            projection.put(KEY_FULLTEXT_SCORE, new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE));
225        }
226    }
227
228    public DBObject walkExpression(Expression expr) {
229        Operator op = expr.operator;
230        Operand lvalue = expr.lvalue;
231        Operand rvalue = expr.rvalue;
232        Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null;
233        String name = ref != null ? ref.name : null;
234        String cast = ref != null ? ref.cast : null;
235        if (DATE_CAST.equals(cast)) {
236            checkDateLiteralForCast(op, rvalue, name);
237        }
238        if (op == Operator.STARTSWITH) {
239            return walkStartsWith(lvalue, rvalue);
240        } else if (NXQL.ECM_PATH.equals(name)) {
241            return walkEcmPath(op, rvalue);
242        } else if (NXQL.ECM_ANCESTORID.equals(name)) {
243            return walkAncestorId(op, rvalue);
244        } else if (name != null && name.startsWith(NXQL.ECM_FULLTEXT) && !NXQL.ECM_FULLTEXT_JOBID.equals(name)) {
245            return walkEcmFulltext(name, op, rvalue);
246        } else if (op == Operator.SUM) {
247            throw new UnsupportedOperationException("SUM");
248        } else if (op == Operator.SUB) {
249            throw new UnsupportedOperationException("SUB");
250        } else if (op == Operator.MUL) {
251            throw new UnsupportedOperationException("MUL");
252        } else if (op == Operator.DIV) {
253            throw new UnsupportedOperationException("DIV");
254        } else if (op == Operator.LT) {
255            return walkLt(lvalue, rvalue);
256        } else if (op == Operator.GT) {
257            return walkGt(lvalue, rvalue);
258        } else if (op == Operator.EQ) {
259            return walkEq(lvalue, rvalue);
260        } else if (op == Operator.NOTEQ) {
261            return walkNotEq(lvalue, rvalue);
262        } else if (op == Operator.LTEQ) {
263            return walkLtEq(lvalue, rvalue);
264        } else if (op == Operator.GTEQ) {
265            return walkGtEq(lvalue, rvalue);
266        } else if (op == Operator.AND) {
267            if (expr instanceof MultiExpression) {
268                return walkMultiExpression((MultiExpression) expr);
269            } else {
270                return walkAnd(lvalue, rvalue);
271            }
272        } else if (op == Operator.NOT) {
273            return walkNot(lvalue);
274        } else if (op == Operator.OR) {
275            return walkOr(lvalue, rvalue);
276        } else if (op == Operator.LIKE) {
277            return walkLike(lvalue, rvalue, true, false);
278        } else if (op == Operator.ILIKE) {
279            return walkLike(lvalue, rvalue, true, true);
280        } else if (op == Operator.NOTLIKE) {
281            return walkLike(lvalue, rvalue, false, false);
282        } else if (op == Operator.NOTILIKE) {
283            return walkLike(lvalue, rvalue, false, true);
284        } else if (op == Operator.IN) {
285            return walkIn(lvalue, rvalue, true);
286        } else if (op == Operator.NOTIN) {
287            return walkIn(lvalue, rvalue, false);
288        } else if (op == Operator.ISNULL) {
289            return walkIsNull(lvalue);
290        } else if (op == Operator.ISNOTNULL) {
291            return walkIsNotNull(lvalue);
292        } else if (op == Operator.BETWEEN) {
293            return walkBetween(lvalue, rvalue, true);
294        } else if (op == Operator.NOTBETWEEN) {
295            return walkBetween(lvalue, rvalue, false);
296        } else {
297            throw new QueryParseException("Unknown operator: " + op);
298        }
299    }
300
301    protected void checkDateLiteralForCast(Operator op, Operand value, String name) {
302        if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) {
303            LiteralList l = (LiteralList) value;
304            checkDateLiteralForCast(l.get(0), name);
305            checkDateLiteralForCast(l.get(1), name);
306        } else {
307            checkDateLiteralForCast(value, name);
308        }
309    }
310
311    protected void checkDateLiteralForCast(Operand value, String name) {
312        if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) {
313            throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name);
314        }
315    }
316
317    protected DBObject walkEcmPath(Operator op, Operand rvalue) {
318        if (op != Operator.EQ && op != Operator.NOTEQ) {
319            throw new QueryParseException(NXQL.ECM_PATH + " requires = or <> operator");
320        }
321        if (!(rvalue instanceof StringLiteral)) {
322            throw new QueryParseException(NXQL.ECM_PATH + " requires literal path as right argument");
323        }
324        String path = ((StringLiteral) rvalue).value;
325        if (path.length() > 1 && path.endsWith("/")) {
326            path = path.substring(0, path.length() - 1);
327        }
328        String id = pathResolver.getIdForPath(path);
329        if (id == null) {
330            // no such path
331            // TODO XXX do better
332            return new BasicDBObject(MONGODB_ID, "__nosuchid__");
333        }
334        if (op == Operator.EQ) {
335            return new BasicDBObject(DBSDocument.KEY_ID, id);
336        } else {
337            return new BasicDBObject(DBSDocument.KEY_ID, new BasicDBObject(QueryOperators.NE, id));
338        }
339    }
340
341    protected DBObject walkAncestorId(Operator op, Operand rvalue) {
342        if (op != Operator.EQ && op != Operator.NOTEQ) {
343            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires = or <> operator");
344        }
345        if (!(rvalue instanceof StringLiteral)) {
346            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires literal id as right argument");
347        }
348        String ancestorId = ((StringLiteral) rvalue).value;
349        if (op == Operator.EQ) {
350            return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
351        } else {
352            return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, new BasicDBObject(QueryOperators.NE, ancestorId));
353        }
354    }
355
356    protected DBObject walkEcmFulltext(String name, Operator op, Operand rvalue) {
357        if (op != Operator.EQ && op != Operator.LIKE) {
358            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires = or LIKE operator");
359        }
360        if (!(rvalue instanceof StringLiteral)) {
361            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires literal string as right argument");
362        }
363        if (fulltextSearchDisabled) {
364            throw new QueryParseException("Fulltext search disabled by configuration");
365        }
366        String fulltextQuery = ((StringLiteral) rvalue).value;
367        if (name.equals(NXQL.ECM_FULLTEXT)) {
368            // standard fulltext query
369            hasFulltext = true;
370            String ft = getMongoDBFulltextQuery(fulltextQuery);
371            if (ft == null) {
372                // empty query, matches nothing
373                return new BasicDBObject(MONGODB_ID, "__nosuchid__");
374            }
375            DBObject textSearch = new BasicDBObject();
376            textSearch.put(QueryOperators.SEARCH, ft);
377            // TODO language?
378            return new BasicDBObject(QueryOperators.TEXT, textSearch);
379        } else {
380            // secondary index match with explicit field
381            // do a regexp on the field
382            if (name.charAt(NXQL.ECM_FULLTEXT.length()) != '.') {
383                throw new QueryParseException(name + " has incorrect syntax" + " for a secondary fulltext index");
384            }
385            String prop = name.substring(NXQL.ECM_FULLTEXT.length() + 1);
386            String ft = fulltextQuery.replace(" ", "%");
387            rvalue = new StringLiteral(ft);
388            return walkLike(new Reference(prop), rvalue, true, true);
389        }
390    }
391
392    // public static for tests
393    public static String getMongoDBFulltextQuery(String query) {
394        FulltextQuery ft = FulltextQueryAnalyzer.analyzeFulltextQuery(query);
395        if (ft == null) {
396            return null;
397        }
398        // translate into MongoDB syntax
399        return translateFulltext(ft, false);
400    }
401
402    /**
403     * Transforms the NXQL fulltext syntax into MongoDB syntax.
404     * <p>
405     * The MongoDB fulltext query syntax is badly documented, but is actually the following:
406     * <ul>
407     * <li>a term is a word,
408     * <li>a phrase is a set of spaced-separated words enclosed in double quotes,
409     * <li>negation is done by prepending a -,
410     * <li>the query is a space-separated set of terms, negated terms, phrases, or negated phrases.
411     * <li>all the words of non-negated phrases are also added to the terms.
412     * </ul>
413     * <p>
414     * The matching algorithm is (excluding stemming and stop words):
415     * <ul>
416     * <li>filter out documents with the negative terms, the negative phrases, or missing the phrases,
417     * <li>then if any term is present in the document then it's a match.
418     * </ul>
419     */
420    protected static String translateFulltext(FulltextQuery ft, boolean and) {
421        List<String> buf = new ArrayList<>();
422        translateFulltext(ft, buf, and);
423        return StringUtils.join(buf, ' ');
424    }
425
426    protected static void translateFulltext(FulltextQuery ft, List<String> buf, boolean and) {
427        if (ft.op == Op.OR) {
428            for (FulltextQuery term : ft.terms) {
429                // don't quote words for OR
430                translateFulltext(term, buf, false);
431            }
432        } else if (ft.op == Op.AND) {
433            for (FulltextQuery term : ft.terms) {
434                // quote words for AND
435                translateFulltext(term, buf, true);
436            }
437        } else {
438            String neg;
439            if (ft.op == Op.NOTWORD) {
440                neg = "-";
441            } else { // Op.WORD
442                neg = "";
443            }
444            String word = ft.word.toLowerCase();
445            if (ft.isPhrase() || and) {
446                buf.add(neg + '"' + word + '"');
447            } else {
448                buf.add(neg + word);
449            }
450        }
451    }
452
453    public DBObject walkNot(Operand value) {
454        Object val = walkOperand(value);
455        Object not = pushDownNot(val);
456        if (!(not instanceof DBObject)) {
457            throw new QueryParseException("Cannot do NOT on: " + val);
458        }
459        return (DBObject) not;
460    }
461
462    protected Object pushDownNot(Object object) {
463        if (!(object instanceof DBObject)) {
464            throw new QueryParseException("Cannot do NOT on: " + object);
465        }
466        DBObject ob = (DBObject) object;
467        Set<String> keySet = ob.keySet();
468        if (keySet.size() != 1) {
469            throw new QueryParseException("Cannot do NOT on: " + ob);
470        }
471        String key = keySet.iterator().next();
472        Object value = ob.get(key);
473        if (!key.startsWith("$")) {
474            if (value instanceof DBObject) {
475                // push down inside dbobject
476                return new BasicDBObject(key, pushDownNot(value));
477            } else {
478                // k = v -> k != v
479                return new BasicDBObject(key, new BasicDBObject(QueryOperators.NE, value));
480            }
481        }
482        if (QueryOperators.NE.equals(key)) {
483            // NOT k != v -> k = v
484            return value;
485        }
486        if (QueryOperators.NOT.equals(key)) {
487            // NOT NOT v -> v
488            return value;
489        }
490        if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) {
491            // boolean algebra
492            // NOT (v1 AND v2) -> NOT v1 OR NOT v2
493            // NOT (v1 OR v2) -> NOT v1 AND NOT v2
494            String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND;
495            List<Object> list = (List<Object>) value;
496            for (int i = 0; i < list.size(); i++) {
497                list.set(i, pushDownNot(list.get(i)));
498            }
499            return new BasicDBObject(op, list);
500        }
501        if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) {
502            // boolean algebra
503            // IN <-> NIN
504            String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN;
505            return new BasicDBObject(op, value);
506        }
507        if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key)
508                || QueryOperators.GTE.equals(key)) {
509            // TODO use inverse operators?
510            return new BasicDBObject(QueryOperators.NOT, ob);
511        }
512        throw new QueryParseException("Unknown operator for NOT: " + key);
513    }
514
515    public DBObject walkIsNull(Operand value) {
516        FieldInfo fieldInfo = walkReference(value);
517        return new FieldInfoDBObject(fieldInfo, null);
518    }
519
520    public DBObject walkIsNotNull(Operand value) {
521        FieldInfo fieldInfo = walkReference(value);
522        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, null));
523    }
524
525    public DBObject walkMultiExpression(MultiExpression expr) {
526        return walkAnd(expr.values);
527    }
528
529    public DBObject walkAnd(Operand lvalue, Operand rvalue) {
530        return walkAnd(Arrays.asList(lvalue, rvalue));
531    }
532
533    protected DBObject walkAnd(List<Operand> values) {
534        List<Object> list = walkOperandList(values);
535        // check wildcards in the operands, extract common prefixes to use $elemMatch
536        Map<String, List<FieldInfoDBObject>> propBaseKeyToDBOs = new LinkedHashMap<>();
537        Map<String, String> propBaseKeyToFieldBase = new HashMap<>();
538        for (Iterator<Object> it = list.iterator(); it.hasNext();) {
539            Object ob = it.next();
540            if (ob instanceof FieldInfoDBObject) {
541                FieldInfoDBObject fidbo = (FieldInfoDBObject) ob;
542                FieldInfo fieldInfo = fidbo.fieldInfo;
543                if (fieldInfo.hasWildcard) {
544                    if (fieldInfo.fieldSuffix != null && fieldInfo.fieldSuffix.contains("*")) {
545                        // a double wildcard of the form foo/*/bar/* is not a problem if bar is an array
546                        // TODO prevent deep complex multiple wildcards
547                        // throw new QueryParseException("Cannot use two wildcards: " + fieldInfo.prop);
548                    }
549                    // generate a key unique per correlation for this element match
550                    String wildcardNumber = fieldInfo.fieldWildcard;
551                    if (wildcardNumber.isEmpty()) {
552                        // negative to not collide with regular correlated wildcards
553                        wildcardNumber = String.valueOf(-counter.incrementAndGet());
554                    }
555                    String propBaseKey = fieldInfo.fieldPrefix + "/*" + wildcardNumber;
556                    // store object for this key
557                    List<FieldInfoDBObject> dbos = propBaseKeyToDBOs.get(propBaseKey);
558                    if (dbos == null) {
559                        propBaseKeyToDBOs.put(propBaseKey, dbos = new LinkedList<>());
560                    }
561                    dbos.add(fidbo);
562                    // remember for which field base this is
563                    String fieldBase = fieldInfo.fieldPrefix.replace("/", ".");
564                    propBaseKeyToFieldBase.put(propBaseKey, fieldBase);
565                    // remove from list, will be re-added later through propBaseKeyToDBOs
566                    it.remove();
567                }
568            }
569        }
570        // generate $elemMatch items for correlated queries
571        for (Entry<String, List<FieldInfoDBObject>> es : propBaseKeyToDBOs.entrySet()) {
572            String propBaseKey = es.getKey();
573            List<FieldInfoDBObject> fidbos = es.getValue();
574            if (fidbos.size() == 1) {
575                // regular uncorrelated match
576                list.addAll(fidbos);
577            } else {
578                DBObject elemMatch = new BasicDBObject();
579                for (FieldInfoDBObject fidbo : fidbos) {
580                    // truncate field name to just the suffix
581                    FieldInfo fieldInfo = fidbo.fieldInfo;
582                    Object value = fidbo.get(fieldInfo.queryField);
583                    String fieldSuffix = fieldInfo.fieldSuffix.replace("/", ".");
584                    if (elemMatch.containsField(fieldSuffix)) {
585                        // ecm:acl/*1/principal = 'bob' AND ecm:acl/*1/principal = 'steve'
586                        // cannot match
587                        // TODO do better
588                        value = "__NOSUCHVALUE__";
589                    }
590                    elemMatch.put(fieldSuffix, value);
591                }
592                String fieldBase = propBaseKeyToFieldBase.get(propBaseKey);
593                BasicDBObject dbo = new BasicDBObject(fieldBase,
594                        new BasicDBObject(QueryOperators.ELEM_MATCH, elemMatch));
595                list.add(dbo);
596            }
597        }
598        if (list.size() == 1) {
599            return (DBObject) list.get(0);
600        } else {
601            return new BasicDBObject(QueryOperators.AND, list);
602        }
603    }
604
605    public DBObject walkOr(Operand lvalue, Operand rvalue) {
606        Object left = walkOperand(lvalue);
607        Object right = walkOperand(rvalue);
608        List<Object> list = new ArrayList<>(Arrays.asList(left, right));
609        return new BasicDBObject(QueryOperators.OR, list);
610    }
611
612    protected Object checkBoolean(FieldInfo fieldInfo, Object right) {
613        if (fieldInfo.isBoolean()) {
614            // convert 0 / 1 to actual booleans
615            if (right instanceof Long) {
616                if (ZERO.equals(right)) {
617                    right = fieldInfo.isTrueOrNullBoolean ? null : FALSE;
618                } else if (ONE.equals(right)) {
619                    right = TRUE;
620                } else {
621                    throw new QueryParseException("Invalid boolean: " + right);
622                }
623            }
624        }
625        return right;
626    }
627
628    public DBObject walkEq(Operand lvalue, Operand rvalue) {
629        FieldInfo fieldInfo = walkReference(lvalue);
630        Object right = walkOperand(rvalue);
631        if (isMixinTypes(fieldInfo)) {
632            if (!(right instanceof String)) {
633                throw new QueryParseException("Invalid EQ rhs: " + rvalue);
634            }
635            return walkMixinTypes(Collections.singletonList((String) right), true);
636        }
637        right = checkBoolean(fieldInfo, right);
638        // TODO check list fields
639        return new FieldInfoDBObject(fieldInfo, right);
640    }
641
642    public DBObject walkNotEq(Operand lvalue, Operand rvalue) {
643        FieldInfo fieldInfo = walkReference(lvalue);
644        Object right = walkOperand(rvalue);
645        if (isMixinTypes(fieldInfo)) {
646            if (!(right instanceof String)) {
647                throw new QueryParseException("Invalid NE rhs: " + rvalue);
648            }
649            return walkMixinTypes(Collections.singletonList((String) right), false);
650        }
651        right = checkBoolean(fieldInfo, right);
652        // TODO check list fields
653        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, right));
654    }
655
656    public DBObject walkLt(Operand lvalue, Operand rvalue) {
657        FieldInfo fieldInfo = walkReference(lvalue);
658        Object right = walkOperand(rvalue);
659        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, right));
660    }
661
662    public DBObject walkGt(Operand lvalue, Operand rvalue) {
663        FieldInfo fieldInfo = walkReference(lvalue);
664        Object right = walkOperand(rvalue);
665        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right));
666    }
667
668    public DBObject walkLtEq(Operand lvalue, Operand rvalue) {
669        FieldInfo fieldInfo = walkReference(lvalue);
670        Object right = walkOperand(rvalue);
671        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LTE, right));
672    }
673
674    public DBObject walkGtEq(Operand lvalue, Operand rvalue) {
675        FieldInfo fieldInfo = walkReference(lvalue);
676        Object right = walkOperand(rvalue);
677        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GTE, right));
678    }
679
680    public DBObject walkBetween(Operand lvalue, Operand rvalue, boolean positive) {
681        LiteralList l = (LiteralList) rvalue;
682        FieldInfo fieldInfo = walkReference(lvalue);
683        Object left = walkOperand(l.get(0));
684        Object right = walkOperand(l.get(1));
685        if (positive) {
686            DBObject range = new BasicDBObject();
687            range.put(QueryOperators.GTE, left);
688            range.put(QueryOperators.LTE, right);
689            return new FieldInfoDBObject(fieldInfo, range);
690        } else {
691            DBObject a = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, left));
692            DBObject b = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right));
693            return new BasicDBObject(QueryOperators.OR, Arrays.asList(a, b));
694        }
695    }
696
697    public DBObject walkIn(Operand lvalue, Operand rvalue, boolean positive) {
698        FieldInfo fieldInfo = walkReference(lvalue);
699        Object right = walkOperand(rvalue);
700        if (!(right instanceof List)) {
701            throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue);
702        }
703        if (isMixinTypes(fieldInfo)) {
704            return walkMixinTypes((List<String>) right, positive);
705        }
706        // TODO check list fields
707        List<Object> list = (List<Object>) right;
708        return new FieldInfoDBObject(fieldInfo,
709                new BasicDBObject(positive ? QueryOperators.IN : QueryOperators.NIN, list));
710    }
711
712    public DBObject walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) {
713        FieldInfo fieldInfo = walkReference(lvalue);
714        if (!(rvalue instanceof StringLiteral)) {
715            throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue);
716        }
717        // TODO check list fields
718        String like = walkStringLiteral((StringLiteral) rvalue);
719        String regex = ExpressionEvaluator.likeToRegex(like);
720
721        int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0;
722        Pattern pattern = Pattern.compile(regex, flags);
723        Object value;
724        if (positive) {
725            value = pattern;
726        } else {
727            value = new BasicDBObject(QueryOperators.NOT, pattern);
728        }
729        return new FieldInfoDBObject(fieldInfo, value);
730    }
731
732    public Object walkOperand(Operand op) {
733        if (op instanceof Literal) {
734            return walkLiteral((Literal) op);
735        } else if (op instanceof LiteralList) {
736            return walkLiteralList((LiteralList) op);
737        } else if (op instanceof Function) {
738            return walkFunction((Function) op);
739        } else if (op instanceof Expression) {
740            return walkExpression((Expression) op);
741        } else if (op instanceof Reference) {
742            return walkReference((Reference) op);
743        } else {
744            throw new QueryParseException("Unknown operand: " + op);
745        }
746    }
747
748    public Object walkLiteral(Literal lit) {
749        if (lit instanceof BooleanLiteral) {
750            return walkBooleanLiteral((BooleanLiteral) lit);
751        } else if (lit instanceof DateLiteral) {
752            return walkDateLiteral((DateLiteral) lit);
753        } else if (lit instanceof DoubleLiteral) {
754            return walkDoubleLiteral((DoubleLiteral) lit);
755        } else if (lit instanceof IntegerLiteral) {
756            return walkIntegerLiteral((IntegerLiteral) lit);
757        } else if (lit instanceof StringLiteral) {
758            return walkStringLiteral((StringLiteral) lit);
759        } else {
760            throw new QueryParseException("Unknown literal: " + lit);
761        }
762    }
763
764    public Object walkBooleanLiteral(BooleanLiteral lit) {
765        return Boolean.valueOf(lit.value);
766    }
767
768    public Date walkDateLiteral(DateLiteral lit) {
769        return lit.value.toDate(); // TODO onlyDate
770    }
771
772    public Double walkDoubleLiteral(DoubleLiteral lit) {
773        return Double.valueOf(lit.value);
774    }
775
776    public Long walkIntegerLiteral(IntegerLiteral lit) {
777        return Long.valueOf(lit.value);
778    }
779
780    public String walkStringLiteral(StringLiteral lit) {
781        return lit.value;
782    }
783
784    public List<Object> walkLiteralList(LiteralList litList) {
785        List<Object> list = new ArrayList<Object>(litList.size());
786        for (Literal lit : litList) {
787            list.add(walkLiteral(lit));
788        }
789        return list;
790    }
791
792    protected List<Object> walkOperandList(List<Operand> values) {
793        List<Object> list = new LinkedList<>();
794        for (Operand value : values) {
795            list.add(walkOperand(value));
796        }
797        return list;
798    }
799
800    public Object walkFunction(Function func) {
801        throw new UnsupportedOperationException(func.name);
802    }
803
804    public DBObject walkStartsWith(Operand lvalue, Operand rvalue) {
805        if (!(lvalue instanceof Reference)) {
806            throw new QueryParseException("Invalid STARTSWITH query, left hand side must be a property: " + lvalue);
807        }
808        String name = ((Reference) lvalue).name;
809        if (!(rvalue instanceof StringLiteral)) {
810            throw new QueryParseException(
811                    "Invalid STARTSWITH query, right hand side must be a literal path: " + rvalue);
812        }
813        String path = ((StringLiteral) rvalue).value;
814        if (path.length() > 1 && path.endsWith("/")) {
815            path = path.substring(0, path.length() - 1);
816        }
817
818        if (NXQL.ECM_PATH.equals(name)) {
819            return walkStartsWithPath(path);
820        } else {
821            return walkStartsWithNonPath(lvalue, path);
822        }
823    }
824
825    protected DBObject walkStartsWithPath(String path) {
826        // resolve path
827        String ancestorId = pathResolver.getIdForPath(path);
828        if (ancestorId == null) {
829            // no such path
830            // TODO XXX do better
831            return new BasicDBObject(MONGODB_ID, "__nosuchid__");
832        }
833        return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
834    }
835
836    protected DBObject walkStartsWithNonPath(Operand lvalue, String path) {
837        FieldInfo fieldInfo = walkReference(lvalue);
838        DBObject eq = new FieldInfoDBObject(fieldInfo, path);
839        // escape except alphanumeric and others not needing escaping
840        String regex = path.replaceAll("([^a-zA-Z0-9 /])", "\\\\$1");
841        Pattern pattern = Pattern.compile(regex + "/.*");
842        DBObject like = new FieldInfoDBObject(fieldInfo, pattern);
843        return new BasicDBObject(QueryOperators.OR, Arrays.asList(eq, like));
844    }
845
846    protected FieldInfo walkReference(Operand value) {
847        if (!(value instanceof Reference)) {
848            throw new QueryParseException("Invalid query, left hand side must be a property: " + value);
849        }
850        return walkReference((Reference) value);
851    }
852
853    // non-canonical index syntax, for replaceAll
854    protected final static Pattern NON_CANON_INDEX = Pattern.compile("[^/\\[\\]]+" // name
855            + "\\[(\\d+|\\*|\\*\\d+)\\]" // index in brackets
856    );
857
858    /**
859     * Canonicalizes a Nuxeo-xpath.
860     * <p>
861     * Replaces {@code a/foo[123]/b} with {@code a/123/b}
862     * <p>
863     * A star or a star followed by digits can be used instead of just the digits as well.
864     *
865     * @param xpath the xpath
866     * @return the canonicalized xpath.
867     */
868    public static String canonicalXPath(String xpath) {
869        while (xpath.length() > 0 && xpath.charAt(0) == '/') {
870            xpath = xpath.substring(1);
871        }
872        if (xpath.indexOf('[') == -1) {
873            return xpath;
874        } else {
875            return NON_CANON_INDEX.matcher(xpath).replaceAll("$1");
876        }
877    }
878
879    /** Splits foo.*.bar into foo, *, bar and split foo.*1.bar into foo, *1, bar with the last bar part optional */
880    protected final static Pattern WILDCARD_SPLIT = Pattern.compile("([^*]*)\\.\\*(\\d*)(?:\\.(.*))?");
881
882    protected static class FieldInfo {
883
884        /** NXQL property. */
885        protected final String prop;
886
887        /** MongoDB field including wildcards (not used as-is). */
888        protected final String fullField;
889
890        /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */
891        protected final String queryField;
892
893        /** MongoDB field for projection. */
894        protected final String projectionField;
895
896        protected final Type type;
897
898        /**
899         * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly.
900         */
901        protected final boolean isTrueOrNullBoolean;
902
903        protected final boolean hasWildcard;
904
905        /** Prefix before the wildcard. */
906        protected final String fieldPrefix;
907
908        /** Wildcard part after * */
909        protected final String fieldWildcard;
910
911        /** Part after wildcard, may be null. */
912        protected final String fieldSuffix;
913
914        protected FieldInfo(String prop, String fullField, String queryField, String projectionField, Type type,
915                boolean isTrueOrNullBoolean) {
916            this.prop = prop;
917            this.fullField = fullField;
918            this.queryField = queryField;
919            this.projectionField = projectionField;
920            this.type = type;
921            this.isTrueOrNullBoolean = isTrueOrNullBoolean;
922            Matcher m = WILDCARD_SPLIT.matcher(fullField);
923            if (m.matches()) {
924                hasWildcard = true;
925                fieldPrefix = m.group(1);
926                fieldWildcard = m.group(2);
927                fieldSuffix = m.group(3);
928            } else {
929                hasWildcard = false;
930                fieldPrefix = fieldWildcard = fieldSuffix = null;
931            }
932        }
933
934        protected boolean isBoolean() {
935            return type instanceof BooleanType;
936        }
937    }
938
939    protected static class FieldInfoDBObject extends BasicDBObject {
940
941        private static final long serialVersionUID = 1L;
942
943        protected FieldInfo fieldInfo;
944
945        public FieldInfoDBObject(FieldInfo fieldInfo, Object value) {
946            super(fieldInfo.queryField, value);
947            this.fieldInfo = fieldInfo;
948        }
949    }
950
951    /**
952     * Returns the MongoDB field for this reference.
953     */
954    public FieldInfo walkReference(Reference ref) {
955        FieldInfo fieldInfo = walkReference(ref.name);
956        if (DATE_CAST.equals(ref.cast)) {
957            Type type = fieldInfo.type;
958            if (!(type instanceof DateType
959                    || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) {
960                throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name);
961            }
962            // fieldInfo.isDateCast = true;
963        }
964        return fieldInfo;
965    }
966
967    protected FieldInfo walkReference(String name) {
968        String prop = canonicalXPath(name);
969        String[] parts = prop.split("/");
970        if (prop.startsWith(NXQL.ECM_PREFIX)) {
971            if (prop.startsWith(NXQL.ECM_ACL + "/")) {
972                return parseACP(prop, parts);
973            }
974            // simple field
975            String field = DBSSession.convToInternal(prop);
976            Type type = DBSSession.getType(field);
977            return new FieldInfo(prop, field, field, field, type, true);
978        } else {
979            String first = parts[0];
980            Field field = schemaManager.getField(first);
981            if (field == null) {
982                if (first.indexOf(':') > -1) {
983                    throw new QueryParseException("No such property: " + name);
984                }
985                // check without prefix
986                // TODO precompute this in SchemaManagerImpl
987                for (Schema schema : schemaManager.getSchemas()) {
988                    if (!StringUtils.isBlank(schema.getNamespace().prefix)) {
989                        // schema with prefix, do not consider as candidate
990                        continue;
991                    }
992                    if (schema != null) {
993                        field = schema.getField(first);
994                        if (field != null) {
995                            break;
996                        }
997                    }
998                }
999                if (field == null) {
1000                    throw new QueryParseException("No such property: " + name);
1001                }
1002            }
1003            Type type = field.getType();
1004            // canonical name
1005            parts[0] = field.getName().getPrefixedName();
1006            // are there wildcards or list indexes?
1007            List<String> queryFieldParts = new LinkedList<>(); // field for query
1008            List<String> projectionFieldParts = new LinkedList<>(); // field for projection
1009            boolean firstPart = true;
1010            for (String part : parts) {
1011                if (NumberUtils.isDigits(part)) {
1012                    // explicit list index
1013                    queryFieldParts.add(part);
1014                    type = ((ListType) type).getFieldType();
1015                } else if (!part.startsWith("*")) {
1016                    // complex sub-property
1017                    queryFieldParts.add(part);
1018                    projectionFieldParts.add(part);
1019                    if (!firstPart) {
1020                        // we already computed the type of the first part
1021                        field = ((ComplexType) type).getField(part);
1022                        if (field == null) {
1023                            throw new QueryParseException("No such property: " + name);
1024                        }
1025                        type = field.getType();
1026                    }
1027                } else {
1028                    // wildcard
1029                    type = ((ListType) type).getFieldType();
1030                }
1031                firstPart = false;
1032            }
1033            String fullField = StringUtils.join(parts, '.');
1034            String queryField = StringUtils.join(queryFieldParts, '.');
1035            String projectionField = StringUtils.join(projectionFieldParts, '.');
1036            return new FieldInfo(prop, fullField, queryField, projectionField, type, false);
1037        }
1038    }
1039
1040    protected FieldInfo parseACP(String prop, String[] parts) {
1041        if (parts.length != 3) {
1042            throw new QueryParseException("No such property: " + prop);
1043        }
1044        String wildcard = parts[1];
1045        if (NumberUtils.isDigits(wildcard)) {
1046            throw new QueryParseException("Cannot use explicit index in ACLs: " + prop);
1047        }
1048        String last = parts[2];
1049        String fullField;
1050        String queryField;
1051        String projectionField;
1052        if (NXQL.ECM_ACL_NAME.equals(last)) {
1053            fullField = KEY_ACP + "." + KEY_ACL_NAME;
1054            queryField = KEY_ACP + "." + KEY_ACL_NAME;
1055            // TODO remember wildcard correlation
1056        } else {
1057            String fieldLast = DBSSession.convToInternalAce(last);
1058            if (fieldLast == null) {
1059                throw new QueryParseException("No such property: " + prop);
1060            }
1061            fullField = KEY_ACP + "." + KEY_ACL + "." + wildcard + "." + fieldLast;
1062            queryField = KEY_ACP + "." + KEY_ACL + "." + fieldLast;
1063        }
1064        Type type = DBSSession.getType(last);
1065        projectionField = queryField;
1066        return new FieldInfo(prop, fullField, queryField, projectionField, type, false);
1067    }
1068
1069    protected boolean isMixinTypes(FieldInfo fieldInfo) {
1070        return fieldInfo.queryField.equals(DBSDocument.KEY_MIXIN_TYPES);
1071    }
1072
1073    protected Set<String> getMixinDocumentTypes(String mixin) {
1074        Set<String> types = schemaManager.getDocumentTypeNamesForFacet(mixin);
1075        return types == null ? Collections.emptySet() : types;
1076    }
1077
1078    protected List<String> getDocumentTypes() {
1079        // TODO precompute in SchemaManager
1080        if (documentTypes == null) {
1081            documentTypes = new ArrayList<>();
1082            for (DocumentType docType : schemaManager.getDocumentTypes()) {
1083                documentTypes.add(docType.getName());
1084            }
1085        }
1086        return documentTypes;
1087    }
1088
1089    protected boolean isNeverPerInstanceMixin(String mixin) {
1090        return schemaManager.getNoPerDocumentQueryFacets().contains(mixin);
1091    }
1092
1093    /**
1094     * Matches the mixin types against a list of values.
1095     * <p>
1096     * Used for:
1097     * <ul>
1098     * <li>ecm:mixinTypes = 'Foo'
1099     * <li>ecm:mixinTypes != 'Foo'
1100     * <li>ecm:mixinTypes IN ('Foo', 'Bar')
1101     * <li>ecm:mixinTypes NOT IN ('Foo', 'Bar')
1102     * </ul>
1103     * <p>
1104     * ecm:mixinTypes IN ('Foo', 'Bar')
1105     *
1106     * <pre>
1107     * { "$or" : [ { "ecm:primaryType" : { "$in" : [ ... types with Foo or Bar ...]}} ,
1108     *             { "ecm:mixinTypes" : { "$in" : [ "Foo" , "Bar]}}]}
1109     * </pre>
1110     *
1111     * ecm:mixinTypes NOT IN ('Foo', 'Bar')
1112     * <p>
1113     *
1114     * <pre>
1115     * { "$and" : [ { "ecm:primaryType" : { "$in" : [ ... types without Foo nor Bar ...]}} ,
1116     *              { "ecm:mixinTypes" : { "$nin" : [ "Foo" , "Bar]}}]}
1117     * </pre>
1118     */
1119    public DBObject walkMixinTypes(List<String> mixins, boolean include) {
1120        /*
1121         * Primary types that match.
1122         */
1123        Set<String> matchPrimaryTypes;
1124        if (include) {
1125            matchPrimaryTypes = new HashSet<String>();
1126            for (String mixin : mixins) {
1127                matchPrimaryTypes.addAll(getMixinDocumentTypes(mixin));
1128            }
1129        } else {
1130            matchPrimaryTypes = new HashSet<String>(getDocumentTypes());
1131            for (String mixin : mixins) {
1132                matchPrimaryTypes.removeAll(getMixinDocumentTypes(mixin));
1133            }
1134        }
1135        /*
1136         * Instance mixins that match.
1137         */
1138        Set<String> matchMixinTypes = new HashSet<String>();
1139        for (String mixin : mixins) {
1140            if (!isNeverPerInstanceMixin(mixin)) {
1141                matchMixinTypes.add(mixin);
1142            }
1143        }
1144        /*
1145         * MongoDB query generation.
1146         */
1147        // match on primary type
1148        DBObject p = new BasicDBObject(DBSDocument.KEY_PRIMARY_TYPE,
1149                new BasicDBObject(QueryOperators.IN, matchPrimaryTypes));
1150        // match on mixin types
1151        // $in/$nin with an array matches if any/no element of the array matches
1152        String innin = include ? QueryOperators.IN : QueryOperators.NIN;
1153        DBObject m = new BasicDBObject(DBSDocument.KEY_MIXIN_TYPES, new BasicDBObject(innin, matchMixinTypes));
1154        // and/or between those
1155        String op = include ? QueryOperators.OR : QueryOperators.AND;
1156        return new BasicDBObject(op, Arrays.asList(p, m));
1157    }
1158
1159}