Source code

001/*
002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_GRANT;
024import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
025import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL_NAME;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SCORE;
028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ID;
029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_NAME;
030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PARENT_ID;
031import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID;
032import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_META;
033import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_TEXT_SCORE;
034
035import java.util.ArrayList;
036import java.util.Arrays;
037import java.util.Collections;
038import java.util.Date;
039import java.util.HashMap;
040import java.util.HashSet;
041import java.util.Iterator;
042import java.util.LinkedHashMap;
043import java.util.LinkedList;
044import java.util.List;
045import java.util.Map;
046import java.util.Map.Entry;
047import java.util.Set;
048import java.util.concurrent.atomic.AtomicInteger;
049import java.util.regex.Matcher;
050import java.util.regex.Pattern;
051
052import org.apache.commons.lang.StringUtils;
053import org.apache.commons.lang.math.NumberUtils;
054import org.nuxeo.ecm.core.query.QueryParseException;
055import org.nuxeo.ecm.core.query.sql.NXQL;
056import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral;
057import org.nuxeo.ecm.core.query.sql.model.DateLiteral;
058import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral;
059import org.nuxeo.ecm.core.query.sql.model.Expression;
060import org.nuxeo.ecm.core.query.sql.model.Function;
061import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral;
062import org.nuxeo.ecm.core.query.sql.model.Literal;
063import org.nuxeo.ecm.core.query.sql.model.LiteralList;
064import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
065import org.nuxeo.ecm.core.query.sql.model.Operand;
066import org.nuxeo.ecm.core.query.sql.model.Operator;
067import org.nuxeo.ecm.core.query.sql.model.OrderByClause;
068import org.nuxeo.ecm.core.query.sql.model.OrderByExpr;
069import org.nuxeo.ecm.core.query.sql.model.Reference;
070import org.nuxeo.ecm.core.query.sql.model.SelectClause;
071import org.nuxeo.ecm.core.query.sql.model.SelectList;
072import org.nuxeo.ecm.core.query.sql.model.StringLiteral;
073import org.nuxeo.ecm.core.schema.DocumentType;
074import org.nuxeo.ecm.core.schema.SchemaManager;
075import org.nuxeo.ecm.core.schema.types.ComplexType;
076import org.nuxeo.ecm.core.schema.types.Field;
077import org.nuxeo.ecm.core.schema.types.ListType;
078import org.nuxeo.ecm.core.schema.types.Schema;
079import org.nuxeo.ecm.core.schema.types.Type;
080import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
081import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
082import org.nuxeo.ecm.core.storage.ExpressionEvaluator.PathResolver;
083import org.nuxeo.ecm.core.storage.dbs.DBSDocument;
084import org.nuxeo.ecm.core.storage.dbs.DBSSession;
085import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer;
086import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.FulltextQuery;
087import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.Op;
088import org.nuxeo.runtime.api.Framework;
089
090import com.mongodb.BasicDBObject;
091import com.mongodb.DBObject;
092import com.mongodb.QueryOperators;
093
094/**
095 * Query builder for a MongoDB query from an {@link Expression}.
096 *
097 * @since 5.9.4
098 */
099public class MongoDBQueryBuilder {
100
101    private static final Long ZERO = Long.valueOf(0);
102
103    private static final Long ONE = Long.valueOf(1);
104
105    private static final Long MINUS_ONE = Long.valueOf(-1);
106
107    protected final AtomicInteger counter = new AtomicInteger();
108
109    protected final SchemaManager schemaManager;
110
111    protected List<String> documentTypes;
112
113    protected final Expression expression;
114
115    protected final SelectClause selectClause;
116
117    protected final OrderByClause orderByClause;
118
119    protected final PathResolver pathResolver;
120
121    public boolean hasFulltext;
122
123    public boolean sortOnFulltextScore;
124
125    protected DBObject query;
126
127    protected DBObject orderBy;
128
129    protected DBObject projection;
130
131    boolean projectionHasWildcard;
132
133    public MongoDBQueryBuilder(Expression expression, SelectClause selectClause, OrderByClause orderByClause,
134            PathResolver pathResolver) {
135        schemaManager = Framework.getLocalService(SchemaManager.class);
136        this.expression = expression;
137        this.selectClause = selectClause;
138        this.orderByClause = orderByClause;
139        this.pathResolver = pathResolver;
140    }
141
142    public void walk() {
143        query = walkExpression(expression); // computes hasFulltext
144        walkOrderBy(); // computes sortOnFulltextScore
145        walkProjection(); // needs hasFulltext and sortOnFulltextScore
146    }
147
148    public DBObject getQuery() {
149        return query;
150    }
151
152    public DBObject getOrderBy() {
153        return orderBy;
154    }
155
156    public DBObject getProjection() {
157        return projection;
158    }
159
160    public boolean hasProjectionWildcard() {
161        return projectionHasWildcard;
162    }
163
164    protected void walkOrderBy() {
165        sortOnFulltextScore = false;
166        if (orderByClause == null) {
167            orderBy = null;
168        } else {
169            orderBy = new BasicDBObject();
170            for (OrderByExpr ob : orderByClause.elements) {
171                Reference ref = ob.reference;
172                boolean desc = ob.isDescending;
173                String field = walkReference(ref).queryField;
174                if (!orderBy.containsField(field)) {
175                    Object value;
176                    if (KEY_FULLTEXT_SCORE.equals(field)) {
177                        if (!desc) {
178                            throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " ascending");
179                        }
180                        sortOnFulltextScore = true;
181                        value = new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE);
182                    } else {
183                        value = desc ? MINUS_ONE : ONE;
184                    }
185                    orderBy.put(field, value);
186                }
187            }
188            if (sortOnFulltextScore && ((BasicDBObject) orderBy).size() > 1) {
189                throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " and other criteria");
190            }
191        }
192    }
193
194    protected void walkProjection() {
195        projection = new BasicDBObject();
196        projection.put(KEY_ID, ONE); // always useful
197        projection.put(KEY_NAME, ONE); // used in ORDER BY ecm:path
198        projection.put(KEY_PARENT_ID, ONE); // used in ORDER BY ecm:path
199        boolean projectionOnFulltextScore = false;
200        for (int i = 0; i < selectClause.elements.size(); i++) {
201            Operand op = selectClause.elements.get(i);
202            if (!(op instanceof Reference)) {
203                throw new QueryParseException("Projection not supported: " + op);
204            }
205            FieldInfo fieldInfo = walkReference((Reference) op);
206            projection.put(fieldInfo.projectionField, ONE);
207            if (fieldInfo.hasWildcard) {
208                projectionHasWildcard = true;
209            }
210            if (fieldInfo.projectionField.equals(KEY_FULLTEXT_SCORE)) {
211                projectionOnFulltextScore = true;
212            }
213        }
214        if (projectionOnFulltextScore || sortOnFulltextScore) {
215            if (!hasFulltext) {
216                throw new QueryParseException(NXQL.ECM_FULLTEXT_SCORE + " cannot be used without " + NXQL.ECM_FULLTEXT);
217            }
218            projection.put(KEY_FULLTEXT_SCORE, new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE));
219        }
220    }
221
222    public DBObject walkExpression(Expression expr) {
223        Operator op = expr.operator;
224        Operand lvalue = expr.lvalue;
225        Operand rvalue = expr.rvalue;
226        String name = lvalue instanceof Reference ? ((Reference) lvalue).name : null;
227        if (op == Operator.STARTSWITH) {
228            return walkStartsWith(lvalue, rvalue);
229        } else if (NXQL.ECM_PATH.equals(name)) {
230            return walkEcmPath(op, rvalue);
231        } else if (NXQL.ECM_ANCESTORID.equals(name)) {
232            return walkAncestorId(op, rvalue);
233        } else if (name != null && name.startsWith(NXQL.ECM_FULLTEXT) && !NXQL.ECM_FULLTEXT_JOBID.equals(name)) {
234            return walkEcmFulltext(name, op, rvalue);
235        } else if (op == Operator.SUM) {
236            throw new UnsupportedOperationException("SUM");
237        } else if (op == Operator.SUB) {
238            throw new UnsupportedOperationException("SUB");
239        } else if (op == Operator.MUL) {
240            throw new UnsupportedOperationException("MUL");
241        } else if (op == Operator.DIV) {
242            throw new UnsupportedOperationException("DIV");
243        } else if (op == Operator.LT) {
244            return walkLt(lvalue, rvalue);
245        } else if (op == Operator.GT) {
246            return walkGt(lvalue, rvalue);
247        } else if (op == Operator.EQ) {
248            return walkEq(lvalue, rvalue);
249        } else if (op == Operator.NOTEQ) {
250            return walkNotEq(lvalue, rvalue);
251        } else if (op == Operator.LTEQ) {
252            return walkLtEq(lvalue, rvalue);
253        } else if (op == Operator.GTEQ) {
254            return walkGtEq(lvalue, rvalue);
255        } else if (op == Operator.AND) {
256            if (expr instanceof MultiExpression) {
257                return walkMultiExpression((MultiExpression) expr);
258            } else {
259                return walkAnd(lvalue, rvalue);
260            }
261        } else if (op == Operator.NOT) {
262            return walkNot(lvalue);
263        } else if (op == Operator.OR) {
264            return walkOr(lvalue, rvalue);
265        } else if (op == Operator.LIKE) {
266            return walkLike(lvalue, rvalue, true, false);
267        } else if (op == Operator.ILIKE) {
268            return walkLike(lvalue, rvalue, true, true);
269        } else if (op == Operator.NOTLIKE) {
270            return walkLike(lvalue, rvalue, false, false);
271        } else if (op == Operator.NOTILIKE) {
272            return walkLike(lvalue, rvalue, false, true);
273        } else if (op == Operator.IN) {
274            return walkIn(lvalue, rvalue, true);
275        } else if (op == Operator.NOTIN) {
276            return walkIn(lvalue, rvalue, false);
277        } else if (op == Operator.ISNULL) {
278            return walkIsNull(lvalue);
279        } else if (op == Operator.ISNOTNULL) {
280            return walkIsNotNull(lvalue);
281        } else if (op == Operator.BETWEEN) {
282            return walkBetween(lvalue, rvalue, true);
283        } else if (op == Operator.NOTBETWEEN) {
284            return walkBetween(lvalue, rvalue, false);
285        } else {
286            throw new QueryParseException("Unknown operator: " + op);
287        }
288    }
289
290    protected DBObject walkEcmPath(Operator op, Operand rvalue) {
291        if (op != Operator.EQ && op != Operator.NOTEQ) {
292            throw new QueryParseException(NXQL.ECM_PATH + " requires = or <> operator");
293        }
294        if (!(rvalue instanceof StringLiteral)) {
295            throw new QueryParseException(NXQL.ECM_PATH + " requires literal path as right argument");
296        }
297        String path = ((StringLiteral) rvalue).value;
298        if (path.length() > 1 && path.endsWith("/")) {
299            path = path.substring(0, path.length() - 1);
300        }
301        String id = pathResolver.getIdForPath(path);
302        if (id == null) {
303            // no such path
304            // TODO XXX do better
305            return new BasicDBObject(MONGODB_ID, "__nosuchid__");
306        }
307        if (op == Operator.EQ) {
308            return new BasicDBObject(DBSDocument.KEY_ID, id);
309        } else {
310            return new BasicDBObject(DBSDocument.KEY_ID, new BasicDBObject(QueryOperators.NE, id));
311        }
312    }
313
314    protected DBObject walkAncestorId(Operator op, Operand rvalue) {
315        if (op != Operator.EQ && op != Operator.NOTEQ) {
316            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires = or <> operator");
317        }
318        if (!(rvalue instanceof StringLiteral)) {
319            throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires literal id as right argument");
320        }
321        String ancestorId = ((StringLiteral) rvalue).value;
322        if (op == Operator.EQ) {
323            return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
324        } else {
325            return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, new BasicDBObject(QueryOperators.NE, ancestorId));
326        }
327    }
328
329    protected DBObject walkEcmFulltext(String name, Operator op, Operand rvalue) {
330        if (op != Operator.EQ && op != Operator.LIKE) {
331            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires = or LIKE operator");
332        }
333        if (!(rvalue instanceof StringLiteral)) {
334            throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires literal string as right argument");
335        }
336        String fulltextQuery = ((StringLiteral) rvalue).value;
337        if (name.equals(NXQL.ECM_FULLTEXT)) {
338            // standard fulltext query
339            hasFulltext = true;
340            String ft = getMongoDBFulltextQuery(fulltextQuery);
341            if (ft == null) {
342                // empty query, matches nothing
343                return new BasicDBObject(MONGODB_ID, "__nosuchid__");
344            }
345            DBObject textSearch = new BasicDBObject();
346            textSearch.put(QueryOperators.SEARCH, ft);
347            // TODO language?
348            return new BasicDBObject(QueryOperators.TEXT, textSearch);
349        } else {
350            // secondary index match with explicit field
351            // do a regexp on the field
352            if (name.charAt(NXQL.ECM_FULLTEXT.length()) != '.') {
353                throw new QueryParseException(name + " has incorrect syntax" + " for a secondary fulltext index");
354            }
355            String prop = name.substring(NXQL.ECM_FULLTEXT.length() + 1);
356            String ft = fulltextQuery.replace(" ", "%");
357            rvalue = new StringLiteral(ft);
358            return walkLike(new Reference(prop), rvalue, true, true);
359        }
360    }
361
362    // public static for tests
363    public static String getMongoDBFulltextQuery(String query) {
364        FulltextQuery ft = FulltextQueryAnalyzer.analyzeFulltextQuery(query);
365        if (ft == null) {
366            return null;
367        }
368        // translate into MongoDB syntax
369        return translateFulltext(ft, false);
370    }
371
372    /**
373     * Transforms the NXQL fulltext syntax into MongoDB syntax.
374     * <p>
375     * The MongoDB fulltext query syntax is badly documented, but is actually the following:
376     * <ul>
377     * <li>a term is a word,
378     * <li>a phrase is a set of spaced-separated words enclosed in double quotes,
379     * <li>negation is done by prepending a -,
380     * <li>the query is a space-separated set of terms, negated terms, phrases, or negated phrases.
381     * <li>all the words of non-negated phrases are also added to the terms.
382     * </ul>
383     * <p>
384     * The matching algorithm is (excluding stemming and stop words):
385     * <ul>
386     * <li>filter out documents with the negative terms, the negative phrases, or missing the phrases,
387     * <li>then if any term is present in the document then it's a match.
388     * </ul>
389     */
390    protected static String translateFulltext(FulltextQuery ft, boolean and) {
391        List<String> buf = new ArrayList<>();
392        translateFulltext(ft, buf, and);
393        return StringUtils.join(buf, ' ');
394    }
395
396    protected static void translateFulltext(FulltextQuery ft, List<String> buf, boolean and) {
397        if (ft.op == Op.OR) {
398            for (FulltextQuery term : ft.terms) {
399                // don't quote words for OR
400                translateFulltext(term, buf, false);
401            }
402        } else if (ft.op == Op.AND) {
403            for (FulltextQuery term : ft.terms) {
404                // quote words for AND
405                translateFulltext(term, buf, true);
406            }
407        } else {
408            String neg;
409            if (ft.op == Op.NOTWORD) {
410                neg = "-";
411            } else { // Op.WORD
412                neg = "";
413            }
414            String word = ft.word.toLowerCase();
415            if (ft.isPhrase() || and) {
416                buf.add(neg + '"' + word + '"');
417            } else {
418                buf.add(neg + word);
419            }
420        }
421    }
422
423    public DBObject walkNot(Operand value) {
424        Object val = walkOperand(value);
425        Object not = pushDownNot(val);
426        if (!(not instanceof DBObject)) {
427            throw new QueryParseException("Cannot do NOT on: " + val);
428        }
429        return (DBObject) not;
430    }
431
432    protected Object pushDownNot(Object object) {
433        if (!(object instanceof DBObject)) {
434            throw new QueryParseException("Cannot do NOT on: " + object);
435        }
436        DBObject ob = (DBObject) object;
437        Set<String> keySet = ob.keySet();
438        if (keySet.size() != 1) {
439            throw new QueryParseException("Cannot do NOT on: " + ob);
440        }
441        String key = keySet.iterator().next();
442        Object value = ob.get(key);
443        if (!key.startsWith("$")) {
444            if (value instanceof DBObject) {
445                // push down inside dbobject
446                return new BasicDBObject(key, pushDownNot(value));
447            } else {
448                // k = v -> k != v
449                return new BasicDBObject(key, new BasicDBObject(QueryOperators.NE, value));
450            }
451        }
452        if (QueryOperators.NE.equals(key)) {
453            // NOT k != v -> k = v
454            return value;
455        }
456        if (QueryOperators.NOT.equals(key)) {
457            // NOT NOT v -> v
458            return value;
459        }
460        if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) {
461            // boolean algebra
462            // NOT (v1 AND v2) -> NOT v1 OR NOT v2
463            // NOT (v1 OR v2) -> NOT v1 AND NOT v2
464            String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND;
465            List<Object> list = (List<Object>) value;
466            for (int i = 0; i < list.size(); i++) {
467                list.set(i, pushDownNot(list.get(i)));
468            }
469            return new BasicDBObject(op, list);
470        }
471        if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) {
472            // boolean algebra
473            // IN <-> NIN
474            String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN;
475            return new BasicDBObject(op, value);
476        }
477        if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key)
478                || QueryOperators.GTE.equals(key)) {
479            // TODO use inverse operators?
480            return new BasicDBObject(QueryOperators.NOT, ob);
481        }
482        throw new QueryParseException("Unknown operator for NOT: " + key);
483    }
484
485    public DBObject walkIsNull(Operand value) {
486        FieldInfo fieldInfo = walkReference(value);
487        return new FieldInfoDBObject(fieldInfo, null);
488    }
489
490    public DBObject walkIsNotNull(Operand value) {
491        FieldInfo fieldInfo = walkReference(value);
492        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, null));
493    }
494
495    public DBObject walkMultiExpression(MultiExpression expr) {
496        return walkAnd(expr.values);
497    }
498
499    public DBObject walkAnd(Operand lvalue, Operand rvalue) {
500        return walkAnd(Arrays.asList(lvalue, rvalue));
501    }
502
503    protected DBObject walkAnd(List<Operand> values) {
504        List<Object> list = walkOperandList(values);
505        // check wildcards in the operands, extract common prefixes to use $elemMatch
506        Map<String, List<FieldInfoDBObject>> propBaseKeyToDBOs = new LinkedHashMap<>();
507        Map<String, String> propBaseKeyToFieldBase = new HashMap<>();
508        for (Iterator<Object> it = list.iterator(); it.hasNext();) {
509            Object ob = it.next();
510            if (ob instanceof FieldInfoDBObject) {
511                FieldInfoDBObject fidbo = (FieldInfoDBObject) ob;
512                FieldInfo fieldInfo = fidbo.fieldInfo;
513                if (fieldInfo.hasWildcard) {
514                    if (fieldInfo.fieldSuffix.contains("*")) {
515                        // a double wildcard of the form foo/*/bar/* is not a problem if bar is an array
516                        // TODO prevent deep complex multiple wildcards
517                        // throw new QueryParseException("Cannot use two wildcards: " + fieldInfo.prop);
518                    }
519                    // generate a key unique per correlation for this element match
520                    String wildcardNumber = fieldInfo.fieldWildcard;
521                    if (wildcardNumber.isEmpty()) {
522                        // negative to not collide with regular correlated wildcards
523                        wildcardNumber = String.valueOf(-counter.incrementAndGet());
524                    }
525                    String propBaseKey = fieldInfo.fieldPrefix + "/*" + wildcardNumber;
526                    // store object for this key
527                    List<FieldInfoDBObject> dbos = propBaseKeyToDBOs.get(propBaseKey);
528                    if (dbos == null) {
529                        propBaseKeyToDBOs.put(propBaseKey, dbos = new LinkedList<>());
530                    }
531                    dbos.add(fidbo);
532                    // remember for which field base this is
533                    String fieldBase = fieldInfo.fieldPrefix.replace("/", ".");
534                    propBaseKeyToFieldBase.put(propBaseKey, fieldBase);
535                    // remove from list, will be re-added later through propBaseKeyToDBOs
536                    it.remove();
537                }
538            }
539        }
540        // generate $elemMatch items for correlated queries
541        for (Entry<String, List<FieldInfoDBObject>> es : propBaseKeyToDBOs.entrySet()) {
542            String propBaseKey = es.getKey();
543            List<FieldInfoDBObject> fidbos = es.getValue();
544            if (fidbos.size() == 1) {
545                // regular uncorrelated match
546                list.addAll(fidbos);
547            } else {
548                DBObject elemMatch = new BasicDBObject();
549                for (FieldInfoDBObject fidbo : fidbos) {
550                    // truncate field name to just the suffix
551                    FieldInfo fieldInfo = fidbo.fieldInfo;
552                    Object value = fidbo.get(fieldInfo.queryField);
553                    String fieldSuffix = fieldInfo.fieldSuffix.replace("/", ".");
554                    if (elemMatch.containsField(fieldSuffix)) {
555                        // ecm:acl/*1/principal = 'bob' AND ecm:acl/*1/principal = 'steve'
556                        // cannot match
557                        // TODO do better
558                        value = "__NOSUCHVALUE__";
559                    }
560                    elemMatch.put(fieldSuffix, value);
561                }
562                String fieldBase = propBaseKeyToFieldBase.get(propBaseKey);
563                BasicDBObject dbo = new BasicDBObject(fieldBase,
564                        new BasicDBObject(QueryOperators.ELEM_MATCH, elemMatch));
565                list.add(dbo);
566            }
567        }
568        if (list.size() == 1) {
569            return (DBObject) list.get(0);
570        } else {
571            return new BasicDBObject(QueryOperators.AND, list);
572        }
573    }
574
575    public DBObject walkOr(Operand lvalue, Operand rvalue) {
576        Object left = walkOperand(lvalue);
577        Object right = walkOperand(rvalue);
578        List<Object> list = new ArrayList<>(Arrays.asList(left, right));
579        return new BasicDBObject(QueryOperators.OR, list);
580    }
581
582    protected Object checkBoolean(FieldInfo fieldInfo, Object right) {
583        if (fieldInfo.isBoolean) {
584            // convert 0 / 1 to actual booleans
585            if (right instanceof Long) {
586                if (ZERO.equals(right)) {
587                    right = fieldInfo.isTrueOrNullBoolean ? null : FALSE;
588                } else if (ONE.equals(right)) {
589                    right = TRUE;
590                } else {
591                    throw new QueryParseException("Invalid boolean: " + right);
592                }
593            }
594        }
595        return right;
596    }
597
598    public DBObject walkEq(Operand lvalue, Operand rvalue) {
599        FieldInfo fieldInfo = walkReference(lvalue);
600        Object right = walkOperand(rvalue);
601        if (isMixinTypes(fieldInfo)) {
602            if (!(right instanceof String)) {
603                throw new QueryParseException("Invalid EQ rhs: " + rvalue);
604            }
605            return walkMixinTypes(Collections.singletonList((String) right), true);
606        }
607        right = checkBoolean(fieldInfo, right);
608        // TODO check list fields
609        return new FieldInfoDBObject(fieldInfo, right);
610    }
611
612    public DBObject walkNotEq(Operand lvalue, Operand rvalue) {
613        FieldInfo fieldInfo = walkReference(lvalue);
614        Object right = walkOperand(rvalue);
615        if (isMixinTypes(fieldInfo)) {
616            if (!(right instanceof String)) {
617                throw new QueryParseException("Invalid NE rhs: " + rvalue);
618            }
619            return walkMixinTypes(Collections.singletonList((String) right), false);
620        }
621        right = checkBoolean(fieldInfo, right);
622        // TODO check list fields
623        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, right));
624    }
625
626    public DBObject walkLt(Operand lvalue, Operand rvalue) {
627        FieldInfo fieldInfo = walkReference(lvalue);
628        Object right = walkOperand(rvalue);
629        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, right));
630    }
631
632    public DBObject walkGt(Operand lvalue, Operand rvalue) {
633        FieldInfo fieldInfo = walkReference(lvalue);
634        Object right = walkOperand(rvalue);
635        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right));
636    }
637
638    public DBObject walkLtEq(Operand lvalue, Operand rvalue) {
639        FieldInfo fieldInfo = walkReference(lvalue);
640        Object right = walkOperand(rvalue);
641        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LTE, right));
642    }
643
644    public DBObject walkGtEq(Operand lvalue, Operand rvalue) {
645        FieldInfo fieldInfo = walkReference(lvalue);
646        Object right = walkOperand(rvalue);
647        return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GTE, right));
648    }
649
650    public DBObject walkBetween(Operand lvalue, Operand rvalue, boolean positive) {
651        LiteralList l = (LiteralList) rvalue;
652        FieldInfo fieldInfo = walkReference(lvalue);
653        Object left = walkOperand(l.get(0));
654        Object right = walkOperand(l.get(1));
655        if (positive) {
656            DBObject range = new BasicDBObject();
657            range.put(QueryOperators.GTE, left);
658            range.put(QueryOperators.LTE, right);
659            return new FieldInfoDBObject(fieldInfo, range);
660        } else {
661            DBObject a = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, left));
662            DBObject b = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right));
663            return new BasicDBObject(QueryOperators.OR, Arrays.asList(a, b));
664        }
665    }
666
667    public DBObject walkIn(Operand lvalue, Operand rvalue, boolean positive) {
668        FieldInfo fieldInfo = walkReference(lvalue);
669        Object right = walkOperand(rvalue);
670        if (!(right instanceof List)) {
671            throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue);
672        }
673        if (isMixinTypes(fieldInfo)) {
674            return walkMixinTypes((List<String>) right, positive);
675        }
676        // TODO check list fields
677        List<Object> list = (List<Object>) right;
678        return new FieldInfoDBObject(fieldInfo,
679                new BasicDBObject(positive ? QueryOperators.IN : QueryOperators.NIN, list));
680    }
681
682    public DBObject walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) {
683        FieldInfo fieldInfo = walkReference(lvalue);
684        if (!(rvalue instanceof StringLiteral)) {
685            throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue);
686        }
687        // TODO check list fields
688        String like = walkStringLiteral((StringLiteral) rvalue);
689        String regex = ExpressionEvaluator.likeToRegex(like);
690
691        int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0;
692        Pattern pattern = Pattern.compile(regex, flags);
693        Object value;
694        if (positive) {
695            value = pattern;
696        } else {
697            value = new BasicDBObject(QueryOperators.NOT, pattern);
698        }
699        return new FieldInfoDBObject(fieldInfo, value);
700    }
701
702    public Object walkOperand(Operand op) {
703        if (op instanceof Literal) {
704            return walkLiteral((Literal) op);
705        } else if (op instanceof LiteralList) {
706            return walkLiteralList((LiteralList) op);
707        } else if (op instanceof Function) {
708            return walkFunction((Function) op);
709        } else if (op instanceof Expression) {
710            return walkExpression((Expression) op);
711        } else if (op instanceof Reference) {
712            return walkReference((Reference) op);
713        } else {
714            throw new QueryParseException("Unknown operand: " + op);
715        }
716    }
717
718    public Object walkLiteral(Literal lit) {
719        if (lit instanceof BooleanLiteral) {
720            return walkBooleanLiteral((BooleanLiteral) lit);
721        } else if (lit instanceof DateLiteral) {
722            return walkDateLiteral((DateLiteral) lit);
723        } else if (lit instanceof DoubleLiteral) {
724            return walkDoubleLiteral((DoubleLiteral) lit);
725        } else if (lit instanceof IntegerLiteral) {
726            return walkIntegerLiteral((IntegerLiteral) lit);
727        } else if (lit instanceof StringLiteral) {
728            return walkStringLiteral((StringLiteral) lit);
729        } else {
730            throw new QueryParseException("Unknown literal: " + lit);
731        }
732    }
733
734    public Object walkBooleanLiteral(BooleanLiteral lit) {
735        return Boolean.valueOf(lit.value);
736    }
737
738    public Date walkDateLiteral(DateLiteral lit) {
739        return lit.value.toDate(); // TODO onlyDate
740    }
741
742    public Double walkDoubleLiteral(DoubleLiteral lit) {
743        return Double.valueOf(lit.value);
744    }
745
746    public Long walkIntegerLiteral(IntegerLiteral lit) {
747        return Long.valueOf(lit.value);
748    }
749
750    public String walkStringLiteral(StringLiteral lit) {
751        return lit.value;
752    }
753
754    public List<Object> walkLiteralList(LiteralList litList) {
755        List<Object> list = new ArrayList<Object>(litList.size());
756        for (Literal lit : litList) {
757            list.add(walkLiteral(lit));
758        }
759        return list;
760    }
761
762    protected List<Object> walkOperandList(List<Operand> values) {
763        List<Object> list = new LinkedList<>();
764        for (Operand value : values) {
765            list.add(walkOperand(value));
766        }
767        return list;
768    }
769
770    public Object walkFunction(Function func) {
771        throw new UnsupportedOperationException(func.name);
772    }
773
774    public DBObject walkStartsWith(Operand lvalue, Operand rvalue) {
775        if (!(lvalue instanceof Reference)) {
776            throw new QueryParseException("Invalid STARTSWITH query, left hand side must be a property: " + lvalue);
777        }
778        String name = ((Reference) lvalue).name;
779        if (!(rvalue instanceof StringLiteral)) {
780            throw new QueryParseException(
781                    "Invalid STARTSWITH query, right hand side must be a literal path: " + rvalue);
782        }
783        String path = ((StringLiteral) rvalue).value;
784        if (path.length() > 1 && path.endsWith("/")) {
785            path = path.substring(0, path.length() - 1);
786        }
787
788        if (NXQL.ECM_PATH.equals(name)) {
789            return walkStartsWithPath(path);
790        } else {
791            return walkStartsWithNonPath(lvalue, path);
792        }
793    }
794
795    protected DBObject walkStartsWithPath(String path) {
796        // resolve path
797        String ancestorId = pathResolver.getIdForPath(path);
798        if (ancestorId == null) {
799            // no such path
800            // TODO XXX do better
801            return new BasicDBObject(MONGODB_ID, "__nosuchid__");
802        }
803        return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId);
804    }
805
806    protected DBObject walkStartsWithNonPath(Operand lvalue, String path) {
807        FieldInfo fieldInfo = walkReference(lvalue);
808        DBObject eq = new FieldInfoDBObject(fieldInfo, path);
809        // escape except alphanumeric and others not needing escaping
810        String regex = path.replaceAll("([^a-zA-Z0-9 /])", "\\\\$1");
811        Pattern pattern = Pattern.compile(regex + "/.*");
812        DBObject like = new FieldInfoDBObject(fieldInfo, pattern);
813        return new BasicDBObject(QueryOperators.OR, Arrays.asList(eq, like));
814    }
815
816    protected FieldInfo walkReference(Operand value) {
817        if (!(value instanceof Reference)) {
818            throw new QueryParseException("Invalid query, left hand side must be a property: " + value);
819        }
820        return walkReference((Reference) value);
821    }
822
823    // non-canonical index syntax, for replaceAll
824    protected final static Pattern NON_CANON_INDEX = Pattern.compile("[^/\\[\\]]+" // name
825            + "\\[(\\d+|\\*|\\*\\d+)\\]" // index in brackets
826    );
827
828    /**
829     * Canonicalizes a Nuxeo-xpath.
830     * <p>
831     * Replaces {@code a/foo[123]/b} with {@code a/123/b}
832     * <p>
833     * A star or a star followed by digits can be used instead of just the digits as well.
834     *
835     * @param xpath the xpath
836     * @return the canonicalized xpath.
837     */
838    public static String canonicalXPath(String xpath) {
839        while (xpath.length() > 0 && xpath.charAt(0) == '/') {
840            xpath = xpath.substring(1);
841        }
842        if (xpath.indexOf('[') == -1) {
843            return xpath;
844        } else {
845            return NON_CANON_INDEX.matcher(xpath).replaceAll("$1");
846        }
847    }
848
849    /** Splits foo.*.bar into foo, *, bar and foo.*1.bar into foo, *1, bar */
850    protected final static Pattern WILDCARD_SPLIT = Pattern.compile("([^*]*)\\.\\*(\\d*)\\.(.*)");
851
852    protected static class FieldInfo {
853
854        /** NXQL property. */
855        protected final String prop;
856
857        /** MongoDB field including wildcards (not used as-is). */
858        protected final String fullField;
859
860        /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */
861        protected final String queryField;
862
863        /** MongoDB field for projection. */
864        protected final String projectionField;
865
866        protected final boolean isBoolean;
867
868        /**
869         * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly.
870         */
871        protected final boolean isTrueOrNullBoolean;
872
873        protected final boolean hasWildcard;
874
875        /** Prefix before the wildcard. */
876        protected final String fieldPrefix;
877
878        /** Wildcard part after * */
879        protected final String fieldWildcard;
880
881        /** Part after wildcard. */
882        protected final String fieldSuffix;
883
884        protected FieldInfo(String prop, String fullField, String queryField, String projectionField, boolean isBoolean,
885                boolean isTrueOrNullBoolean) {
886            this.prop = prop;
887            this.fullField = fullField;
888            this.queryField = queryField;
889            this.projectionField = projectionField;
890            this.isBoolean = isBoolean;
891            this.isTrueOrNullBoolean = isTrueOrNullBoolean;
892            Matcher m = WILDCARD_SPLIT.matcher(fullField);
893            if (m.matches()) {
894                hasWildcard = true;
895                fieldPrefix = m.group(1);
896                fieldWildcard = m.group(2);
897                fieldSuffix = m.group(3);
898            } else {
899                hasWildcard = false;
900                fieldPrefix = fieldWildcard = fieldSuffix = null;
901            }
902        }
903    }
904
905    protected static class FieldInfoDBObject extends BasicDBObject {
906
907        private static final long serialVersionUID = 1L;
908
909        protected FieldInfo fieldInfo;
910
911        public FieldInfoDBObject(FieldInfo fieldInfo, Object value) {
912            super(fieldInfo.queryField, value);
913            this.fieldInfo = fieldInfo;
914        }
915    }
916
917    /**
918     * Returns the MongoDB field for this reference.
919     */
920    public FieldInfo walkReference(Reference ref) {
921        String prop = ref.name;
922        prop = canonicalXPath(prop);
923        String[] parts = prop.split("/");
924        if (prop.startsWith(NXQL.ECM_PREFIX)) {
925            if (prop.startsWith(NXQL.ECM_ACL + "/")) {
926                if (parts.length != 3) {
927                    throw new QueryParseException("No such property: " + ref.name);
928                }
929                String wildcard = parts[1];
930                if (NumberUtils.isDigits(wildcard)) {
931                    throw new QueryParseException("Cannot use explicit index in ACLs: " + ref.name);
932                }
933                String last = parts[2];
934                String fullField;
935                String queryField;
936                String projectionField;
937                boolean isBoolean;
938                if (NXQL.ECM_ACL_NAME.equals(last)) {
939                    fullField = KEY_ACP + "." + KEY_ACL_NAME;
940                    queryField = KEY_ACP + "." + KEY_ACL_NAME;
941                    // TODO remember wildcard correlation
942                    isBoolean = false;
943                } else {
944                    String fieldLast = DBSSession.convToInternalAce(last);
945                    if (fieldLast == null) {
946                        throw new QueryParseException("No such property: " + ref.name);
947                    }
948                    isBoolean = KEY_ACE_GRANT.equals(fieldLast);
949                    fullField = KEY_ACP + "." + KEY_ACL + "." + wildcard + "." + fieldLast;
950                    queryField = KEY_ACP + "." + KEY_ACL + "." + fieldLast;
951                }
952                projectionField = queryField;
953                return new FieldInfo(prop, fullField, queryField, projectionField, isBoolean, false);
954            } else {
955                // simple field
956                String field = DBSSession.convToInternal(prop);
957                boolean isBoolean = DBSSession.isBoolean(field);
958                return new FieldInfo(prop, field, field, field, isBoolean, true);
959            }
960        } else {
961            String first = parts[0];
962            Field field = schemaManager.getField(first);
963            if (field == null) {
964                if (first.indexOf(':') > -1) {
965                    throw new QueryParseException("No such property: " + ref.name);
966                }
967                // check without prefix
968                // TODO precompute this in SchemaManagerImpl
969                for (Schema schema : schemaManager.getSchemas()) {
970                    if (!StringUtils.isBlank(schema.getNamespace().prefix)) {
971                        // schema with prefix, do not consider as candidate
972                        continue;
973                    }
974                    if (schema != null) {
975                        field = schema.getField(first);
976                        if (field != null) {
977                            break;
978                        }
979                    }
980                }
981                if (field == null) {
982                    throw new QueryParseException("No such property: " + ref.name);
983                }
984            }
985            Type type = field.getType();
986            // canonical name
987            parts[0] = field.getName().getPrefixedName();
988            // are there wildcards or list indexes?
989            List<String> queryFieldParts = new LinkedList<>(); // field for query
990            List<String> projectionFieldParts = new LinkedList<>(); // field for projection
991            boolean firstPart = true;
992            for (String part : parts) {
993                if (NumberUtils.isDigits(part)) {
994                    // explicit list index
995                    queryFieldParts.add(part);
996                    type = ((ListType) type).getFieldType();
997                } else if (!part.startsWith("*")) {
998                    // complex sub-property
999                    queryFieldParts.add(part);
1000                    projectionFieldParts.add(part);
1001                    if (!firstPart) {
1002                        // we already computed the type of the first part
1003                        field = ((ComplexType) type).getField(part);
1004                        if (field == null) {
1005                            throw new QueryParseException("No such property: " + ref.name);
1006                        }
1007                        type = field.getType();
1008                    }
1009                } else {
1010                    // wildcard
1011                    type = ((ListType) type).getFieldType();
1012                }
1013                firstPart = false;
1014            }
1015            String fullField = StringUtils.join(parts, '.');
1016            String queryField = StringUtils.join(queryFieldParts, '.');
1017            String projectionField = StringUtils.join(projectionFieldParts, '.');
1018            boolean isBoolean = type instanceof BooleanType;
1019            return new FieldInfo(prop, fullField, queryField, projectionField, isBoolean, false);
1020        }
1021    }
1022
1023    protected boolean isMixinTypes(FieldInfo fieldInfo) {
1024        return fieldInfo.queryField.equals(DBSDocument.KEY_MIXIN_TYPES);
1025    }
1026
1027    protected Set<String> getMixinDocumentTypes(String mixin) {
1028        Set<String> types = schemaManager.getDocumentTypeNamesForFacet(mixin);
1029        return types == null ? Collections.emptySet() : types;
1030    }
1031
1032    protected List<String> getDocumentTypes() {
1033        // TODO precompute in SchemaManager
1034        if (documentTypes == null) {
1035            documentTypes = new ArrayList<>();
1036            for (DocumentType docType : schemaManager.getDocumentTypes()) {
1037                documentTypes.add(docType.getName());
1038            }
1039        }
1040        return documentTypes;
1041    }
1042
1043    protected boolean isNeverPerInstanceMixin(String mixin) {
1044        return schemaManager.getNoPerDocumentQueryFacets().contains(mixin);
1045    }
1046
1047    /**
1048     * Matches the mixin types against a list of values.
1049     * <p>
1050     * Used for:
1051     * <ul>
1052     * <li>ecm:mixinTypes = 'Foo'
1053     * <li>ecm:mixinTypes != 'Foo'
1054     * <li>ecm:mixinTypes IN ('Foo', 'Bar')
1055     * <li>ecm:mixinTypes NOT IN ('Foo', 'Bar')
1056     * </ul>
1057     * <p>
1058     * ecm:mixinTypes IN ('Foo', 'Bar')
1059     *
1060     * <pre>
1061     * { "$or" : [ { "ecm:primaryType" : { "$in" : [ ... types with Foo or Bar ...]}} ,
1062     *             { "ecm:mixinTypes" : { "$in" : [ "Foo" , "Bar]}}]}
1063     * </pre>
1064     *
1065     * ecm:mixinTypes NOT IN ('Foo', 'Bar')
1066     * <p>
1067     *
1068     * <pre>
1069     * { "$and" : [ { "ecm:primaryType" : { "$in" : [ ... types without Foo nor Bar ...]}} ,
1070     *              { "ecm:mixinTypes" : { "$nin" : [ "Foo" , "Bar]}}]}
1071     * </pre>
1072     */
1073    public DBObject walkMixinTypes(List<String> mixins, boolean include) {
1074        /*
1075         * Primary types that match.
1076         */
1077        Set<String> matchPrimaryTypes;
1078        if (include) {
1079            matchPrimaryTypes = new HashSet<String>();
1080            for (String mixin : mixins) {
1081                matchPrimaryTypes.addAll(getMixinDocumentTypes(mixin));
1082            }
1083        } else {
1084            matchPrimaryTypes = new HashSet<String>(getDocumentTypes());
1085            for (String mixin : mixins) {
1086                matchPrimaryTypes.removeAll(getMixinDocumentTypes(mixin));
1087            }
1088        }
1089        /*
1090         * Instance mixins that match.
1091         */
1092        Set<String> matchMixinTypes = new HashSet<String>();
1093        for (String mixin : mixins) {
1094            if (!isNeverPerInstanceMixin(mixin)) {
1095                matchMixinTypes.add(mixin);
1096            }
1097        }
1098        /*
1099         * MongoDB query generation.
1100         */
1101        // match on primary type
1102        DBObject p = new BasicDBObject(DBSDocument.KEY_PRIMARY_TYPE,
1103                new BasicDBObject(QueryOperators.IN, matchPrimaryTypes));
1104        // match on mixin types
1105        // $in/$nin with an array matches if any/no element of the array matches
1106        String innin = include ? QueryOperators.IN : QueryOperators.NIN;
1107        DBObject m = new BasicDBObject(DBSDocument.KEY_MIXIN_TYPES, new BasicDBObject(innin, matchMixinTypes));
1108        // and/or between those
1109        String op = include ? QueryOperators.OR : QueryOperators.AND;
1110        return new BasicDBObject(op, Arrays.asList(p, m));
1111    }
1112
1113}