001/* 002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.storage.mongodb; 020 021import static java.lang.Boolean.FALSE; 022import static java.lang.Boolean.TRUE; 023import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL; 024import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL_NAME; 025import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP; 026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SCORE; 027import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID; 028import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_META; 029import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_TEXT_SCORE; 030 031import java.util.ArrayList; 032import java.util.Arrays; 033import java.util.Collections; 034import java.util.Date; 035import java.util.HashMap; 036import java.util.HashSet; 037import java.util.Iterator; 038import java.util.LinkedHashMap; 039import java.util.LinkedList; 040import java.util.List; 041import java.util.Map; 042import java.util.Map.Entry; 043import java.util.Set; 044import java.util.concurrent.atomic.AtomicInteger; 045import java.util.regex.Matcher; 046import java.util.regex.Pattern; 047 048import org.apache.commons.lang.StringUtils; 049import org.apache.commons.lang.math.NumberUtils; 050import org.nuxeo.ecm.core.query.QueryParseException; 051import org.nuxeo.ecm.core.query.sql.NXQL; 052import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral; 053import org.nuxeo.ecm.core.query.sql.model.DateLiteral; 054import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral; 055import org.nuxeo.ecm.core.query.sql.model.Expression; 056import org.nuxeo.ecm.core.query.sql.model.Function; 057import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral; 058import org.nuxeo.ecm.core.query.sql.model.Literal; 059import org.nuxeo.ecm.core.query.sql.model.LiteralList; 060import org.nuxeo.ecm.core.query.sql.model.MultiExpression; 061import org.nuxeo.ecm.core.query.sql.model.Operand; 062import org.nuxeo.ecm.core.query.sql.model.Operator; 063import org.nuxeo.ecm.core.query.sql.model.OrderByClause; 064import org.nuxeo.ecm.core.query.sql.model.OrderByExpr; 065import org.nuxeo.ecm.core.query.sql.model.Reference; 066import org.nuxeo.ecm.core.query.sql.model.SelectClause; 067import org.nuxeo.ecm.core.query.sql.model.StringLiteral; 068import org.nuxeo.ecm.core.schema.DocumentType; 069import org.nuxeo.ecm.core.schema.SchemaManager; 070import org.nuxeo.ecm.core.schema.types.ComplexType; 071import org.nuxeo.ecm.core.schema.types.Field; 072import org.nuxeo.ecm.core.schema.types.ListType; 073import org.nuxeo.ecm.core.schema.types.Schema; 074import org.nuxeo.ecm.core.schema.types.Type; 075import org.nuxeo.ecm.core.schema.types.primitives.BooleanType; 076import org.nuxeo.ecm.core.schema.types.primitives.DateType; 077import org.nuxeo.ecm.core.storage.ExpressionEvaluator; 078import org.nuxeo.ecm.core.storage.ExpressionEvaluator.PathResolver; 079import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer; 080import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.FulltextQuery; 081import org.nuxeo.ecm.core.storage.FulltextQueryAnalyzer.Op; 082import org.nuxeo.ecm.core.storage.dbs.DBSDocument; 083import org.nuxeo.ecm.core.storage.dbs.DBSSession; 084import org.nuxeo.runtime.api.Framework; 085 086import com.mongodb.BasicDBObject; 087import com.mongodb.DBObject; 088import com.mongodb.QueryOperators; 089 090/** 091 * Query builder for a MongoDB query from an {@link Expression}. 092 * 093 * @since 5.9.4 094 */ 095public class MongoDBQueryBuilder { 096 097 private static final Long ZERO = Long.valueOf(0); 098 099 private static final Long ONE = Long.valueOf(1); 100 101 private static final Long MINUS_ONE = Long.valueOf(-1); 102 103 protected static final String DATE_CAST = "DATE"; 104 105 protected final AtomicInteger counter = new AtomicInteger(); 106 107 protected final SchemaManager schemaManager; 108 109 protected final MongoDBConverter converter; 110 111 protected final String idKey; 112 113 protected List<String> documentTypes; 114 115 protected final Expression expression; 116 117 protected final SelectClause selectClause; 118 119 protected final OrderByClause orderByClause; 120 121 protected final PathResolver pathResolver; 122 123 public boolean hasFulltext; 124 125 public boolean sortOnFulltextScore; 126 127 protected DBObject query; 128 129 protected DBObject orderBy; 130 131 protected DBObject projection; 132 133 boolean projectionHasWildcard; 134 135 private boolean fulltextSearchDisabled; 136 137 public MongoDBQueryBuilder(MongoDBRepository repository, Expression expression, SelectClause selectClause, 138 OrderByClause orderByClause, PathResolver pathResolver, boolean fulltextSearchDisabled) { 139 schemaManager = Framework.getLocalService(SchemaManager.class); 140 converter = repository.converter; 141 idKey = repository.idKey; 142 this.expression = expression; 143 this.selectClause = selectClause; 144 this.orderByClause = orderByClause; 145 this.pathResolver = pathResolver; 146 this.fulltextSearchDisabled = fulltextSearchDisabled; 147 } 148 149 public void walk() { 150 query = walkExpression(expression); // computes hasFulltext 151 walkOrderBy(); // computes sortOnFulltextScore 152 walkProjection(); // needs hasFulltext and sortOnFulltextScore 153 } 154 155 public DBObject getQuery() { 156 return query; 157 } 158 159 public DBObject getOrderBy() { 160 return orderBy; 161 } 162 163 public DBObject getProjection() { 164 return projection; 165 } 166 167 public boolean hasProjectionWildcard() { 168 return projectionHasWildcard; 169 } 170 171 protected void walkOrderBy() { 172 sortOnFulltextScore = false; 173 if (orderByClause == null) { 174 orderBy = null; 175 } else { 176 orderBy = new BasicDBObject(); 177 for (OrderByExpr ob : orderByClause.elements) { 178 Reference ref = ob.reference; 179 boolean desc = ob.isDescending; 180 String field = walkReference(ref).queryField; 181 if (!orderBy.containsField(field)) { 182 Object value; 183 if (KEY_FULLTEXT_SCORE.equals(field)) { 184 if (!desc) { 185 throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " ascending"); 186 } 187 sortOnFulltextScore = true; 188 value = new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE); 189 } else { 190 value = desc ? MINUS_ONE : ONE; 191 } 192 orderBy.put(field, value); 193 } 194 } 195 if (sortOnFulltextScore && ((BasicDBObject) orderBy).size() > 1) { 196 throw new QueryParseException("Cannot sort by " + NXQL.ECM_FULLTEXT_SCORE + " and other criteria"); 197 } 198 } 199 } 200 201 protected void walkProjection() { 202 projection = new BasicDBObject(); 203 boolean projectionOnFulltextScore = false; 204 for (Operand op : selectClause.getSelectList().values()) { 205 if (!(op instanceof Reference)) { 206 throw new QueryParseException("Projection not supported: " + op); 207 } 208 FieldInfo fieldInfo = walkReference((Reference) op); 209 projection.put(fieldInfo.projectionField, ONE); 210 if (fieldInfo.hasWildcard) { 211 projectionHasWildcard = true; 212 } 213 if (fieldInfo.projectionField.equals(KEY_FULLTEXT_SCORE)) { 214 projectionOnFulltextScore = true; 215 } 216 } 217 if (projectionOnFulltextScore || sortOnFulltextScore) { 218 if (!hasFulltext) { 219 throw new QueryParseException(NXQL.ECM_FULLTEXT_SCORE + " cannot be used without " + NXQL.ECM_FULLTEXT); 220 } 221 projection.put(KEY_FULLTEXT_SCORE, new BasicDBObject(MONGODB_META, MONGODB_TEXT_SCORE)); 222 } 223 } 224 225 public DBObject walkExpression(Expression expr) { 226 Operator op = expr.operator; 227 Operand lvalue = expr.lvalue; 228 Operand rvalue = expr.rvalue; 229 Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null; 230 String name = ref != null ? ref.name : null; 231 String cast = ref != null ? ref.cast : null; 232 if (DATE_CAST.equals(cast)) { 233 checkDateLiteralForCast(op, rvalue, name); 234 } 235 if (op == Operator.STARTSWITH) { 236 return walkStartsWith(lvalue, rvalue); 237 } else if (NXQL.ECM_PATH.equals(name)) { 238 return walkEcmPath(op, rvalue); 239 } else if (NXQL.ECM_ANCESTORID.equals(name)) { 240 return walkAncestorId(op, rvalue); 241 } else if (name != null && name.startsWith(NXQL.ECM_FULLTEXT) && !NXQL.ECM_FULLTEXT_JOBID.equals(name)) { 242 return walkEcmFulltext(name, op, rvalue); 243 } else if (op == Operator.SUM) { 244 throw new UnsupportedOperationException("SUM"); 245 } else if (op == Operator.SUB) { 246 throw new UnsupportedOperationException("SUB"); 247 } else if (op == Operator.MUL) { 248 throw new UnsupportedOperationException("MUL"); 249 } else if (op == Operator.DIV) { 250 throw new UnsupportedOperationException("DIV"); 251 } else if (op == Operator.LT) { 252 return walkLt(lvalue, rvalue); 253 } else if (op == Operator.GT) { 254 return walkGt(lvalue, rvalue); 255 } else if (op == Operator.EQ) { 256 return walkEq(lvalue, rvalue); 257 } else if (op == Operator.NOTEQ) { 258 return walkNotEq(lvalue, rvalue); 259 } else if (op == Operator.LTEQ) { 260 return walkLtEq(lvalue, rvalue); 261 } else if (op == Operator.GTEQ) { 262 return walkGtEq(lvalue, rvalue); 263 } else if (op == Operator.AND) { 264 if (expr instanceof MultiExpression) { 265 return walkMultiExpression((MultiExpression) expr); 266 } else { 267 return walkAnd(lvalue, rvalue); 268 } 269 } else if (op == Operator.NOT) { 270 return walkNot(lvalue); 271 } else if (op == Operator.OR) { 272 return walkOr(lvalue, rvalue); 273 } else if (op == Operator.LIKE) { 274 return walkLike(lvalue, rvalue, true, false); 275 } else if (op == Operator.ILIKE) { 276 return walkLike(lvalue, rvalue, true, true); 277 } else if (op == Operator.NOTLIKE) { 278 return walkLike(lvalue, rvalue, false, false); 279 } else if (op == Operator.NOTILIKE) { 280 return walkLike(lvalue, rvalue, false, true); 281 } else if (op == Operator.IN) { 282 return walkIn(lvalue, rvalue, true); 283 } else if (op == Operator.NOTIN) { 284 return walkIn(lvalue, rvalue, false); 285 } else if (op == Operator.ISNULL) { 286 return walkIsNull(lvalue); 287 } else if (op == Operator.ISNOTNULL) { 288 return walkIsNotNull(lvalue); 289 } else if (op == Operator.BETWEEN) { 290 return walkBetween(lvalue, rvalue, true); 291 } else if (op == Operator.NOTBETWEEN) { 292 return walkBetween(lvalue, rvalue, false); 293 } else { 294 throw new QueryParseException("Unknown operator: " + op); 295 } 296 } 297 298 protected void checkDateLiteralForCast(Operator op, Operand value, String name) { 299 if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) { 300 LiteralList l = (LiteralList) value; 301 checkDateLiteralForCast(l.get(0), name); 302 checkDateLiteralForCast(l.get(1), name); 303 } else { 304 checkDateLiteralForCast(value, name); 305 } 306 } 307 308 protected void checkDateLiteralForCast(Operand value, String name) { 309 if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) { 310 throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name); 311 } 312 } 313 314 protected DBObject walkEcmPath(Operator op, Operand rvalue) { 315 if (op != Operator.EQ && op != Operator.NOTEQ) { 316 throw new QueryParseException(NXQL.ECM_PATH + " requires = or <> operator"); 317 } 318 if (!(rvalue instanceof StringLiteral)) { 319 throw new QueryParseException(NXQL.ECM_PATH + " requires literal path as right argument"); 320 } 321 String path = ((StringLiteral) rvalue).value; 322 if (path.length() > 1 && path.endsWith("/")) { 323 path = path.substring(0, path.length() - 1); 324 } 325 String id = pathResolver.getIdForPath(path); 326 if (id == null) { 327 // no such path 328 // TODO XXX do better 329 return new BasicDBObject(MONGODB_ID, "__nosuchid__"); 330 } 331 if (op == Operator.EQ) { 332 return new BasicDBObject(idKey, id); 333 } else { 334 return new BasicDBObject(idKey, new BasicDBObject(QueryOperators.NE, id)); 335 } 336 } 337 338 protected DBObject walkAncestorId(Operator op, Operand rvalue) { 339 if (op != Operator.EQ && op != Operator.NOTEQ) { 340 throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires = or <> operator"); 341 } 342 if (!(rvalue instanceof StringLiteral)) { 343 throw new QueryParseException(NXQL.ECM_ANCESTORID + " requires literal id as right argument"); 344 } 345 String ancestorId = ((StringLiteral) rvalue).value; 346 if (op == Operator.EQ) { 347 return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId); 348 } else { 349 return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, new BasicDBObject(QueryOperators.NE, ancestorId)); 350 } 351 } 352 353 protected DBObject walkEcmFulltext(String name, Operator op, Operand rvalue) { 354 if (op != Operator.EQ && op != Operator.LIKE) { 355 throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires = or LIKE operator"); 356 } 357 if (!(rvalue instanceof StringLiteral)) { 358 throw new QueryParseException(NXQL.ECM_FULLTEXT + " requires literal string as right argument"); 359 } 360 if (fulltextSearchDisabled) { 361 throw new QueryParseException("Fulltext search disabled by configuration"); 362 } 363 String fulltextQuery = ((StringLiteral) rvalue).value; 364 if (name.equals(NXQL.ECM_FULLTEXT)) { 365 // standard fulltext query 366 hasFulltext = true; 367 String ft = getMongoDBFulltextQuery(fulltextQuery); 368 if (ft == null) { 369 // empty query, matches nothing 370 return new BasicDBObject(MONGODB_ID, "__nosuchid__"); 371 } 372 DBObject textSearch = new BasicDBObject(); 373 textSearch.put(QueryOperators.SEARCH, ft); 374 // TODO language? 375 return new BasicDBObject(QueryOperators.TEXT, textSearch); 376 } else { 377 // secondary index match with explicit field 378 // do a regexp on the field 379 if (name.charAt(NXQL.ECM_FULLTEXT.length()) != '.') { 380 throw new QueryParseException(name + " has incorrect syntax" + " for a secondary fulltext index"); 381 } 382 String prop = name.substring(NXQL.ECM_FULLTEXT.length() + 1); 383 String ft = fulltextQuery.replace(" ", "%"); 384 rvalue = new StringLiteral(ft); 385 return walkLike(new Reference(prop), rvalue, true, true); 386 } 387 } 388 389 // public static for tests 390 public static String getMongoDBFulltextQuery(String query) { 391 FulltextQuery ft = FulltextQueryAnalyzer.analyzeFulltextQuery(query); 392 if (ft == null) { 393 return null; 394 } 395 // translate into MongoDB syntax 396 return translateFulltext(ft, false); 397 } 398 399 /** 400 * Transforms the NXQL fulltext syntax into MongoDB syntax. 401 * <p> 402 * The MongoDB fulltext query syntax is badly documented, but is actually the following: 403 * <ul> 404 * <li>a term is a word, 405 * <li>a phrase is a set of spaced-separated words enclosed in double quotes, 406 * <li>negation is done by prepending a -, 407 * <li>the query is a space-separated set of terms, negated terms, phrases, or negated phrases. 408 * <li>all the words of non-negated phrases are also added to the terms. 409 * </ul> 410 * <p> 411 * The matching algorithm is (excluding stemming and stop words): 412 * <ul> 413 * <li>filter out documents with the negative terms, the negative phrases, or missing the phrases, 414 * <li>then if any term is present in the document then it's a match. 415 * </ul> 416 */ 417 protected static String translateFulltext(FulltextQuery ft, boolean and) { 418 List<String> buf = new ArrayList<>(); 419 translateFulltext(ft, buf, and); 420 return StringUtils.join(buf, ' '); 421 } 422 423 protected static void translateFulltext(FulltextQuery ft, List<String> buf, boolean and) { 424 if (ft.op == Op.OR) { 425 for (FulltextQuery term : ft.terms) { 426 // don't quote words for OR 427 translateFulltext(term, buf, false); 428 } 429 } else if (ft.op == Op.AND) { 430 for (FulltextQuery term : ft.terms) { 431 // quote words for AND 432 translateFulltext(term, buf, true); 433 } 434 } else { 435 String neg; 436 if (ft.op == Op.NOTWORD) { 437 neg = "-"; 438 } else { // Op.WORD 439 neg = ""; 440 } 441 String word = ft.word.toLowerCase(); 442 if (ft.isPhrase() || and) { 443 buf.add(neg + '"' + word + '"'); 444 } else { 445 buf.add(neg + word); 446 } 447 } 448 } 449 450 public DBObject walkNot(Operand value) { 451 Object val = walkOperand(value); 452 Object not = pushDownNot(val); 453 if (!(not instanceof DBObject)) { 454 throw new QueryParseException("Cannot do NOT on: " + val); 455 } 456 return (DBObject) not; 457 } 458 459 protected Object pushDownNot(Object object) { 460 if (!(object instanceof DBObject)) { 461 throw new QueryParseException("Cannot do NOT on: " + object); 462 } 463 DBObject ob = (DBObject) object; 464 Set<String> keySet = ob.keySet(); 465 if (keySet.size() != 1) { 466 throw new QueryParseException("Cannot do NOT on: " + ob); 467 } 468 String key = keySet.iterator().next(); 469 Object value = ob.get(key); 470 if (!key.startsWith("$")) { 471 if (value instanceof DBObject) { 472 // push down inside dbobject 473 return new BasicDBObject(key, pushDownNot(value)); 474 } else { 475 // k = v -> k != v 476 return new BasicDBObject(key, new BasicDBObject(QueryOperators.NE, value)); 477 } 478 } 479 if (QueryOperators.NE.equals(key)) { 480 // NOT k != v -> k = v 481 return value; 482 } 483 if (QueryOperators.NOT.equals(key)) { 484 // NOT NOT v -> v 485 return value; 486 } 487 if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) { 488 // boolean algebra 489 // NOT (v1 AND v2) -> NOT v1 OR NOT v2 490 // NOT (v1 OR v2) -> NOT v1 AND NOT v2 491 String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND; 492 List<Object> list = (List<Object>) value; 493 for (int i = 0; i < list.size(); i++) { 494 list.set(i, pushDownNot(list.get(i))); 495 } 496 return new BasicDBObject(op, list); 497 } 498 if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) { 499 // boolean algebra 500 // IN <-> NIN 501 String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN; 502 return new BasicDBObject(op, value); 503 } 504 if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key) 505 || QueryOperators.GTE.equals(key)) { 506 // TODO use inverse operators? 507 return new BasicDBObject(QueryOperators.NOT, ob); 508 } 509 throw new QueryParseException("Unknown operator for NOT: " + key); 510 } 511 512 public DBObject walkIsNull(Operand value) { 513 FieldInfo fieldInfo = walkReference(value); 514 return new FieldInfoDBObject(fieldInfo, null); 515 } 516 517 public DBObject walkIsNotNull(Operand value) { 518 FieldInfo fieldInfo = walkReference(value); 519 return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, null)); 520 } 521 522 public DBObject walkMultiExpression(MultiExpression expr) { 523 return walkAnd(expr.values); 524 } 525 526 public DBObject walkAnd(Operand lvalue, Operand rvalue) { 527 return walkAnd(Arrays.asList(lvalue, rvalue)); 528 } 529 530 protected DBObject walkAnd(List<Operand> values) { 531 List<Object> list = walkOperandList(values); 532 // check wildcards in the operands, extract common prefixes to use $elemMatch 533 Map<String, List<FieldInfoDBObject>> propBaseKeyToDBOs = new LinkedHashMap<>(); 534 Map<String, String> propBaseKeyToFieldBase = new HashMap<>(); 535 for (Iterator<Object> it = list.iterator(); it.hasNext();) { 536 Object ob = it.next(); 537 if (ob instanceof FieldInfoDBObject) { 538 FieldInfoDBObject fidbo = (FieldInfoDBObject) ob; 539 FieldInfo fieldInfo = fidbo.fieldInfo; 540 if (fieldInfo.hasWildcard) { 541 if (fieldInfo.fieldSuffix != null && fieldInfo.fieldSuffix.contains("*")) { 542 // a double wildcard of the form foo/*/bar/* is not a problem if bar is an array 543 // TODO prevent deep complex multiple wildcards 544 // throw new QueryParseException("Cannot use two wildcards: " + fieldInfo.prop); 545 } 546 // generate a key unique per correlation for this element match 547 String wildcardNumber = fieldInfo.fieldWildcard; 548 if (wildcardNumber.isEmpty()) { 549 // negative to not collide with regular correlated wildcards 550 wildcardNumber = String.valueOf(-counter.incrementAndGet()); 551 } 552 String propBaseKey = fieldInfo.fieldPrefix + "/*" + wildcardNumber; 553 // store object for this key 554 List<FieldInfoDBObject> dbos = propBaseKeyToDBOs.get(propBaseKey); 555 if (dbos == null) { 556 propBaseKeyToDBOs.put(propBaseKey, dbos = new LinkedList<>()); 557 } 558 dbos.add(fidbo); 559 // remember for which field base this is 560 String fieldBase = fieldInfo.fieldPrefix.replace("/", "."); 561 propBaseKeyToFieldBase.put(propBaseKey, fieldBase); 562 // remove from list, will be re-added later through propBaseKeyToDBOs 563 it.remove(); 564 } 565 } 566 } 567 // generate $elemMatch items for correlated queries 568 for (Entry<String, List<FieldInfoDBObject>> es : propBaseKeyToDBOs.entrySet()) { 569 String propBaseKey = es.getKey(); 570 List<FieldInfoDBObject> fidbos = es.getValue(); 571 if (fidbos.size() == 1) { 572 // regular uncorrelated match 573 list.addAll(fidbos); 574 } else { 575 DBObject elemMatch = new BasicDBObject(); 576 for (FieldInfoDBObject fidbo : fidbos) { 577 // truncate field name to just the suffix 578 FieldInfo fieldInfo = fidbo.fieldInfo; 579 Object value = fidbo.get(fieldInfo.queryField); 580 String fieldSuffix = fieldInfo.fieldSuffix.replace("/", "."); 581 if (elemMatch.containsField(fieldSuffix)) { 582 // ecm:acl/*1/principal = 'bob' AND ecm:acl/*1/principal = 'steve' 583 // cannot match 584 // TODO do better 585 value = "__NOSUCHVALUE__"; 586 } 587 elemMatch.put(fieldSuffix, value); 588 } 589 String fieldBase = propBaseKeyToFieldBase.get(propBaseKey); 590 BasicDBObject dbo = new BasicDBObject(fieldBase, 591 new BasicDBObject(QueryOperators.ELEM_MATCH, elemMatch)); 592 list.add(dbo); 593 } 594 } 595 if (list.size() == 1) { 596 return (DBObject) list.get(0); 597 } else { 598 return new BasicDBObject(QueryOperators.AND, list); 599 } 600 } 601 602 public DBObject walkOr(Operand lvalue, Operand rvalue) { 603 Object left = walkOperand(lvalue); 604 Object right = walkOperand(rvalue); 605 List<Object> list = new ArrayList<>(Arrays.asList(left, right)); 606 return new BasicDBObject(QueryOperators.OR, list); 607 } 608 609 protected Object checkBoolean(FieldInfo fieldInfo, Object right) { 610 if (fieldInfo.isBoolean()) { 611 // convert 0 / 1 to actual booleans 612 if (right instanceof Long) { 613 if (ZERO.equals(right)) { 614 right = fieldInfo.isTrueOrNullBoolean ? null : FALSE; 615 } else if (ONE.equals(right)) { 616 right = TRUE; 617 } else { 618 throw new QueryParseException("Invalid boolean: " + right); 619 } 620 } 621 } 622 return right; 623 } 624 625 public DBObject walkEq(Operand lvalue, Operand rvalue) { 626 FieldInfo fieldInfo = walkReference(lvalue); 627 Object right = walkOperand(rvalue); 628 if (isMixinTypes(fieldInfo)) { 629 if (!(right instanceof String)) { 630 throw new QueryParseException("Invalid EQ rhs: " + rvalue); 631 } 632 return walkMixinTypes(Collections.singletonList((String) right), true); 633 } 634 right = checkBoolean(fieldInfo, right); 635 // TODO check list fields 636 return new FieldInfoDBObject(fieldInfo, right); 637 } 638 639 public DBObject walkNotEq(Operand lvalue, Operand rvalue) { 640 FieldInfo fieldInfo = walkReference(lvalue); 641 Object right = walkOperand(rvalue); 642 if (isMixinTypes(fieldInfo)) { 643 if (!(right instanceof String)) { 644 throw new QueryParseException("Invalid NE rhs: " + rvalue); 645 } 646 return walkMixinTypes(Collections.singletonList((String) right), false); 647 } 648 right = checkBoolean(fieldInfo, right); 649 // TODO check list fields 650 return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.NE, right)); 651 } 652 653 public DBObject walkLt(Operand lvalue, Operand rvalue) { 654 FieldInfo fieldInfo = walkReference(lvalue); 655 Object right = walkOperand(rvalue); 656 return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, right)); 657 } 658 659 public DBObject walkGt(Operand lvalue, Operand rvalue) { 660 FieldInfo fieldInfo = walkReference(lvalue); 661 Object right = walkOperand(rvalue); 662 return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right)); 663 } 664 665 public DBObject walkLtEq(Operand lvalue, Operand rvalue) { 666 FieldInfo fieldInfo = walkReference(lvalue); 667 Object right = walkOperand(rvalue); 668 return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LTE, right)); 669 } 670 671 public DBObject walkGtEq(Operand lvalue, Operand rvalue) { 672 FieldInfo fieldInfo = walkReference(lvalue); 673 Object right = walkOperand(rvalue); 674 return new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GTE, right)); 675 } 676 677 public DBObject walkBetween(Operand lvalue, Operand rvalue, boolean positive) { 678 LiteralList l = (LiteralList) rvalue; 679 FieldInfo fieldInfo = walkReference(lvalue); 680 Object left = walkOperand(l.get(0)); 681 Object right = walkOperand(l.get(1)); 682 if (positive) { 683 DBObject range = new BasicDBObject(); 684 range.put(QueryOperators.GTE, left); 685 range.put(QueryOperators.LTE, right); 686 return new FieldInfoDBObject(fieldInfo, range); 687 } else { 688 DBObject a = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.LT, left)); 689 DBObject b = new FieldInfoDBObject(fieldInfo, new BasicDBObject(QueryOperators.GT, right)); 690 return new BasicDBObject(QueryOperators.OR, Arrays.asList(a, b)); 691 } 692 } 693 694 public DBObject walkIn(Operand lvalue, Operand rvalue, boolean positive) { 695 FieldInfo fieldInfo = walkReference(lvalue); 696 Object right = walkOperand(rvalue); 697 if (!(right instanceof List)) { 698 throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue); 699 } 700 if (isMixinTypes(fieldInfo)) { 701 return walkMixinTypes((List<String>) right, positive); 702 } 703 // TODO check list fields 704 List<Object> list = (List<Object>) right; 705 return new FieldInfoDBObject(fieldInfo, 706 new BasicDBObject(positive ? QueryOperators.IN : QueryOperators.NIN, list)); 707 } 708 709 public DBObject walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) { 710 FieldInfo fieldInfo = walkReference(lvalue); 711 if (!(rvalue instanceof StringLiteral)) { 712 throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue); 713 } 714 // TODO check list fields 715 String like = walkStringLiteral((StringLiteral) rvalue); 716 String regex = ExpressionEvaluator.likeToRegex(like); 717 718 int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0; 719 Pattern pattern = Pattern.compile(regex, flags); 720 Object value; 721 if (positive) { 722 value = pattern; 723 } else { 724 value = new BasicDBObject(QueryOperators.NOT, pattern); 725 } 726 return new FieldInfoDBObject(fieldInfo, value); 727 } 728 729 public Object walkOperand(Operand op) { 730 if (op instanceof Literal) { 731 return walkLiteral((Literal) op); 732 } else if (op instanceof LiteralList) { 733 return walkLiteralList((LiteralList) op); 734 } else if (op instanceof Function) { 735 return walkFunction((Function) op); 736 } else if (op instanceof Expression) { 737 return walkExpression((Expression) op); 738 } else if (op instanceof Reference) { 739 return walkReference((Reference) op); 740 } else { 741 throw new QueryParseException("Unknown operand: " + op); 742 } 743 } 744 745 public Object walkLiteral(Literal lit) { 746 if (lit instanceof BooleanLiteral) { 747 return walkBooleanLiteral((BooleanLiteral) lit); 748 } else if (lit instanceof DateLiteral) { 749 return walkDateLiteral((DateLiteral) lit); 750 } else if (lit instanceof DoubleLiteral) { 751 return walkDoubleLiteral((DoubleLiteral) lit); 752 } else if (lit instanceof IntegerLiteral) { 753 return walkIntegerLiteral((IntegerLiteral) lit); 754 } else if (lit instanceof StringLiteral) { 755 return walkStringLiteral((StringLiteral) lit); 756 } else { 757 throw new QueryParseException("Unknown literal: " + lit); 758 } 759 } 760 761 public Object walkBooleanLiteral(BooleanLiteral lit) { 762 return Boolean.valueOf(lit.value); 763 } 764 765 public Date walkDateLiteral(DateLiteral lit) { 766 return lit.value.toDate(); // TODO onlyDate 767 } 768 769 public Double walkDoubleLiteral(DoubleLiteral lit) { 770 return Double.valueOf(lit.value); 771 } 772 773 public Long walkIntegerLiteral(IntegerLiteral lit) { 774 return Long.valueOf(lit.value); 775 } 776 777 public String walkStringLiteral(StringLiteral lit) { 778 return lit.value; 779 } 780 781 public List<Object> walkLiteralList(LiteralList litList) { 782 List<Object> list = new ArrayList<>(litList.size()); 783 for (Literal lit : litList) { 784 list.add(walkLiteral(lit)); 785 } 786 return list; 787 } 788 789 protected List<Object> walkOperandList(List<Operand> values) { 790 List<Object> list = new LinkedList<>(); 791 for (Operand value : values) { 792 list.add(walkOperand(value)); 793 } 794 return list; 795 } 796 797 public Object walkFunction(Function func) { 798 throw new UnsupportedOperationException(func.name); 799 } 800 801 public DBObject walkStartsWith(Operand lvalue, Operand rvalue) { 802 if (!(lvalue instanceof Reference)) { 803 throw new QueryParseException("Invalid STARTSWITH query, left hand side must be a property: " + lvalue); 804 } 805 String name = ((Reference) lvalue).name; 806 if (!(rvalue instanceof StringLiteral)) { 807 throw new QueryParseException( 808 "Invalid STARTSWITH query, right hand side must be a literal path: " + rvalue); 809 } 810 String path = ((StringLiteral) rvalue).value; 811 if (path.length() > 1 && path.endsWith("/")) { 812 path = path.substring(0, path.length() - 1); 813 } 814 815 if (NXQL.ECM_PATH.equals(name)) { 816 return walkStartsWithPath(path); 817 } else { 818 return walkStartsWithNonPath(lvalue, path); 819 } 820 } 821 822 protected DBObject walkStartsWithPath(String path) { 823 // resolve path 824 String ancestorId = pathResolver.getIdForPath(path); 825 if (ancestorId == null) { 826 // no such path 827 // TODO XXX do better 828 return new BasicDBObject(MONGODB_ID, "__nosuchid__"); 829 } 830 return new BasicDBObject(DBSDocument.KEY_ANCESTOR_IDS, ancestorId); 831 } 832 833 protected DBObject walkStartsWithNonPath(Operand lvalue, String path) { 834 FieldInfo fieldInfo = walkReference(lvalue); 835 DBObject eq = new FieldInfoDBObject(fieldInfo, path); 836 // escape except alphanumeric and others not needing escaping 837 String regex = path.replaceAll("([^a-zA-Z0-9 /])", "\\\\$1"); 838 Pattern pattern = Pattern.compile(regex + "/.*"); 839 DBObject like = new FieldInfoDBObject(fieldInfo, pattern); 840 return new BasicDBObject(QueryOperators.OR, Arrays.asList(eq, like)); 841 } 842 843 protected FieldInfo walkReference(Operand value) { 844 if (!(value instanceof Reference)) { 845 throw new QueryParseException("Invalid query, left hand side must be a property: " + value); 846 } 847 return walkReference((Reference) value); 848 } 849 850 // non-canonical index syntax, for replaceAll 851 protected final static Pattern NON_CANON_INDEX = Pattern.compile("[^/\\[\\]]+" // name 852 + "\\[(\\d+|\\*|\\*\\d+)\\]" // index in brackets 853 ); 854 855 /** 856 * Canonicalizes a Nuxeo-xpath. 857 * <p> 858 * Replaces {@code a/foo[123]/b} with {@code a/123/b} 859 * <p> 860 * A star or a star followed by digits can be used instead of just the digits as well. 861 * 862 * @param xpath the xpath 863 * @return the canonicalized xpath. 864 */ 865 public static String canonicalXPath(String xpath) { 866 while (xpath.length() > 0 && xpath.charAt(0) == '/') { 867 xpath = xpath.substring(1); 868 } 869 if (xpath.indexOf('[') == -1) { 870 return xpath; 871 } else { 872 return NON_CANON_INDEX.matcher(xpath).replaceAll("$1"); 873 } 874 } 875 876 /** Splits foo.*.bar into foo, *, bar and split foo.*1.bar into foo, *1, bar with the last bar part optional */ 877 protected final static Pattern WILDCARD_SPLIT = Pattern.compile("([^*]*)\\.\\*(\\d*)(?:\\.(.*))?"); 878 879 protected static class FieldInfo { 880 881 /** NXQL property. */ 882 protected final String prop; 883 884 /** MongoDB field including wildcards (not used as-is). */ 885 protected final String fullField; 886 887 /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */ 888 protected final String queryField; 889 890 /** MongoDB field for projection. */ 891 protected final String projectionField; 892 893 protected final Type type; 894 895 /** 896 * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly. 897 */ 898 protected final boolean isTrueOrNullBoolean; 899 900 protected final boolean hasWildcard; 901 902 /** Prefix before the wildcard. */ 903 protected final String fieldPrefix; 904 905 /** Wildcard part after * */ 906 protected final String fieldWildcard; 907 908 /** Part after wildcard, may be null. */ 909 protected final String fieldSuffix; 910 911 protected FieldInfo(String prop, String fullField, String queryField, String projectionField, Type type, 912 boolean isTrueOrNullBoolean) { 913 this.prop = prop; 914 this.fullField = fullField; 915 this.queryField = queryField; 916 this.projectionField = projectionField; 917 this.type = type; 918 this.isTrueOrNullBoolean = isTrueOrNullBoolean; 919 Matcher m = WILDCARD_SPLIT.matcher(fullField); 920 if (m.matches()) { 921 hasWildcard = true; 922 fieldPrefix = m.group(1); 923 fieldWildcard = m.group(2); 924 fieldSuffix = m.group(3); 925 } else { 926 hasWildcard = false; 927 fieldPrefix = fieldWildcard = fieldSuffix = null; 928 } 929 } 930 931 protected boolean isBoolean() { 932 return type instanceof BooleanType; 933 } 934 } 935 936 protected static class FieldInfoDBObject extends BasicDBObject { 937 938 private static final long serialVersionUID = 1L; 939 940 protected FieldInfo fieldInfo; 941 942 public FieldInfoDBObject(FieldInfo fieldInfo, Object value) { 943 super(fieldInfo.queryField, value); 944 this.fieldInfo = fieldInfo; 945 } 946 } 947 948 /** 949 * Returns the MongoDB field for this reference. 950 */ 951 public FieldInfo walkReference(Reference ref) { 952 FieldInfo fieldInfo = walkReference(ref.name); 953 if (DATE_CAST.equals(ref.cast)) { 954 Type type = fieldInfo.type; 955 if (!(type instanceof DateType 956 || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) { 957 throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name); 958 } 959 // fieldInfo.isDateCast = true; 960 } 961 return fieldInfo; 962 } 963 964 protected FieldInfo walkReference(String name) { 965 String prop = canonicalXPath(name); 966 String[] parts = prop.split("/"); 967 if (prop.startsWith(NXQL.ECM_PREFIX)) { 968 if (prop.startsWith(NXQL.ECM_ACL + "/")) { 969 return parseACP(prop, parts); 970 } 971 // simple field 972 String field = DBSSession.convToInternal(prop); 973 Type type = DBSSession.getType(field); 974 String queryField = converter.keyToBson(field); 975 return new FieldInfo(prop, field, queryField, field, type, true); 976 } else { 977 String first = parts[0]; 978 Field field = schemaManager.getField(first); 979 if (field == null) { 980 if (first.indexOf(':') > -1) { 981 throw new QueryParseException("No such property: " + name); 982 } 983 // check without prefix 984 // TODO precompute this in SchemaManagerImpl 985 for (Schema schema : schemaManager.getSchemas()) { 986 if (!StringUtils.isBlank(schema.getNamespace().prefix)) { 987 // schema with prefix, do not consider as candidate 988 continue; 989 } 990 if (schema != null) { 991 field = schema.getField(first); 992 if (field != null) { 993 break; 994 } 995 } 996 } 997 if (field == null) { 998 throw new QueryParseException("No such property: " + name); 999 } 1000 } 1001 Type type = field.getType(); 1002 // canonical name 1003 parts[0] = field.getName().getPrefixedName(); 1004 // are there wildcards or list indexes? 1005 List<String> queryFieldParts = new LinkedList<>(); // field for query 1006 List<String> projectionFieldParts = new LinkedList<>(); // field for projection 1007 boolean firstPart = true; 1008 for (String part : parts) { 1009 if (NumberUtils.isDigits(part)) { 1010 // explicit list index 1011 queryFieldParts.add(part); 1012 type = ((ListType) type).getFieldType(); 1013 } else if (!part.startsWith("*")) { 1014 // complex sub-property 1015 queryFieldParts.add(part); 1016 projectionFieldParts.add(part); 1017 if (!firstPart) { 1018 // we already computed the type of the first part 1019 field = ((ComplexType) type).getField(part); 1020 if (field == null) { 1021 throw new QueryParseException("No such property: " + name); 1022 } 1023 type = field.getType(); 1024 } 1025 } else { 1026 // wildcard 1027 type = ((ListType) type).getFieldType(); 1028 } 1029 firstPart = false; 1030 } 1031 String fullField = StringUtils.join(parts, '.'); 1032 String queryField = StringUtils.join(queryFieldParts, '.'); 1033 String projectionField = StringUtils.join(projectionFieldParts, '.'); 1034 return new FieldInfo(prop, fullField, queryField, projectionField, type, false); 1035 } 1036 } 1037 1038 protected FieldInfo parseACP(String prop, String[] parts) { 1039 if (parts.length != 3) { 1040 throw new QueryParseException("No such property: " + prop); 1041 } 1042 String wildcard = parts[1]; 1043 if (NumberUtils.isDigits(wildcard)) { 1044 throw new QueryParseException("Cannot use explicit index in ACLs: " + prop); 1045 } 1046 String last = parts[2]; 1047 String fullField; 1048 String queryField; 1049 String projectionField; 1050 if (NXQL.ECM_ACL_NAME.equals(last)) { 1051 fullField = KEY_ACP + "." + KEY_ACL_NAME; 1052 queryField = KEY_ACP + "." + KEY_ACL_NAME; 1053 // TODO remember wildcard correlation 1054 } else { 1055 String fieldLast = DBSSession.convToInternalAce(last); 1056 if (fieldLast == null) { 1057 throw new QueryParseException("No such property: " + prop); 1058 } 1059 fullField = KEY_ACP + "." + KEY_ACL + "." + wildcard + "." + fieldLast; 1060 queryField = KEY_ACP + "." + KEY_ACL + "." + fieldLast; 1061 } 1062 Type type = DBSSession.getType(last); 1063 projectionField = queryField; 1064 return new FieldInfo(prop, fullField, queryField, projectionField, type, false); 1065 } 1066 1067 protected boolean isMixinTypes(FieldInfo fieldInfo) { 1068 return fieldInfo.queryField.equals(DBSDocument.KEY_MIXIN_TYPES); 1069 } 1070 1071 protected Set<String> getMixinDocumentTypes(String mixin) { 1072 Set<String> types = schemaManager.getDocumentTypeNamesForFacet(mixin); 1073 return types == null ? Collections.emptySet() : types; 1074 } 1075 1076 protected List<String> getDocumentTypes() { 1077 // TODO precompute in SchemaManager 1078 if (documentTypes == null) { 1079 documentTypes = new ArrayList<>(); 1080 for (DocumentType docType : schemaManager.getDocumentTypes()) { 1081 documentTypes.add(docType.getName()); 1082 } 1083 } 1084 return documentTypes; 1085 } 1086 1087 protected boolean isNeverPerInstanceMixin(String mixin) { 1088 return schemaManager.getNoPerDocumentQueryFacets().contains(mixin); 1089 } 1090 1091 /** 1092 * Matches the mixin types against a list of values. 1093 * <p> 1094 * Used for: 1095 * <ul> 1096 * <li>ecm:mixinTypes = 'Foo' 1097 * <li>ecm:mixinTypes != 'Foo' 1098 * <li>ecm:mixinTypes IN ('Foo', 'Bar') 1099 * <li>ecm:mixinTypes NOT IN ('Foo', 'Bar') 1100 * </ul> 1101 * <p> 1102 * ecm:mixinTypes IN ('Foo', 'Bar') 1103 * 1104 * <pre> 1105 * { "$or" : [ { "ecm:primaryType" : { "$in" : [ ... types with Foo or Bar ...]}} , 1106 * { "ecm:mixinTypes" : { "$in" : [ "Foo" , "Bar]}}]} 1107 * </pre> 1108 * 1109 * ecm:mixinTypes NOT IN ('Foo', 'Bar') 1110 * <p> 1111 * 1112 * <pre> 1113 * { "$and" : [ { "ecm:primaryType" : { "$in" : [ ... types without Foo nor Bar ...]}} , 1114 * { "ecm:mixinTypes" : { "$nin" : [ "Foo" , "Bar]}}]} 1115 * </pre> 1116 */ 1117 public DBObject walkMixinTypes(List<String> mixins, boolean include) { 1118 /* 1119 * Primary types that match. 1120 */ 1121 Set<String> matchPrimaryTypes; 1122 if (include) { 1123 matchPrimaryTypes = new HashSet<>(); 1124 for (String mixin : mixins) { 1125 matchPrimaryTypes.addAll(getMixinDocumentTypes(mixin)); 1126 } 1127 } else { 1128 matchPrimaryTypes = new HashSet<>(getDocumentTypes()); 1129 for (String mixin : mixins) { 1130 matchPrimaryTypes.removeAll(getMixinDocumentTypes(mixin)); 1131 } 1132 } 1133 /* 1134 * Instance mixins that match. 1135 */ 1136 Set<String> matchMixinTypes = new HashSet<>(); 1137 for (String mixin : mixins) { 1138 if (!isNeverPerInstanceMixin(mixin)) { 1139 matchMixinTypes.add(mixin); 1140 } 1141 } 1142 /* 1143 * MongoDB query generation. 1144 */ 1145 // match on primary type 1146 DBObject p = new BasicDBObject(DBSDocument.KEY_PRIMARY_TYPE, 1147 new BasicDBObject(QueryOperators.IN, matchPrimaryTypes)); 1148 // match on mixin types 1149 // $in/$nin with an array matches if any/no element of the array matches 1150 String innin = include ? QueryOperators.IN : QueryOperators.NIN; 1151 DBObject m = new BasicDBObject(DBSDocument.KEY_MIXIN_TYPES, new BasicDBObject(innin, matchMixinTypes)); 1152 // and/or between those 1153 String op = include ? QueryOperators.OR : QueryOperators.AND; 1154 return new BasicDBObject(op, Arrays.asList(p, m)); 1155 } 1156 1157}