001/* 002 * (C) Copyright 2014-2020 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.storage.mongodb; 020 021import static java.lang.Boolean.FALSE; 022import static java.lang.Boolean.TRUE; 023 024import java.time.ZonedDateTime; 025import java.util.ArrayList; 026import java.util.Arrays; 027import java.util.Date; 028import java.util.LinkedList; 029import java.util.List; 030import java.util.Set; 031import java.util.regex.Pattern; 032 033import org.bson.Document; 034import org.nuxeo.common.utils.DateUtils; 035import org.nuxeo.ecm.core.query.QueryParseException; 036import org.nuxeo.ecm.core.query.sql.NXQL; 037import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral; 038import org.nuxeo.ecm.core.query.sql.model.DateLiteral; 039import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral; 040import org.nuxeo.ecm.core.query.sql.model.Expression; 041import org.nuxeo.ecm.core.query.sql.model.Function; 042import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral; 043import org.nuxeo.ecm.core.query.sql.model.Literal; 044import org.nuxeo.ecm.core.query.sql.model.LiteralList; 045import org.nuxeo.ecm.core.query.sql.model.MultiExpression; 046import org.nuxeo.ecm.core.query.sql.model.Operand; 047import org.nuxeo.ecm.core.query.sql.model.Operator; 048import org.nuxeo.ecm.core.query.sql.model.Reference; 049import org.nuxeo.ecm.core.query.sql.model.StringLiteral; 050import org.nuxeo.ecm.core.schema.types.ListType; 051import org.nuxeo.ecm.core.schema.types.Type; 052import org.nuxeo.ecm.core.schema.types.primitives.BooleanType; 053import org.nuxeo.ecm.core.schema.types.primitives.DateType; 054import org.nuxeo.ecm.core.storage.ExpressionEvaluator; 055import org.nuxeo.ecm.core.storage.QueryOptimizer.PrefixInfo; 056import org.nuxeo.runtime.api.Framework; 057import org.nuxeo.runtime.mongodb.MongoDBOperators; 058import org.nuxeo.runtime.services.config.ConfigurationService; 059 060/** 061 * Abstract query builder for a MongoDB query from an {@link Expression}. 062 * <p> 063 * Must be customized by defining an implementation for the {@link #walkReference(String)} method. 064 * 065 * @since 5.9.4 066 */ 067public abstract class MongoDBAbstractQueryBuilder { 068 069 public static final Long LONG_ZERO = Long.valueOf(0); 070 071 public static final Long LONG_ONE = Long.valueOf(1); 072 073 public static final Double ONE = Double.valueOf(1); 074 075 public static final Double MINUS_ONE = Double.valueOf(-1); 076 077 protected static final String DATE_CAST = "DATE"; 078 079 protected static final String LIKE_ANCHORED_PROP = "nuxeo.mongodb.like.anchored"; 080 081 protected final MongoDBConverter converter; 082 083 protected final Expression expression; 084 085 protected Document query; 086 087 /** 088 * Prefix to remove for $elemMatch (including final dot), or {@code null} if there's no current prefix to remove. 089 */ 090 protected String elemMatchPrefix; 091 092 protected boolean likeAnchored; 093 094 public MongoDBAbstractQueryBuilder(MongoDBConverter converter, Expression expression) { 095 this.converter = converter; 096 this.expression = expression; 097 likeAnchored = Framework.getService(ConfigurationService.class).isBooleanTrue(LIKE_ANCHORED_PROP); 098 } 099 100 public void walk() { 101 if (expression instanceof MultiExpression && ((MultiExpression) expression).predicates.isEmpty()) { 102 // special-case empty query 103 query = new Document(); 104 } else { 105 query = walkExpression(expression); 106 } 107 } 108 109 public Document getQuery() { 110 return query; 111 } 112 113 public Document walkExpression(Expression expr) { 114 Operator op = expr.operator; 115 Operand lvalue = expr.lvalue; 116 Operand rvalue = expr.rvalue; 117 Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null; 118 String name = ref != null ? ref.name : null; 119 String cast = ref != null ? ref.cast : null; 120 if (DATE_CAST.equals(cast)) { 121 checkDateLiteralForCast(op, rvalue, name); 122 } 123 if (op == Operator.SUM) { 124 throw new UnsupportedOperationException("SUM"); 125 } else if (op == Operator.SUB) { 126 throw new UnsupportedOperationException("SUB"); 127 } else if (op == Operator.MUL) { 128 throw new UnsupportedOperationException("MUL"); 129 } else if (op == Operator.DIV) { 130 throw new UnsupportedOperationException("DIV"); 131 } else if (op == Operator.LT) { 132 return walkLt(lvalue, rvalue); 133 } else if (op == Operator.GT) { 134 return walkGt(lvalue, rvalue); 135 } else if (op == Operator.EQ) { 136 return walkEq(lvalue, rvalue); 137 } else if (op == Operator.NOTEQ) { 138 return walkNotEq(lvalue, rvalue); 139 } else if (op == Operator.LTEQ) { 140 return walkLtEq(lvalue, rvalue); 141 } else if (op == Operator.GTEQ) { 142 return walkGtEq(lvalue, rvalue); 143 } else if (op == Operator.AND || op == Operator.OR) { 144 if (expr instanceof MultiExpression) { 145 return walkAndOrMultiExpression((MultiExpression) expr); 146 } else { 147 return walkAndOr(expr); 148 } 149 } else if (op == Operator.NOT) { 150 return walkNot(lvalue); 151 } else if (op == Operator.LIKE) { 152 return walkLike(lvalue, rvalue, true, false); 153 } else if (op == Operator.ILIKE) { 154 return walkLike(lvalue, rvalue, true, true); 155 } else if (op == Operator.NOTLIKE) { 156 return walkLike(lvalue, rvalue, false, false); 157 } else if (op == Operator.NOTILIKE) { 158 return walkLike(lvalue, rvalue, false, true); 159 } else if (op == Operator.IN) { 160 return walkIn(lvalue, rvalue, true); 161 } else if (op == Operator.NOTIN) { 162 return walkIn(lvalue, rvalue, false); 163 } else if (op == Operator.ISNULL) { 164 return walkIsNull(lvalue); 165 } else if (op == Operator.ISNOTNULL) { 166 return walkIsNotNull(lvalue); 167 } else if (op == Operator.BETWEEN) { 168 return walkBetween(lvalue, rvalue, true); 169 } else if (op == Operator.NOTBETWEEN) { 170 return walkBetween(lvalue, rvalue, false); 171 } else { 172 throw new QueryParseException("Unknown operator: " + op); 173 } 174 } 175 176 protected void checkDateLiteralForCast(Operator op, Operand value, String name) { 177 if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) { 178 LiteralList l = (LiteralList) value; 179 checkDateLiteralForCast(l.get(0), name); 180 checkDateLiteralForCast(l.get(1), name); 181 } else { 182 checkDateLiteralForCast(value, name); 183 } 184 } 185 186 protected void checkDateLiteralForCast(Operand value, String name) { 187 if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) { 188 throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name); 189 } 190 } 191 192 public Document walkNot(Operand value) { 193 Object val = walkOperand(null, value); 194 Object not = pushDownNot(val); 195 if (!(not instanceof Document)) { 196 throw new QueryParseException("Cannot do NOT on: " + val); 197 } 198 return (Document) not; 199 } 200 201 protected Object pushDownNot(Object object) { 202 if (!(object instanceof Document)) { 203 throw new QueryParseException("Cannot do NOT on: " + object); 204 } 205 Document ob = (Document) object; 206 Set<String> keySet = ob.keySet(); 207 if (keySet.size() != 1) { 208 throw new QueryParseException("Cannot do NOT on: " + ob); 209 } 210 String key = keySet.iterator().next(); 211 Object value = ob.get(key); 212 if (!key.startsWith("$")) { 213 if (value instanceof Document) { 214 // push down inside dbobject 215 return new Document(key, pushDownNot(value)); 216 } else { 217 // k = v -> k != v 218 return new Document(key, new Document(MongoDBOperators.NE, value)); 219 } 220 } 221 if (MongoDBOperators.NE.equals(key)) { 222 // NOT k != v -> k = v 223 return value; 224 } 225 if (MongoDBOperators.NOT.equals(key)) { 226 // NOT NOT v -> v 227 return value; 228 } 229 if (MongoDBOperators.AND.equals(key) || MongoDBOperators.OR.equals(key)) { 230 // boolean algebra 231 // NOT (v1 AND v2) -> NOT v1 OR NOT v2 232 // NOT (v1 OR v2) -> NOT v1 AND NOT v2 233 String op = MongoDBOperators.AND.equals(key) ? MongoDBOperators.OR : MongoDBOperators.AND; 234 List<Object> list = (List<Object>) value; 235 for (int i = 0; i < list.size(); i++) { 236 list.set(i, pushDownNot(list.get(i))); 237 } 238 return new Document(op, list); 239 } 240 if (MongoDBOperators.IN.equals(key) || MongoDBOperators.NIN.equals(key)) { 241 // boolean algebra 242 // IN <-> NIN 243 String op = MongoDBOperators.IN.equals(key) ? MongoDBOperators.NIN : MongoDBOperators.IN; 244 return new Document(op, value); 245 } 246 if (MongoDBOperators.LT.equals(key) || MongoDBOperators.GT.equals(key) || MongoDBOperators.LTE.equals(key) 247 || MongoDBOperators.GTE.equals(key)) { 248 // TODO use inverse operators? 249 return new Document(MongoDBOperators.NOT, ob); 250 } 251 throw new QueryParseException("Unknown operator for NOT: " + key); 252 } 253 254 protected Document newDocumentWithField(FieldInfo fieldInfo, Object value) { 255 return new Document(fieldInfo.queryField, value); 256 } 257 258 public Document walkIsNull(Operand value) { 259 FieldInfo fieldInfo = walkReference(value); 260 return newDocumentWithField(fieldInfo, null); 261 } 262 263 public Document walkIsNotNull(Operand value) { 264 FieldInfo fieldInfo = walkReference(value); 265 return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.NE, null)); 266 } 267 268 public Document walkAndOrMultiExpression(MultiExpression expr) { 269 return walkAndOr(expr, expr.predicates); 270 } 271 272 public Document walkAndOr(Expression expr) { 273 return walkAndOr(expr, Arrays.asList(expr.lvalue, expr.rvalue)); 274 } 275 276 protected static final Pattern SLASH_WILDCARD_SLASH = Pattern.compile("/\\*\\d+(/)?"); 277 278 protected Document walkAndOr(Expression expr, List<? extends Operand> values) { 279 if (values.size() == 1) { 280 return (Document) walkOperand(null, values.get(0)); 281 } 282 boolean and = expr.operator == Operator.AND; 283 String op = and ? MongoDBOperators.AND : MongoDBOperators.OR; 284 // PrefixInfo was computed by the QueryOptimizer for common AND predicates 285 PrefixInfo info = (PrefixInfo) expr.getInfo(); 286 if (info == null || info.count < 2 || !and) { 287 List<Object> list = walkOperandList(values); 288 return new Document(op, list); 289 } 290 291 // we have a common prefix for all underlying references, extract it into an $elemMatch node 292 293 // info.prefix is the DBS common prefix, ex: foo/bar/*1; ecm:acp/*1/acl/*1 294 // compute MongoDB prefix: foo.bar.; ecm:acp.acl. 295 String prefix = SLASH_WILDCARD_SLASH.matcher(info.prefix).replaceAll("."); 296 // remove current prefix and trailing . for actual field match 297 String fieldBase = stripElemMatchPrefix(prefix.substring(0, prefix.length() - 1)); 298 299 String previousElemMatchPrefix = elemMatchPrefix; 300 elemMatchPrefix = prefix; 301 List<Object> list = walkOperandList(values); 302 elemMatchPrefix = previousElemMatchPrefix; 303 304 return new Document(fieldBase, new Document(MongoDBOperators.ELEM_MATCH, new Document(op, list))); 305 } 306 307 protected String stripElemMatchPrefix(String field) { 308 if (elemMatchPrefix != null && field.startsWith(elemMatchPrefix)) { 309 field = field.substring(elemMatchPrefix.length()); 310 } 311 return field; 312 } 313 314 public Document walkEq(Operand lvalue, Operand rvalue) { 315 FieldInfo fieldInfo = walkReference(lvalue); 316 return walkEq(fieldInfo, rvalue); 317 } 318 319 public Document walkEq(FieldInfo fieldInfo, Operand rvalue) { 320 Object right = walkOperand(fieldInfo, rvalue); 321 return newDocumentWithField(fieldInfo, right); 322 } 323 324 public Document walkNotEq(Operand lvalue, Operand rvalue) { 325 FieldInfo fieldInfo = walkReference(lvalue); 326 return walkNotEq(fieldInfo, rvalue); 327 } 328 329 public Document walkNotEq(FieldInfo fieldInfo, Operand rvalue) { 330 Object right = walkOperand(fieldInfo, rvalue); 331 return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.NE, right)); 332 } 333 334 public Document walkLt(Operand lvalue, Operand rvalue) { 335 FieldInfo fieldInfo = walkReference(lvalue); 336 Object right = walkOperand(fieldInfo, rvalue); 337 return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.LT, right)); 338 } 339 340 public Document walkGt(Operand lvalue, Operand rvalue) { 341 FieldInfo fieldInfo = walkReference(lvalue); 342 Object right = walkOperand(fieldInfo, rvalue); 343 return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.GT, right)); 344 } 345 346 public Document walkLtEq(Operand lvalue, Operand rvalue) { 347 FieldInfo fieldInfo = walkReference(lvalue); 348 Object right = walkOperand(fieldInfo, rvalue); 349 return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.LTE, right)); 350 } 351 352 public Document walkGtEq(Operand lvalue, Operand rvalue) { 353 FieldInfo fieldInfo = walkReference(lvalue); 354 Object right = walkOperand(fieldInfo, rvalue); 355 return newDocumentWithField(fieldInfo, new Document(MongoDBOperators.GTE, right)); 356 } 357 358 public Document walkBetween(Operand lvalue, Operand rvalue, boolean positive) { 359 LiteralList l = (LiteralList) rvalue; 360 FieldInfo fieldInfo = walkReference(lvalue); 361 Object left = walkOperand(fieldInfo, l.get(0)); 362 Object right = walkOperand(fieldInfo, l.get(1)); 363 if (positive) { 364 Document range = new Document(); 365 range.put(MongoDBOperators.GTE, left); 366 range.put(MongoDBOperators.LTE, right); 367 return newDocumentWithField(fieldInfo, range); 368 } else { 369 Document a = newDocumentWithField(fieldInfo, new Document(MongoDBOperators.LT, left)); 370 Document b = newDocumentWithField(fieldInfo, new Document(MongoDBOperators.GT, right)); 371 return new Document(MongoDBOperators.OR, Arrays.asList(a, b)); 372 } 373 } 374 375 public Document walkIn(Operand lvalue, Operand rvalue, boolean positive) { 376 FieldInfo fieldInfo = walkReference(lvalue); 377 return walkIn(fieldInfo, rvalue, positive); 378 } 379 380 public Document walkIn(FieldInfo fieldInfo, Operand rvalue, boolean positive) { 381 Object right = walkOperand(fieldInfo, rvalue); 382 if (!(right instanceof List)) { 383 throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue); 384 } 385 // TODO check list fields 386 List<Object> list = (List<Object>) right; 387 return newDocumentWithField(fieldInfo, 388 new Document(positive ? MongoDBOperators.IN : MongoDBOperators.NIN, list)); 389 } 390 391 public Document walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) { 392 FieldInfo fieldInfo = walkReference(lvalue); 393 if (!(rvalue instanceof StringLiteral)) { 394 throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue); 395 } 396 // TODO check list fields 397 String like = (String) walkStringLiteral(fieldInfo, (StringLiteral) rvalue); 398 String regex = ExpressionEvaluator.likeToRegex(like); 399 // MongoDB native matches are unanchored: optimize the regex for faster matches 400 if (regex.startsWith(".*")) { 401 regex = regex.substring(2); 402 } else if (likeAnchored) { 403 regex = "^" + regex; 404 } 405 if (regex.endsWith(".*")) { 406 regex = regex.substring(0, regex.length() - 2); // better range index use 407 } else if (likeAnchored) { 408 regex = regex + "$"; 409 } 410 411 int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0; 412 Pattern pattern = Pattern.compile(regex, flags); 413 Object value; 414 if (positive) { 415 value = pattern; 416 } else { 417 value = new Document(MongoDBOperators.NOT, pattern); 418 } 419 return newDocumentWithField(fieldInfo, value); 420 } 421 422 public Object walkOperand(FieldInfo fieldInfo, Operand op) { 423 if (op instanceof Literal) { 424 return walkLiteral(fieldInfo, (Literal) op); 425 } else if (op instanceof LiteralList) { 426 return walkLiteralList(fieldInfo, (LiteralList) op); 427 } else if (op instanceof Function) { 428 return walkFunction((Function) op); 429 } else if (op instanceof Expression) { 430 return walkExpression((Expression) op); 431 } else if (op instanceof Reference) { 432 return walkReference((Reference) op); 433 } else { 434 throw new QueryParseException("Unknown operand: " + op); 435 } 436 } 437 438 public Object walkLiteral(FieldInfo fieldInfo, Literal lit) { 439 if (lit instanceof BooleanLiteral) { 440 return walkBooleanLiteral(fieldInfo, (BooleanLiteral) lit); 441 } else if (lit instanceof DateLiteral) { 442 return walkDateLiteral(fieldInfo, (DateLiteral) lit); 443 } else if (lit instanceof DoubleLiteral) { 444 return walkDoubleLiteral(fieldInfo, (DoubleLiteral) lit); 445 } else if (lit instanceof IntegerLiteral) { 446 return walkIntegerLiteral(fieldInfo, (IntegerLiteral) lit); 447 } else if (lit instanceof StringLiteral) { 448 return walkStringLiteral(fieldInfo, (StringLiteral) lit); 449 } else { 450 throw new QueryParseException("Unknown literal: " + lit); 451 } 452 } 453 454 public Object walkBooleanLiteral(FieldInfo fieldInfo, BooleanLiteral lit) { 455 return Boolean.valueOf(lit.value); 456 } 457 458 public Date walkDateLiteral(FieldInfo fieldInfo, DateLiteral lit) { 459 return DateUtils.toDate(lit.value); // TODO onlyDate 460 } 461 462 public Double walkDoubleLiteral(FieldInfo fieldInfo, DoubleLiteral lit) { 463 return Double.valueOf(lit.value); 464 } 465 466 public Object walkIntegerLiteral(FieldInfo fieldInfo, IntegerLiteral lit) { 467 long value = lit.value; 468 if (fieldInfo != null && fieldInfo.isBoolean()) { 469 // convert 0 / 1 to actual booleans 470 Boolean b; 471 if (value == 0) { 472 b = FALSE; 473 } else if (value == 1) { 474 b = TRUE; 475 } else { 476 throw new QueryParseException("Invalid boolean: " + value); 477 } 478 return converter.serializableToBson(fieldInfo.key, b); 479 } 480 return Long.valueOf(value); 481 } 482 483 public Object walkStringLiteral(FieldInfo fieldInfo, StringLiteral lit) { 484 String value = lit.value; 485 if (fieldInfo != null) { 486 return converter.serializableToBson(fieldInfo.key, value); 487 } 488 return value; 489 } 490 491 public List<Object> walkLiteralList(FieldInfo fieldInfo, LiteralList litList) { 492 List<Object> list = new ArrayList<>(litList.size()); 493 for (Literal lit : litList) { 494 list.add(walkLiteral(fieldInfo, lit)); 495 } 496 return list; 497 } 498 499 protected List<Object> walkOperandList(List<? extends Operand> values) { 500 List<Object> list = new LinkedList<>(); 501 for (Operand value : values) { 502 list.add(walkOperand(null, value)); 503 } 504 return list; 505 } 506 507 public Object walkFunction(Function func) { 508 String name = func.name; 509 if (NXQL.NOW_FUNCTION.equalsIgnoreCase(name)) { 510 String periodAndDurationText; 511 if (func.args == null || func.args.size() != 1) { 512 periodAndDurationText = null; 513 } else { 514 periodAndDurationText = ((StringLiteral) func.args.get(0)).value; 515 } 516 ZonedDateTime dateTime; 517 try { 518 dateTime = NXQL.nowPlusPeriodAndDuration(periodAndDurationText); 519 } catch (IllegalArgumentException e) { 520 throw new QueryParseException(e); 521 } 522 DateLiteral dateLiteral = new DateLiteral(dateTime); 523 return walkDateLiteral(null, dateLiteral); 524 } else { 525 throw new QueryParseException("Function not supported: " + func); 526 } 527 } 528 529 protected FieldInfo walkReference(Operand value) { 530 if (!(value instanceof Reference)) { 531 throw new QueryParseException("Invalid query, left hand side must be a property: " + value); 532 } 533 return walkReference((Reference) value); 534 } 535 536 public static class FieldInfo { 537 538 /** NXQL property. */ 539 public final String prop; 540 541 /** 542 * DBS key. 543 * 544 * @since 11.1 545 */ 546 public final String key; 547 548 /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */ 549 public final String queryField; 550 551 /** MongoDB field for projection. */ 552 public final String projectionField; 553 554 public final Type type; 555 556 public FieldInfo(String prop, String key, String queryField, String projectionField, Type type) { 557 this.prop = prop; 558 this.key = key; 559 this.queryField = queryField; 560 this.projectionField = projectionField; 561 this.type = type; 562 } 563 564 public boolean isBoolean() { 565 return type instanceof BooleanType; 566 } 567 } 568 569 /** 570 * Returns the MongoDB field for this reference. 571 */ 572 public FieldInfo walkReference(Reference ref) { 573 FieldInfo fieldInfo = walkReference(ref.name); 574 if (DATE_CAST.equals(ref.cast)) { 575 Type type = fieldInfo.type; 576 if (!(type instanceof DateType 577 || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) { 578 throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name); 579 } 580 // fieldInfo.isDateCast = true; 581 } 582 return fieldInfo; 583 } 584 585 /** 586 * Walks a reference, and returns field info about it. 587 */ 588 protected abstract FieldInfo walkReference(String name); 589 590}