001/* 002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.storage.mongodb; 020 021import static java.lang.Boolean.FALSE; 022import static java.lang.Boolean.TRUE; 023 024import java.util.ArrayList; 025import java.util.Arrays; 026import java.util.Date; 027import java.util.HashMap; 028import java.util.LinkedList; 029import java.util.List; 030import java.util.Map; 031import java.util.Set; 032import java.util.regex.Pattern; 033 034import org.bson.Document; 035import org.nuxeo.ecm.core.query.QueryParseException; 036import org.nuxeo.ecm.core.query.sql.model.BooleanLiteral; 037import org.nuxeo.ecm.core.query.sql.model.DateLiteral; 038import org.nuxeo.ecm.core.query.sql.model.DoubleLiteral; 039import org.nuxeo.ecm.core.query.sql.model.Expression; 040import org.nuxeo.ecm.core.query.sql.model.Function; 041import org.nuxeo.ecm.core.query.sql.model.IntegerLiteral; 042import org.nuxeo.ecm.core.query.sql.model.Literal; 043import org.nuxeo.ecm.core.query.sql.model.LiteralList; 044import org.nuxeo.ecm.core.query.sql.model.MultiExpression; 045import org.nuxeo.ecm.core.query.sql.model.Operand; 046import org.nuxeo.ecm.core.query.sql.model.Operator; 047import org.nuxeo.ecm.core.query.sql.model.Reference; 048import org.nuxeo.ecm.core.query.sql.model.StringLiteral; 049import org.nuxeo.ecm.core.schema.types.ListType; 050import org.nuxeo.ecm.core.schema.types.Type; 051import org.nuxeo.ecm.core.schema.types.primitives.BooleanType; 052import org.nuxeo.ecm.core.schema.types.primitives.DateType; 053import org.nuxeo.ecm.core.storage.ExpressionEvaluator; 054import org.nuxeo.ecm.core.storage.QueryOptimizer.PrefixInfo; 055import org.nuxeo.runtime.api.Framework; 056import org.nuxeo.runtime.services.config.ConfigurationService; 057 058import com.mongodb.QueryOperators; 059 060/** 061 * Abstract query builder for a MongoDB query from an {@link Expression}. 062 * <p> 063 * Must be customized by defining an implementation for the {@link #walkReference(String)} method. 064 * 065 * @since 5.9.4 066 */ 067public abstract class MongoDBAbstractQueryBuilder { 068 069 public static final Long LONG_ZERO = Long.valueOf(0); 070 071 public static final Long LONG_ONE = Long.valueOf(1); 072 073 public static final Double ONE = Double.valueOf(1); 074 075 public static final Double MINUS_ONE = Double.valueOf(-1); 076 077 protected static final String DATE_CAST = "DATE"; 078 079 protected static final String LIKE_ANCHORED_PROP = "nuxeo.mongodb.like.anchored"; 080 081 protected final MongoDBConverter converter; 082 083 protected final Expression expression; 084 085 protected Document query; 086 087 /** 088 * Prefix to remove for $elemMatch (including final dot), or {@code null} if there's no current prefix to remove. 089 */ 090 protected String elemMatchPrefix; 091 092 protected boolean likeAnchored; 093 094 public MongoDBAbstractQueryBuilder(MongoDBConverter converter, Expression expression) { 095 this.converter = converter; 096 this.expression = expression; 097 likeAnchored = !Framework.getService(ConfigurationService.class).isBooleanPropertyFalse(LIKE_ANCHORED_PROP); 098 } 099 100 public void walk() { 101 if (expression instanceof MultiExpression && ((MultiExpression) expression).predicates.isEmpty()) { 102 // special-case empty query 103 query = new Document(); 104 } else { 105 query = walkExpression(expression); 106 } 107 } 108 109 public Document getQuery() { 110 return query; 111 } 112 113 public Document walkExpression(Expression expr) { 114 Operator op = expr.operator; 115 Operand lvalue = expr.lvalue; 116 Operand rvalue = expr.rvalue; 117 Reference ref = lvalue instanceof Reference ? (Reference) lvalue : null; 118 String name = ref != null ? ref.name : null; 119 String cast = ref != null ? ref.cast : null; 120 if (DATE_CAST.equals(cast)) { 121 checkDateLiteralForCast(op, rvalue, name); 122 } 123 if (op == Operator.SUM) { 124 throw new UnsupportedOperationException("SUM"); 125 } else if (op == Operator.SUB) { 126 throw new UnsupportedOperationException("SUB"); 127 } else if (op == Operator.MUL) { 128 throw new UnsupportedOperationException("MUL"); 129 } else if (op == Operator.DIV) { 130 throw new UnsupportedOperationException("DIV"); 131 } else if (op == Operator.LT) { 132 return walkLt(lvalue, rvalue); 133 } else if (op == Operator.GT) { 134 return walkGt(lvalue, rvalue); 135 } else if (op == Operator.EQ) { 136 return walkEq(lvalue, rvalue); 137 } else if (op == Operator.NOTEQ) { 138 return walkNotEq(lvalue, rvalue); 139 } else if (op == Operator.LTEQ) { 140 return walkLtEq(lvalue, rvalue); 141 } else if (op == Operator.GTEQ) { 142 return walkGtEq(lvalue, rvalue); 143 } else if (op == Operator.AND || op == Operator.OR) { 144 if (expr instanceof MultiExpression) { 145 return walkAndOrMultiExpression((MultiExpression) expr); 146 } else { 147 return walkAndOr(expr); 148 } 149 } else if (op == Operator.NOT) { 150 return walkNot(lvalue); 151 } else if (op == Operator.LIKE) { 152 return walkLike(lvalue, rvalue, true, false); 153 } else if (op == Operator.ILIKE) { 154 return walkLike(lvalue, rvalue, true, true); 155 } else if (op == Operator.NOTLIKE) { 156 return walkLike(lvalue, rvalue, false, false); 157 } else if (op == Operator.NOTILIKE) { 158 return walkLike(lvalue, rvalue, false, true); 159 } else if (op == Operator.IN) { 160 return walkIn(lvalue, rvalue, true); 161 } else if (op == Operator.NOTIN) { 162 return walkIn(lvalue, rvalue, false); 163 } else if (op == Operator.ISNULL) { 164 return walkIsNull(lvalue); 165 } else if (op == Operator.ISNOTNULL) { 166 return walkIsNotNull(lvalue); 167 } else if (op == Operator.BETWEEN) { 168 return walkBetween(lvalue, rvalue, true); 169 } else if (op == Operator.NOTBETWEEN) { 170 return walkBetween(lvalue, rvalue, false); 171 } else { 172 throw new QueryParseException("Unknown operator: " + op); 173 } 174 } 175 176 protected void checkDateLiteralForCast(Operator op, Operand value, String name) { 177 if (op == Operator.BETWEEN || op == Operator.NOTBETWEEN) { 178 LiteralList l = (LiteralList) value; 179 checkDateLiteralForCast(l.get(0), name); 180 checkDateLiteralForCast(l.get(1), name); 181 } else { 182 checkDateLiteralForCast(value, name); 183 } 184 } 185 186 protected void checkDateLiteralForCast(Operand value, String name) { 187 if (value instanceof DateLiteral && !((DateLiteral) value).onlyDate) { 188 throw new QueryParseException("DATE() cast must be used with DATE literal, not TIMESTAMP: " + name); 189 } 190 } 191 192 public Document walkNot(Operand value) { 193 Object val = walkOperand(value); 194 Object not = pushDownNot(val); 195 if (!(not instanceof Document)) { 196 throw new QueryParseException("Cannot do NOT on: " + val); 197 } 198 return (Document) not; 199 } 200 201 protected Object pushDownNot(Object object) { 202 if (!(object instanceof Document)) { 203 throw new QueryParseException("Cannot do NOT on: " + object); 204 } 205 Document ob = (Document) object; 206 Set<String> keySet = ob.keySet(); 207 if (keySet.size() != 1) { 208 throw new QueryParseException("Cannot do NOT on: " + ob); 209 } 210 String key = keySet.iterator().next(); 211 Object value = ob.get(key); 212 if (!key.startsWith("$")) { 213 if (value instanceof Document) { 214 // push down inside dbobject 215 return new Document(key, pushDownNot(value)); 216 } else { 217 // k = v -> k != v 218 return new Document(key, new Document(QueryOperators.NE, value)); 219 } 220 } 221 if (QueryOperators.NE.equals(key)) { 222 // NOT k != v -> k = v 223 return value; 224 } 225 if (QueryOperators.NOT.equals(key)) { 226 // NOT NOT v -> v 227 return value; 228 } 229 if (QueryOperators.AND.equals(key) || QueryOperators.OR.equals(key)) { 230 // boolean algebra 231 // NOT (v1 AND v2) -> NOT v1 OR NOT v2 232 // NOT (v1 OR v2) -> NOT v1 AND NOT v2 233 String op = QueryOperators.AND.equals(key) ? QueryOperators.OR : QueryOperators.AND; 234 List<Object> list = (List<Object>) value; 235 for (int i = 0; i < list.size(); i++) { 236 list.set(i, pushDownNot(list.get(i))); 237 } 238 return new Document(op, list); 239 } 240 if (QueryOperators.IN.equals(key) || QueryOperators.NIN.equals(key)) { 241 // boolean algebra 242 // IN <-> NIN 243 String op = QueryOperators.IN.equals(key) ? QueryOperators.NIN : QueryOperators.IN; 244 return new Document(op, value); 245 } 246 if (QueryOperators.LT.equals(key) || QueryOperators.GT.equals(key) || QueryOperators.LTE.equals(key) 247 || QueryOperators.GTE.equals(key)) { 248 // TODO use inverse operators? 249 return new Document(QueryOperators.NOT, ob); 250 } 251 throw new QueryParseException("Unknown operator for NOT: " + key); 252 } 253 254 protected Document newDocumentWithField(FieldInfo fieldInfo, Object value) { 255 return new Document(fieldInfo.queryField, value); 256 } 257 258 public Document walkIsNull(Operand value) { 259 FieldInfo fieldInfo = walkReference(value); 260 return newDocumentWithField(fieldInfo, null); 261 } 262 263 public Document walkIsNotNull(Operand value) { 264 FieldInfo fieldInfo = walkReference(value); 265 return newDocumentWithField(fieldInfo, new Document(QueryOperators.NE, null)); 266 } 267 268 public Document walkAndOrMultiExpression(MultiExpression expr) { 269 return walkAndOr(expr, expr.predicates); 270 } 271 272 public Document walkAndOr(Expression expr) { 273 return walkAndOr(expr, Arrays.asList(expr.lvalue, expr.rvalue)); 274 } 275 276 protected static final Pattern SLASH_WILDCARD_SLASH = Pattern.compile("/\\*\\d+(/)?"); 277 278 protected Document walkAndOr(Expression expr, List<? extends Operand> values) { 279 if (values.size() == 1) { 280 return (Document) walkOperand(values.get(0)); 281 } 282 boolean and = expr.operator == Operator.AND; 283 String op = and ? QueryOperators.AND : QueryOperators.OR; 284 // PrefixInfo was computed by the QueryOptimizer for common AND predicates 285 PrefixInfo info = (PrefixInfo) expr.getInfo(); 286 if (info == null || info.count < 2 || !and) { 287 List<Object> list = walkOperandList(values); 288 return new Document(op, list); 289 } 290 291 // we have a common prefix for all underlying references, extract it into an $elemMatch node 292 293 // info.prefix is the DBS common prefix, ex: foo/bar/*1; ecm:acp/*1/acl/*1 294 // compute MongoDB prefix: foo.bar.; ecm:acp.acl. 295 String prefix = SLASH_WILDCARD_SLASH.matcher(info.prefix).replaceAll("."); 296 // remove current prefix and trailing . for actual field match 297 String fieldBase = stripElemMatchPrefix(prefix.substring(0, prefix.length() - 1)); 298 299 String previousElemMatchPrefix = elemMatchPrefix; 300 elemMatchPrefix = prefix; 301 List<Object> list = walkOperandList(values); 302 elemMatchPrefix = previousElemMatchPrefix; 303 304 return new Document(fieldBase, new Document(QueryOperators.ELEM_MATCH, new Document(op, list))); 305 } 306 307 protected String stripElemMatchPrefix(String field) { 308 if (elemMatchPrefix != null && field.startsWith(elemMatchPrefix)) { 309 field = field.substring(elemMatchPrefix.length()); 310 } 311 return field; 312 } 313 314 protected Object checkBoolean(FieldInfo fieldInfo, Object right) { 315 if (fieldInfo.isBoolean()) { 316 // convert 0 / 1 to actual booleans 317 if (right instanceof Long) { 318 if (LONG_ZERO.equals(right)) { 319 right = fieldInfo.isTrueOrNullBoolean ? null : FALSE; 320 } else if (LONG_ONE.equals(right)) { 321 right = TRUE; 322 } else { 323 throw new QueryParseException("Invalid boolean: " + right); 324 } 325 } 326 } 327 return right; 328 } 329 330 public Document walkEq(Operand lvalue, Operand rvalue) { 331 FieldInfo fieldInfo = walkReference(lvalue); 332 Object right = walkOperand(rvalue); 333 right = checkBoolean(fieldInfo, right); 334 // TODO check list fields 335 return newDocumentWithField(fieldInfo, right); 336 } 337 338 public Document walkNotEq(Operand lvalue, Operand rvalue) { 339 FieldInfo fieldInfo = walkReference(lvalue); 340 Object right = walkOperand(rvalue); 341 right = checkBoolean(fieldInfo, right); 342 // TODO check list fields 343 return newDocumentWithField(fieldInfo, new Document(QueryOperators.NE, right)); 344 } 345 346 public Document walkLt(Operand lvalue, Operand rvalue) { 347 FieldInfo fieldInfo = walkReference(lvalue); 348 Object right = walkOperand(rvalue); 349 return newDocumentWithField(fieldInfo, new Document(QueryOperators.LT, right)); 350 } 351 352 public Document walkGt(Operand lvalue, Operand rvalue) { 353 FieldInfo fieldInfo = walkReference(lvalue); 354 Object right = walkOperand(rvalue); 355 return newDocumentWithField(fieldInfo, new Document(QueryOperators.GT, right)); 356 } 357 358 public Document walkLtEq(Operand lvalue, Operand rvalue) { 359 FieldInfo fieldInfo = walkReference(lvalue); 360 Object right = walkOperand(rvalue); 361 return newDocumentWithField(fieldInfo, new Document(QueryOperators.LTE, right)); 362 } 363 364 public Document walkGtEq(Operand lvalue, Operand rvalue) { 365 FieldInfo fieldInfo = walkReference(lvalue); 366 Object right = walkOperand(rvalue); 367 return newDocumentWithField(fieldInfo, new Document(QueryOperators.GTE, right)); 368 } 369 370 public Document walkBetween(Operand lvalue, Operand rvalue, boolean positive) { 371 LiteralList l = (LiteralList) rvalue; 372 FieldInfo fieldInfo = walkReference(lvalue); 373 Object left = walkOperand(l.get(0)); 374 Object right = walkOperand(l.get(1)); 375 if (positive) { 376 Document range = new Document(); 377 range.put(QueryOperators.GTE, left); 378 range.put(QueryOperators.LTE, right); 379 return newDocumentWithField(fieldInfo, range); 380 } else { 381 Document a = newDocumentWithField(fieldInfo, new Document(QueryOperators.LT, left)); 382 Document b = newDocumentWithField(fieldInfo, new Document(QueryOperators.GT, right)); 383 return new Document(QueryOperators.OR, Arrays.asList(a, b)); 384 } 385 } 386 387 public Document walkIn(Operand lvalue, Operand rvalue, boolean positive) { 388 FieldInfo fieldInfo = walkReference(lvalue); 389 Object right = walkOperand(rvalue); 390 if (!(right instanceof List)) { 391 throw new QueryParseException("Invalid IN, right hand side must be a list: " + rvalue); 392 } 393 // TODO check list fields 394 List<Object> list = (List<Object>) right; 395 return newDocumentWithField(fieldInfo, new Document(positive ? QueryOperators.IN : QueryOperators.NIN, list)); 396 } 397 398 public Document walkLike(Operand lvalue, Operand rvalue, boolean positive, boolean caseInsensitive) { 399 FieldInfo fieldInfo = walkReference(lvalue); 400 if (!(rvalue instanceof StringLiteral)) { 401 throw new QueryParseException("Invalid LIKE/ILIKE, right hand side must be a string: " + rvalue); 402 } 403 // TODO check list fields 404 String like = walkStringLiteral((StringLiteral) rvalue); 405 String regex = ExpressionEvaluator.likeToRegex(like); 406 // MongoDB native matches are unanchored: optimize the regex for faster matches 407 if (regex.startsWith(".*")) { 408 regex = regex.substring(2); 409 } else if (likeAnchored) { 410 regex = "^" + regex; 411 } 412 if (regex.endsWith(".*")) { 413 regex = regex.substring(0, regex.length() - 2); // better range index use 414 } else if (likeAnchored) { 415 regex = regex + "$"; 416 } 417 418 int flags = caseInsensitive ? Pattern.CASE_INSENSITIVE : 0; 419 Pattern pattern = Pattern.compile(regex, flags); 420 Object value; 421 if (positive) { 422 value = pattern; 423 } else { 424 value = new Document(QueryOperators.NOT, pattern); 425 } 426 return newDocumentWithField(fieldInfo, value); 427 } 428 429 public Object walkOperand(Operand op) { 430 if (op instanceof Literal) { 431 return walkLiteral((Literal) op); 432 } else if (op instanceof LiteralList) { 433 return walkLiteralList((LiteralList) op); 434 } else if (op instanceof Function) { 435 return walkFunction((Function) op); 436 } else if (op instanceof Expression) { 437 return walkExpression((Expression) op); 438 } else if (op instanceof Reference) { 439 return walkReference((Reference) op); 440 } else { 441 throw new QueryParseException("Unknown operand: " + op); 442 } 443 } 444 445 public Object walkLiteral(Literal lit) { 446 if (lit instanceof BooleanLiteral) { 447 return walkBooleanLiteral((BooleanLiteral) lit); 448 } else if (lit instanceof DateLiteral) { 449 return walkDateLiteral((DateLiteral) lit); 450 } else if (lit instanceof DoubleLiteral) { 451 return walkDoubleLiteral((DoubleLiteral) lit); 452 } else if (lit instanceof IntegerLiteral) { 453 return walkIntegerLiteral((IntegerLiteral) lit); 454 } else if (lit instanceof StringLiteral) { 455 return walkStringLiteral((StringLiteral) lit); 456 } else { 457 throw new QueryParseException("Unknown literal: " + lit); 458 } 459 } 460 461 public Object walkBooleanLiteral(BooleanLiteral lit) { 462 return Boolean.valueOf(lit.value); 463 } 464 465 public Date walkDateLiteral(DateLiteral lit) { 466 return lit.value.toDate(); // TODO onlyDate 467 } 468 469 public Double walkDoubleLiteral(DoubleLiteral lit) { 470 return Double.valueOf(lit.value); 471 } 472 473 public Long walkIntegerLiteral(IntegerLiteral lit) { 474 return Long.valueOf(lit.value); 475 } 476 477 public String walkStringLiteral(StringLiteral lit) { 478 return lit.value; 479 } 480 481 public List<Object> walkLiteralList(LiteralList litList) { 482 List<Object> list = new ArrayList<>(litList.size()); 483 for (Literal lit : litList) { 484 list.add(walkLiteral(lit)); 485 } 486 return list; 487 } 488 489 protected List<Object> walkOperandList(List<? extends Operand> values) { 490 List<Object> list = new LinkedList<>(); 491 for (Operand value : values) { 492 list.add(walkOperand(value)); 493 } 494 return list; 495 } 496 497 public Object walkFunction(Function func) { 498 throw new UnsupportedOperationException(func.name); 499 } 500 501 protected FieldInfo walkReference(Operand value) { 502 if (!(value instanceof Reference)) { 503 throw new QueryParseException("Invalid query, left hand side must be a property: " + value); 504 } 505 return walkReference((Reference) value); 506 } 507 508 public static class FieldInfo { 509 510 /** NXQL property. */ 511 public final String prop; 512 513 /** MongoDB field for query. foo/0/bar -> foo.0.bar; foo / * / bar -> foo.bar */ 514 public final String queryField; 515 516 /** MongoDB field for projection. */ 517 public final String projectionField; 518 519 public final Type type; 520 521 /** 522 * Boolean system properties only use TRUE or NULL, not FALSE, so queries must be updated accordingly. 523 */ 524 public final boolean isTrueOrNullBoolean; 525 526 public FieldInfo(String prop, String queryField, String projectionField, Type type, 527 boolean isTrueOrNullBoolean) { 528 this.prop = prop; 529 this.queryField = queryField; 530 this.projectionField = projectionField; 531 this.type = type; 532 this.isTrueOrNullBoolean = isTrueOrNullBoolean; 533 } 534 535 public boolean isBoolean() { 536 return type instanceof BooleanType; 537 } 538 } 539 540 /** 541 * Returns the MongoDB field for this reference. 542 */ 543 public FieldInfo walkReference(Reference ref) { 544 FieldInfo fieldInfo = walkReference(ref.name); 545 if (DATE_CAST.equals(ref.cast)) { 546 Type type = fieldInfo.type; 547 if (!(type instanceof DateType 548 || (type instanceof ListType && ((ListType) type).getFieldType() instanceof DateType))) { 549 throw new QueryParseException("Cannot cast to " + ref.cast + ": " + ref.name); 550 } 551 // fieldInfo.isDateCast = true; 552 } 553 return fieldInfo; 554 } 555 556 /** 557 * Walks a reference, and returns field info about it. 558 */ 559 protected abstract FieldInfo walkReference(String name); 560 561}