001/*
002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Tiry
018 *     bdelbosc
019 */
020package org.nuxeo.elasticsearch.query;
021
022import static org.elasticsearch.common.xcontent.DeprecationHandler.THROW_UNSUPPORTED_OPERATION;
023import static org.nuxeo.elasticsearch.ElasticSearchConstants.DOC_TYPE;
024import static org.nuxeo.elasticsearch.ElasticSearchConstants.ES_SCORE_FIELD;
025import static org.nuxeo.elasticsearch.ElasticSearchConstants.FULLTEXT_FIELD;
026
027import java.io.ByteArrayOutputStream;
028import java.io.IOException;
029import java.io.StringReader;
030import java.util.ArrayList;
031import java.util.Collection;
032import java.util.Iterator;
033import java.util.LinkedHashMap;
034import java.util.LinkedList;
035import java.util.List;
036import java.util.Map;
037import java.util.Set;
038
039import org.apache.commons.lang3.StringUtils;
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042import org.elasticsearch.common.geo.GeoPoint;
043import org.elasticsearch.common.geo.GeoUtils;
044import org.elasticsearch.common.geo.ShapeRelation;
045import org.elasticsearch.common.xcontent.NamedXContentRegistry;
046import org.elasticsearch.common.xcontent.XContentBuilder;
047import org.elasticsearch.common.xcontent.XContentParser;
048import org.elasticsearch.common.xcontent.json.JsonXContent;
049import org.elasticsearch.index.query.BoolQueryBuilder;
050import org.elasticsearch.index.query.CommonTermsQueryBuilder;
051import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
052import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
053import org.elasticsearch.index.query.MatchQueryBuilder;
054import org.elasticsearch.index.query.MoreLikeThisQueryBuilder;
055import org.elasticsearch.index.query.MultiMatchQueryBuilder;
056import org.elasticsearch.index.query.QueryBuilder;
057import org.elasticsearch.index.query.QueryBuilders;
058import org.elasticsearch.index.query.QueryStringQueryBuilder;
059import org.elasticsearch.index.query.SimpleQueryStringBuilder;
060import org.nuxeo.ecm.core.NXCore;
061import org.nuxeo.ecm.core.api.CoreSession;
062import org.nuxeo.ecm.core.api.DocumentModel;
063import org.nuxeo.ecm.core.api.DocumentNotFoundException;
064import org.nuxeo.ecm.core.api.IdRef;
065import org.nuxeo.ecm.core.api.LifeCycleConstants;
066import org.nuxeo.ecm.core.api.SortInfo;
067import org.nuxeo.ecm.core.api.repository.RepositoryManager;
068import org.nuxeo.ecm.core.api.trash.TrashService;
069import org.nuxeo.ecm.core.api.trash.TrashService.Feature;
070import org.nuxeo.ecm.core.query.QueryParseException;
071import org.nuxeo.ecm.core.query.sql.NXQL;
072import org.nuxeo.ecm.core.query.sql.SQLQueryParser;
073import org.nuxeo.ecm.core.query.sql.model.DefaultQueryVisitor;
074import org.nuxeo.ecm.core.query.sql.model.EsHint;
075import org.nuxeo.ecm.core.query.sql.model.Expression;
076import org.nuxeo.ecm.core.query.sql.model.FromClause;
077import org.nuxeo.ecm.core.query.sql.model.FromList;
078import org.nuxeo.ecm.core.query.sql.model.Literal;
079import org.nuxeo.ecm.core.query.sql.model.LiteralList;
080import org.nuxeo.ecm.core.query.sql.model.MultiExpression;
081import org.nuxeo.ecm.core.query.sql.model.Operand;
082import org.nuxeo.ecm.core.query.sql.model.Operator;
083import org.nuxeo.ecm.core.query.sql.model.OrderByExpr;
084import org.nuxeo.ecm.core.query.sql.model.Predicate;
085import org.nuxeo.ecm.core.query.sql.model.Reference;
086import org.nuxeo.ecm.core.query.sql.model.SQLQuery;
087import org.nuxeo.ecm.core.query.sql.model.SelectClause;
088import org.nuxeo.ecm.core.schema.SchemaManager;
089import org.nuxeo.ecm.core.schema.types.Field;
090import org.nuxeo.ecm.core.schema.types.Type;
091import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
092import org.nuxeo.ecm.core.storage.sql.jdbc.NXQLQueryMaker;
093import org.nuxeo.elasticsearch.api.ElasticSearchAdmin;
094import org.nuxeo.runtime.api.Framework;
095
096/**
097 * Helper class that holds the conversion logic. Conversion is based on the existing NXQL Parser, we are just using a
098 * visitor to build the ES request.
099 */
100public final class NxqlQueryConverter {
101    private static final Log log = LogFactory.getLog(NxqlQueryConverter.class);
102
103    private static final String SELECT_ALL = "SELECT * FROM Document";
104
105    private static final String SELECT_ALL_WHERE = "SELECT * FROM Document WHERE ";
106
107    private static final String SIMPLE_QUERY_PREFIX = "es: ";
108
109    protected static final int MORE_LIKE_THIS_MIN_TERM_FREQ = 1;
110
111    protected static final int MORE_LIKE_THIS_MIN_DOC_FREQ = 3;
112
113    protected static final int MORE_LIKE_THIS_MAX_QUERY_TERMS = 12;
114
115    private NxqlQueryConverter() {
116    }
117
118    public static QueryBuilder toESQueryBuilder(final String nxql) {
119        return toESQueryBuilder(nxql, null);
120    }
121
122    public static QueryBuilder toESQueryBuilder(final String nxql, final CoreSession session) {
123        final LinkedList<ExpressionBuilder> builders = new LinkedList<>();
124        SQLQuery nxqlQuery = getSqlQuery(nxql);
125        if (session != null) {
126            nxqlQuery = addSecurityPolicy(session, nxqlQuery);
127        }
128        final ExpressionBuilder ret = new ExpressionBuilder(null);
129        builders.add(ret);
130        final ArrayList<String> fromList = new ArrayList<>();
131        nxqlQuery.accept(new DefaultQueryVisitor() {
132
133            @Override
134            public void visitFromClause(FromClause node) {
135                FromList elements = node.elements;
136                SchemaManager schemaManager = Framework.getService(SchemaManager.class);
137
138                for (String type : elements.values()) {
139                    if (NXQLQueryMaker.TYPE_DOCUMENT.equalsIgnoreCase(type)) {
140                        // From Document means all doc types
141                        fromList.clear();
142                        return;
143                    }
144                    Set<String> types = schemaManager.getDocumentTypeNamesExtending(type);
145                    if (types != null) {
146                        fromList.addAll(types);
147                    }
148                }
149            }
150
151            @Override
152            public void visitMultiExpression(MultiExpression node) {
153                for (Iterator<Predicate> it = node.predicates.iterator(); it.hasNext();) {
154                    it.next().accept(this);
155                    if (it.hasNext()) {
156                        node.operator.accept(this);
157                    }
158                }
159            }
160
161            @Override
162            public void visitSelectClause(SelectClause node) {
163                // NOP
164            }
165
166            @Override
167            public void visitExpression(Expression node) {
168                Operator op = node.operator;
169                if (op == Operator.AND || op == Operator.OR || op == Operator.NOT) {
170                    builders.add(new ExpressionBuilder(op.toString()));
171                    super.visitExpression(node);
172                    ExpressionBuilder expr = builders.removeLast();
173                    if (!builders.isEmpty()) {
174                        builders.getLast().merge(expr);
175                    }
176                } else {
177                    Reference ref = node.lvalue instanceof Reference ? (Reference) node.lvalue : null;
178                    String name = ref != null ? ref.name : node.lvalue.toString();
179                    String value = null;
180                    if (node.rvalue instanceof Literal) {
181                        value = ((Literal) node.rvalue).asString();
182                    } else if (node.rvalue != null) {
183                        value = node.rvalue.toString();
184                    }
185                    Object[] values = null;
186                    if (node.rvalue instanceof LiteralList) {
187                        LiteralList items = (LiteralList) node.rvalue;
188                        values = new Object[items.size()];
189                        int i = 0;
190                        for (Literal item : items) {
191                            values[i++] = item.asString();
192                        }
193                    }
194                    // add expression to the last builder
195                    EsHint hint = (ref != null) ? ref.esHint : null;
196                    builders.getLast()
197                            .add(makeQueryFromSimpleExpression(op.toString(), name, value, values, hint, session));
198                }
199            }
200        });
201        QueryBuilder queryBuilder = ret.get();
202        if (!fromList.isEmpty()) {
203            return QueryBuilders.boolQuery()
204                                .must(queryBuilder)
205                                .filter(makeQueryFromSimpleExpression("IN", NXQL.ECM_PRIMARYTYPE, null,
206                                        fromList.toArray(), null, null).filter);
207        }
208        return queryBuilder;
209    }
210
211    protected static SQLQuery getSqlQuery(String nxql) {
212        String query = completeQueryWithSelect(nxql);
213        SQLQuery nxqlQuery;
214        try {
215            nxqlQuery = SQLQueryParser.parse(new StringReader(query));
216        } catch (QueryParseException e) {
217            if (log.isDebugEnabled()) {
218                log.debug(e.getMessage() + " for query:\n" + query);
219            }
220            throw e;
221        }
222        return nxqlQuery;
223    }
224
225    protected static SQLQuery addSecurityPolicy(CoreSession session, SQLQuery query) {
226        Collection<SQLQuery.Transformer> transformers = NXCore.getSecurityService()
227                                                              .getPoliciesQueryTransformers(
228                                                                      session.getRepositoryName());
229        for (SQLQuery.Transformer trans : transformers) {
230            query = trans.transform(session.getPrincipal(), query);
231        }
232        return query;
233    }
234
235    protected static String completeQueryWithSelect(String nxql) {
236        String query = (nxql == null) ? "" : nxql.trim();
237        if (query.isEmpty()) {
238            query = SELECT_ALL;
239        } else if (!query.toLowerCase().startsWith("select ")) {
240            query = SELECT_ALL_WHERE + nxql;
241        }
242        return query;
243    }
244
245    public static QueryAndFilter makeQueryFromSimpleExpression(String op, String nxqlName, Object value,
246            Object[] values, EsHint hint, CoreSession session) {
247        QueryBuilder query = null;
248        QueryBuilder filter = null;
249        String name = getFieldName(nxqlName, hint);
250        if (hint != null && hint.operator != null) {
251            if (hint.operator.startsWith("geo")) {
252                filter = makeHintFilter(name, values, hint);
253            } else {
254                query = makeHintQuery(name, value, hint);
255            }
256        } else if (nxqlName.startsWith(NXQL.ECM_FULLTEXT) && ("=".equals(op) || "!=".equals(op) || "<>".equals(op)
257                || "LIKE".equals(op) || "NOT LIKE".equals(op))) {
258            query = makeFulltextQuery(nxqlName, (String) value, hint);
259            if ("!=".equals(op) || "<>".equals(op) || "NOT LIKE".equals(op)) {
260                filter = QueryBuilders.boolQuery().mustNot(query);
261                query = null;
262            }
263        } else if (nxqlName.startsWith(NXQL.ECM_ANCESTORID)) {
264            filter = makeAncestorIdFilter((String) value, session);
265            if ("!=".equals(op) || "<>".equals(op)) {
266                filter = QueryBuilders.boolQuery().mustNot(filter);
267            }
268        } else if (nxqlName.equals(NXQL.ECM_ISTRASHED)) {
269            filter = makeTrashedFilter(op, name, (String) value);
270
271        } else
272            switch (op) {
273            case "=":
274                filter = QueryBuilders.termQuery(name, checkBoolValue(nxqlName, value));
275                break;
276            case "<>":
277            case "!=":
278                filter = QueryBuilders.boolQuery()
279                                      .mustNot(QueryBuilders.termQuery(name, checkBoolValue(nxqlName, value)));
280                break;
281            case ">":
282                filter = QueryBuilders.rangeQuery(name).gt(value);
283                break;
284            case "<":
285                filter = QueryBuilders.rangeQuery(name).lt(value);
286                break;
287            case ">=":
288                filter = QueryBuilders.rangeQuery(name).gte(value);
289                break;
290            case "<=":
291                filter = QueryBuilders.rangeQuery(name).lte(value);
292                break;
293            case "BETWEEN":
294            case "NOT BETWEEN":
295                filter = QueryBuilders.rangeQuery(name).from(values[0]).to(values[1]);
296                if (op.startsWith("NOT")) {
297                    filter = QueryBuilders.boolQuery().mustNot(filter);
298                }
299                break;
300            case "IN":
301            case "NOT IN":
302                filter = QueryBuilders.termsQuery(name, values);
303                if (op.startsWith("NOT")) {
304                    filter = QueryBuilders.boolQuery().mustNot(filter);
305                }
306                break;
307            case "IS NULL":
308                filter = QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(name));
309                break;
310            case "IS NOT NULL":
311                filter = QueryBuilders.existsQuery(name);
312                break;
313            case "LIKE":
314            case "ILIKE":
315            case "NOT LIKE":
316            case "NOT ILIKE":
317                query = makeLikeQuery(op, name, (String) value, hint);
318                if (op.startsWith("NOT")) {
319                    filter = QueryBuilders.boolQuery().mustNot(query);
320                    query = null;
321                }
322                break;
323            case "STARTSWITH":
324                filter = makeStartsWithQuery(name, value);
325                break;
326            default:
327                throw new UnsupportedOperationException("Operator: '" + op + "' is unknown");
328            }
329        return new QueryAndFilter(query, filter);
330    }
331
332    protected static Object checkBoolValue(String nxqlName, Object value) {
333        if (!"0".equals(value) && !"1".equals(value)) {
334            return value;
335        }
336        switch (nxqlName) {
337        case NXQL.ECM_ISPROXY:
338        case NXQL.ECM_ISCHECKEDIN:
339        case NXQL.ECM_ISTRASHED:
340        case NXQL.ECM_ISVERSION:
341        case NXQL.ECM_ISVERSION_OLD:
342        case NXQL.ECM_ISLATESTMAJORVERSION:
343        case NXQL.ECM_ISLATESTVERSION:
344            break;
345        default:
346            SchemaManager schemaManager = Framework.getService(SchemaManager.class);
347            Field field = schemaManager.getField(nxqlName);
348            if (field == null || !BooleanType.ID.equals(field.getType().getName())) {
349                return value;
350            }
351        }
352        return "0".equals(value) ? "false" : "true";
353    }
354
355    protected static QueryBuilder makeTrashedFilter(String op, String name, String value) {
356        boolean equalsDeleted;
357        switch (op) {
358        case "=":
359            equalsDeleted = true;
360            break;
361        case "<>":
362        case "!=":
363            equalsDeleted = false;
364            break;
365        default:
366            throw new IllegalArgumentException(NXQL.ECM_ISTRASHED + " requires = or <> operator");
367        }
368        if ("0".equals(value)) {
369            equalsDeleted = !equalsDeleted;
370        } else if ("1".equals(value)) {
371            // equalsDeleted unchanged
372        } else {
373            throw new IllegalArgumentException(NXQL.ECM_ISTRASHED + " requires literal 0 or 1 as right argument");
374        }
375        TrashService trashService = Framework.getService(TrashService.class);
376        QueryBuilder filter = null;
377        if (trashService.hasFeature(Feature.TRASHED_STATE_IS_DEDUCED_FROM_LIFECYCLE)) {
378            filter = QueryBuilders.termQuery(NXQL.ECM_LIFECYCLESTATE, LifeCycleConstants.DELETED_STATE);
379        } else if (trashService.hasFeature(Feature.TRASHED_STATE_IN_MIGRATION)) {
380            filter = QueryBuilders.boolQuery()
381                                  .should(QueryBuilders.termQuery(NXQL.ECM_LIFECYCLESTATE,
382                                          LifeCycleConstants.DELETED_STATE))
383                                  .should(QueryBuilders.termQuery(name, true));
384        } else if (trashService.hasFeature(Feature.TRASHED_STATE_IS_DEDICATED_PROPERTY)) {
385            filter = QueryBuilders.termQuery(name, true);
386        }
387        if (!equalsDeleted) {
388            filter = QueryBuilders.boolQuery().mustNot(filter);
389        }
390        return filter;
391    }
392
393    private static QueryBuilder makeHintFilter(String name, Object[] values, EsHint hint) {
394        QueryBuilder ret;
395        switch (hint.operator) {
396        case "geo_bounding_box":
397            if (values.length != 2) {
398                throw new IllegalArgumentException(String.format(
399                        "Operator: %s requires 2 parameters: bottomLeft " + "and topRight point", hint.operator));
400            }
401            GeoPoint bottomLeft = parseGeoPointString((String) values[0]);
402            GeoPoint topRight = parseGeoPointString((String) values[1]);
403            ret = QueryBuilders.geoBoundingBoxQuery(name).setCornersOGC(bottomLeft, topRight);
404            break;
405        case "geo_distance":
406            if (values.length != 2) {
407                throw new IllegalArgumentException(
408                        String.format("Operator: %s requires 2 parameters: point and " + "distance", hint.operator));
409            }
410            GeoPoint center = parseGeoPointString((String) values[0]);
411            String distance = (String) values[1];
412            ret = QueryBuilders.geoDistanceQuery(name).point(center.lat(), center.lon()).distance(distance);
413            break;
414        case "geo_distance_range":
415            throw new UnsupportedOperationException(
416                    "Operator: '" + hint.operator + "' is deprecated since Elasticsearch 6.1");
417        case "geo_hash_cell":
418            throw new UnsupportedOperationException(
419                    "Operator: '" + hint.operator + "' is deprecated since Elasticsearch 6.2");
420        case "geo_shape":
421            if (values.length != 4) {
422                throw new IllegalArgumentException(String.format(
423                        "Operator: %s requires 4 parameters: shapeId, type, " + "index and path", hint.operator));
424            }
425            String shapeId = (String) values[0];
426            String shapeType = (String) values[1];
427            String shapeIndex = (String) values[2];
428            String shapePath = (String) values[3];
429
430            ret = QueryBuilders.geoShapeQuery(name, shapeId, shapeType)
431                               .relation(ShapeRelation.WITHIN)
432                               .indexedShapeIndex(shapeIndex)
433                               .indexedShapePath(shapePath);
434            break;
435        default:
436            throw new UnsupportedOperationException("Operator: '" + hint.operator + "' is unknown");
437        }
438        return ret;
439
440    }
441
442    private static GeoPoint parseGeoPointString(String value) {
443        try {
444            XContentBuilder content = JsonXContent.contentBuilder();
445            content.value(value);
446            content.flush();
447            content.close();
448            try (XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
449                    THROW_UNSUPPORTED_OPERATION, ((ByteArrayOutputStream) content.getOutputStream()).toByteArray())) {
450                parser.nextToken();
451                return GeoUtils.parseGeoPoint(parser);
452            }
453        } catch (IOException e) {
454            throw new IllegalArgumentException("Invalid value for geopoint: " + e.getMessage());
455        }
456    }
457
458    private static QueryBuilder makeHintQuery(String name, Object value, EsHint hint) {
459        QueryBuilder ret;
460        switch (hint.operator) {
461        case "match":
462            MatchQueryBuilder matchQuery = QueryBuilders.matchQuery(name, value);
463            if (hint.analyzer != null) {
464                matchQuery.analyzer(hint.analyzer);
465            }
466            ret = matchQuery;
467            break;
468        case "match_phrase":
469            MatchPhraseQueryBuilder matchPhraseQuery = QueryBuilders.matchPhraseQuery(name, value);
470            if (hint.analyzer != null) {
471                matchPhraseQuery.analyzer(hint.analyzer);
472            }
473            ret = matchPhraseQuery;
474            break;
475        case "match_phrase_prefix":
476            String valueString = (String) value;
477            if (valueString.endsWith("*") && valueString.length() > 2) {
478                // remove useless trailing *, this is not mandatory but cleaner
479                value = valueString.substring(0, valueString.length() - 1);
480            }
481            MatchPhrasePrefixQueryBuilder matchPhrasePrefixQuery = QueryBuilders.matchPhrasePrefixQuery(name, value);
482            if (hint.analyzer != null) {
483                matchPhrasePrefixQuery.analyzer(hint.analyzer);
484            }
485            ret = matchPhrasePrefixQuery;
486            break;
487        case "multi_match":
488            // multiMatchQuery requires at least 1 field on creation, so we set them twice, incase there's a field boost
489            MultiMatchQueryBuilder multiMatchQuery = QueryBuilders.multiMatchQuery(value, hint.getIndexFieldNames());
490            hint.getIndex().forEach(fieldHint -> multiMatchQuery.field(fieldHint.getField(), fieldHint.getBoost()));
491            if (hint.analyzer != null) {
492                multiMatchQuery.analyzer(hint.analyzer);
493            }
494            ret = multiMatchQuery;
495            break;
496        case "regex":
497            ret = QueryBuilders.regexpQuery(name, (String) value);
498            break;
499        case "fuzzy":
500            ret = QueryBuilders.fuzzyQuery(name, (String) value);
501            break;
502        case "wildcard":
503            ret = QueryBuilders.wildcardQuery(name, (String) value);
504            break;
505        case "common":
506            CommonTermsQueryBuilder commonQuery = QueryBuilders.commonTermsQuery(name, value);
507            if (hint.analyzer != null) {
508                commonQuery.analyzer(hint.analyzer);
509            }
510            ret = commonQuery;
511            break;
512        case "query_string":
513            QueryStringQueryBuilder queryString = QueryBuilders.queryStringQuery((String) value);
514            if (hint.index != null) {
515                for (EsHint.FieldHint fieldHint : hint.getIndex()) {
516                    queryString.field(fieldHint.getField(), fieldHint.getBoost());
517                }
518            } else {
519                queryString.defaultField(name);
520            }
521            if (hint.analyzer != null) {
522                queryString.analyzer(hint.analyzer);
523            }
524            ret = queryString;
525            break;
526        case "simple_query_string":
527            SimpleQueryStringBuilder querySimpleString = QueryBuilders.simpleQueryStringQuery((String) value);
528            if (hint.index != null) {
529                for (EsHint.FieldHint fieldHint : hint.getIndex()) {
530                    querySimpleString.field(fieldHint.getField(), fieldHint.getBoost());
531                }
532            } else {
533                querySimpleString.field(name);
534            }
535            if (hint.analyzer != null) {
536                querySimpleString.analyzer(hint.analyzer);
537            }
538            ret = querySimpleString;
539            break;
540        case "more_like_this":
541            String[] indexFields = hint.getIndexFieldNames();
542            String[] fields = indexFields.length > 0 ? indexFields : new String[] { name };
543            MoreLikeThisQueryBuilder moreLikeThisBuilder = QueryBuilders.moreLikeThisQuery(fields, null,
544                    getItems(value));
545            if (hint.analyzer != null) {
546                moreLikeThisBuilder.analyzer(hint.analyzer);
547            }
548            moreLikeThisBuilder.minDocFreq(MORE_LIKE_THIS_MIN_DOC_FREQ);
549            moreLikeThisBuilder.minTermFreq(MORE_LIKE_THIS_MIN_TERM_FREQ);
550            moreLikeThisBuilder.maxQueryTerms(MORE_LIKE_THIS_MAX_QUERY_TERMS);
551            ret = moreLikeThisBuilder;
552            break;
553        default:
554            throw new UnsupportedOperationException("Operator: '" + hint.operator + "' is unknown");
555        }
556        return ret;
557    }
558
559    protected static MoreLikeThisQueryBuilder.Item[] getItems(Object value) {
560        RepositoryManager rm = Framework.getService(RepositoryManager.class);
561        String repo = rm.getDefaultRepository().getName();
562        ElasticSearchAdmin esa = Framework.getService(ElasticSearchAdmin.class);
563        String esIndex = esa.getIndexNameForRepository(repo);
564        String[] values;
565        if (value instanceof Object[]) {
566            values = (String[]) value;
567        } else {
568            values = new String[] { (String) value };
569        }
570        MoreLikeThisQueryBuilder.Item[] ret = new MoreLikeThisQueryBuilder.Item[values.length];
571        for (int i = 0; i < values.length; i++) {
572            ret[i] = new MoreLikeThisQueryBuilder.Item(esIndex, DOC_TYPE, values[i]);
573        }
574        return ret;
575    }
576
577    public static QueryBuilder makeStartsWithQuery(String name, Object value) {
578        QueryBuilder filter;
579        String indexName = name + ".children";
580        if ("/".equals(value)) {
581            if (NXQL.ECM_PATH.equals(name)) {
582                // any non orphan|place-less document must have a path starting with "/"
583                filter = QueryBuilders.existsQuery(NXQL.ECM_PARENTID);
584            } else {
585                // match any document with a populated field
586                filter = QueryBuilders.existsQuery(indexName);
587            }
588        } else {
589            String v = String.valueOf(value);
590            if (v.endsWith("/")) {
591                v = v.replaceAll("/$", "");
592            }
593            if (NXQL.ECM_PATH.equals(name)) {
594                // we don't want to return the parent when searching on ecm:path, see NXP-18955
595                filter = QueryBuilders.boolQuery()
596                                      .must(QueryBuilders.termQuery(indexName, v))
597                                      .mustNot(QueryBuilders.termQuery(name, value));
598            } else {
599                filter = QueryBuilders.termQuery(indexName, v);
600            }
601        }
602        return filter;
603    }
604
605    private static QueryBuilder makeAncestorIdFilter(String value, CoreSession session) {
606        String path;
607        if (session == null) {
608            return QueryBuilders.existsQuery("ancestorid-without-session");
609        } else {
610            try {
611                DocumentModel doc = session.getDocument(new IdRef(value));
612                path = doc.getPathAsString();
613            } catch (DocumentNotFoundException e) {
614                return QueryBuilders.existsQuery("ancestorid-not-found");
615            }
616        }
617        return makeStartsWithQuery(NXQL.ECM_PATH, path);
618    }
619
620    private static QueryBuilder makeLikeQuery(String op, String name, String value, EsHint hint) {
621        String fieldName = name;
622        if (op.contains("ILIKE")) {
623            // ILIKE will work only with a correct mapping
624            value = value.toLowerCase();
625            fieldName = name + ".lowercase";
626        }
627        if (hint != null && hint.index != null) {
628            fieldName = hint.index;
629        }
630        // convert the value to a wildcard query
631        String wildcard = likeToWildcard(value);
632        // use match phrase prefix when possible
633        if (StringUtils.countMatches(wildcard, "*") == 1 && wildcard.endsWith("*") && !wildcard.contains("?")
634                && !wildcard.contains("\\")) {
635            MatchPhrasePrefixQueryBuilder query = QueryBuilders.matchPhrasePrefixQuery(fieldName,
636                    wildcard.replace("*", ""));
637            if (hint != null && hint.analyzer != null) {
638                query.analyzer(hint.analyzer);
639            }
640            return query;
641        }
642        return QueryBuilders.wildcardQuery(fieldName, wildcard);
643    }
644
645    /**
646     * Turns a NXQL LIKE pattern into a wildcard for WildcardQuery.
647     * <p>
648     * % and _ are standard wildcards, and \ escapes them.
649     *
650     * @since 7.4
651     */
652    protected static String likeToWildcard(String like) {
653        StringBuilder wildcard = new StringBuilder();
654        char[] chars = like.toCharArray();
655        boolean escape = false;
656        for (char c : chars) {
657            boolean escapeNext = false;
658            switch (c) {
659            case '?':
660                wildcard.append("\\?");
661                break;
662            case '*': // compat, * = % in NXQL (for some backends)
663            case '%':
664                if (escape) {
665                    wildcard.append(c);
666                } else {
667                    wildcard.append("*");
668                }
669                break;
670            case '_':
671                if (escape) {
672                    wildcard.append(c);
673                } else {
674                    wildcard.append("?");
675                }
676                break;
677            case '\\':
678                if (escape) {
679                    wildcard.append("\\\\");
680                } else {
681                    escapeNext = true;
682                }
683                break;
684            default:
685                wildcard.append(c);
686                break;
687            }
688            escape = escapeNext;
689        }
690        if (escape) {
691            // invalid string terminated by escape character, ignore
692        }
693        return wildcard.toString();
694    }
695
696    private static QueryBuilder makeFulltextQuery(String nxqlName, String value, EsHint hint) {
697        String name = nxqlName.replace(NXQL.ECM_FULLTEXT, "");
698        if (name.startsWith(".")) {
699            name = name.substring(1) + ".fulltext";
700        } else {
701            // map ecm:fulltext_someindex to default
702            name = FULLTEXT_FIELD;
703        }
704        String queryString = value;
705        org.elasticsearch.index.query.Operator defaultOperator;
706        if (queryString.startsWith(SIMPLE_QUERY_PREFIX)) {
707            // elasticsearch-specific syntax
708            queryString = queryString.substring(SIMPLE_QUERY_PREFIX.length());
709            defaultOperator = org.elasticsearch.index.query.Operator.OR;
710        } else {
711            queryString = translateFulltextQuery(queryString);
712            defaultOperator = org.elasticsearch.index.query.Operator.AND;
713        }
714        String analyzer = (hint != null && hint.analyzer != null) ? hint.analyzer : "fulltext";
715        SimpleQueryStringBuilder query = QueryBuilders.simpleQueryStringQuery(queryString)
716                                                      .defaultOperator(defaultOperator)
717                                                      .analyzer(analyzer);
718        if (hint != null && hint.index != null) {
719            for (EsHint.FieldHint fieldHint : hint.getIndex()) {
720                query.field(fieldHint.getField(), fieldHint.getBoost());
721            }
722        } else {
723            query.field(name);
724        }
725        return query;
726    }
727
728    private static String getFieldName(String name, EsHint hint) {
729        if (hint != null && hint.index != null) {
730            return hint.index;
731        }
732        // compat
733        if (NXQL.ECM_ISVERSION_OLD.equals(name)) {
734            name = NXQL.ECM_ISVERSION;
735        }
736        // complex field
737        name = name.replace("/*", "");
738        name = name.replace("/", ".");
739        return name;
740    }
741
742    public static List<SortInfo> getSortInfo(String nxql) {
743        final List<SortInfo> sortInfos = new ArrayList<>();
744        SQLQuery nxqlQuery = getSqlQuery(nxql);
745        nxqlQuery.accept(new DefaultQueryVisitor() {
746
747            @Override
748            public void visitOrderByExpr(OrderByExpr node) {
749                String name = getFieldName(node.reference.name, null);
750                if (NXQL.ECM_FULLTEXT_SCORE.equals(name)) {
751                    name = ES_SCORE_FIELD;
752                }
753                sortInfos.add(new SortInfo(name, !node.isDescending));
754            }
755        });
756        return sortInfos;
757    }
758
759    public static Map<String, Type> getSelectClauseFields(String nxql) {
760        final Map<String, Type> fieldsAndTypes = new LinkedHashMap<>();
761        SQLQuery nxqlQuery = getSqlQuery(nxql);
762        nxqlQuery.accept(new DefaultQueryVisitor() {
763
764            @Override
765            public void visitSelectClause(SelectClause selectClause) {
766                SchemaManager schemaManager = Framework.getService(SchemaManager.class);
767                for (int i = 0; i < selectClause.getSelectList().size(); i++) {
768                    Operand op = selectClause.get(i);
769                    if (!(op instanceof Reference)) {
770                        // ignore it
771                        continue;
772                    }
773                    String name = ((Reference) op).name;
774                    Field field = schemaManager.getField(name);
775                    fieldsAndTypes.put(name, field == null ? null : field.getType());
776                }
777            }
778        });
779        return fieldsAndTypes;
780    }
781
782    /**
783     * Translates from Nuxeo syntax to Elasticsearch simple_query_string syntax.
784     */
785    public static String translateFulltextQuery(String query) {
786        // The AND operator does not exist in NXQL it is the default operator
787        return query.replace(" OR ", " | ").replace(" or ", " | ");
788    }
789
790    /**
791     * Class to hold both a query and a filter
792     */
793    public static class QueryAndFilter {
794
795        public final QueryBuilder query;
796
797        public final QueryBuilder filter;
798
799        public QueryAndFilter(QueryBuilder query, QueryBuilder filter) {
800            this.query = query;
801            this.filter = filter;
802        }
803    }
804
805    public static class ExpressionBuilder {
806
807        public final String operator;
808
809        public QueryBuilder query;
810
811        public ExpressionBuilder(final String op) {
812            this.operator = op;
813            this.query = null;
814        }
815
816        public void add(final QueryAndFilter qf) {
817            if (qf != null) {
818                add(qf.query, qf.filter);
819            }
820        }
821
822        public void add(QueryBuilder q) {
823            add(q, null);
824        }
825
826        public void add(final QueryBuilder q, final QueryBuilder f) {
827            if (q == null && f == null) {
828                return;
829            }
830            QueryBuilder inputQuery = q;
831            if (inputQuery == null) {
832                inputQuery = QueryBuilders.constantScoreQuery(f);
833            }
834            if (operator == null) {
835                // first level expression
836                query = inputQuery;
837            } else {
838                // boolean query
839                if (query == null) {
840                    query = QueryBuilders.boolQuery();
841                }
842                BoolQueryBuilder boolQuery = (BoolQueryBuilder) query;
843                if ("AND".equals(operator)) {
844                    boolQuery.must(inputQuery);
845                } else if ("OR".equals(operator)) {
846                    boolQuery.should(inputQuery);
847                } else if ("NOT".equals(operator)) {
848                    boolQuery.mustNot(inputQuery);
849                }
850            }
851        }
852
853        public void merge(ExpressionBuilder expr) {
854            if ((expr.operator != null) && expr.operator.equals(operator) && (query == null)) {
855                query = expr.query;
856            } else {
857                add(new QueryAndFilter(expr.query, null));
858            }
859        }
860
861        public QueryBuilder get() {
862            if (query == null) {
863                return QueryBuilders.matchAllQuery();
864            }
865            return query;
866        }
867
868        @Override
869        public String toString() {
870            return query.toString();
871        }
872
873    }
874}