Source code

001/*
002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     bdelbosc
018 */
019package org.nuxeo.elasticsearch.query;
020
021import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL;
022import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD;
023import static org.nuxeo.elasticsearch.ElasticSearchConstants.ES_SCORE_FIELD;
024import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY;
025
026import java.util.ArrayList;
027import java.util.Collections;
028import java.util.List;
029import java.util.Map;
030import java.util.Set;
031
032import org.elasticsearch.action.search.SearchResponse;
033import org.elasticsearch.index.query.BoolQueryBuilder;
034import org.elasticsearch.index.query.QueryBuilder;
035import org.elasticsearch.index.query.QueryBuilders;
036import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
037import org.elasticsearch.search.aggregations.Aggregation;
038import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
039import org.elasticsearch.search.builder.SearchSourceBuilder;
040import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
041import org.elasticsearch.search.sort.FieldSortBuilder;
042import org.elasticsearch.search.sort.SortBuilder;
043import org.elasticsearch.search.sort.SortOrder;
044import org.nuxeo.ecm.core.api.CoreInstance;
045import org.nuxeo.ecm.core.api.CoreSession;
046import org.nuxeo.ecm.core.api.NuxeoPrincipal;
047import org.nuxeo.ecm.core.api.SortInfo;
048import org.nuxeo.ecm.core.schema.SchemaManager;
049import org.nuxeo.ecm.core.schema.types.Type;
050import org.nuxeo.ecm.core.security.SecurityService;
051import org.nuxeo.ecm.platform.query.api.Aggregate;
052import org.nuxeo.ecm.platform.query.api.Bucket;
053import org.nuxeo.elasticsearch.ElasticSearchConstants;
054import org.nuxeo.elasticsearch.aggregate.AggregateEsBase;
055import org.nuxeo.elasticsearch.api.EsResult;
056import org.nuxeo.elasticsearch.fetcher.EsFetcher;
057import org.nuxeo.elasticsearch.fetcher.Fetcher;
058import org.nuxeo.elasticsearch.fetcher.VcsFetcher;
059import org.nuxeo.runtime.api.Framework;
060
061/**
062 * Elasticsearch query builder for the Nuxeo ES api.
063 *
064 * @since 5.9.5
065 */
066public class NxQueryBuilder {
067
068    private static final int DEFAULT_LIMIT = 10;
069
070    private static final String AGG_FILTER_SUFFIX = "_filter";
071
072    private final CoreSession session;
073
074    private final List<SortInfo> sortInfos = new ArrayList<>();
075
076    private final List<String> repositories = new ArrayList<>();
077
078    private final List<AggregateEsBase<Aggregation, Bucket>> aggregates = new ArrayList<>();
079
080    private int limit = DEFAULT_LIMIT;
081
082    private int offset = 0;
083
084    private String nxql;
085
086    private org.elasticsearch.index.query.QueryBuilder esQueryBuilder;
087
088    private boolean fetchFromElasticsearch = false;
089
090    private boolean searchOnAllRepo = false;
091
092    private String[] selectFields = { ElasticSearchConstants.ID_FIELD };
093
094    private Map<String, Type> selectFieldsAndTypes;
095
096    private boolean returnsDocuments = true;
097
098    private boolean esOnly = false;
099
100    private List<String> highlightFields;
101
102    private EsFetcher.HitDocConsumer hitDocConsumer;
103
104    protected boolean useUnrestrictedSession;
105
106    public NxQueryBuilder(CoreSession coreSession) {
107        session = coreSession;
108        repositories.add(coreSession.getRepositoryName());
109        fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false"));
110    }
111
112    public static String getAggregateFilterId(Aggregate<?> agg) {
113        return agg.getId() + AGG_FILTER_SUFFIX;
114    }
115
116    /**
117     * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}. Since Nuxeo 8.4 and ES 2.x, we can
118     * not give -1 to this method as the default configuration on ES allows to have a search window of 10000 documents
119     * at maximum. This settings could be changed on ES by changing {index.max_result_window}, but it is preferable to
120     * use the scan &amp; scroll API.
121     */
122    public NxQueryBuilder limit(int limit) {
123        // For compatibility only, deprecated since 8.4
124        if (limit < 0) {
125            limit = Integer.MAX_VALUE;
126        }
127        this.limit = limit;
128        return this;
129    }
130
131    /**
132     * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then
133     * offset documents are skipped before starting to count the limit documents that are returned.
134     */
135    public NxQueryBuilder offset(int offset) {
136        this.offset = offset;
137        return this;
138    }
139
140    public NxQueryBuilder addSort(SortInfo sortInfo) {
141        sortInfos.add(sortInfo);
142        return this;
143    }
144
145    public NxQueryBuilder addSort(SortInfo[] sortInfos) {
146        if (sortInfos != null && sortInfos.length > 0) {
147            Collections.addAll(this.sortInfos, sortInfos);
148        }
149        return this;
150    }
151
152    /**
153     * Build the query from a NXQL string. You should either use nxql, either esQuery, not both.
154     */
155    public NxQueryBuilder nxql(String nxql) {
156        this.nxql = nxql;
157        this.esQueryBuilder = null;
158        return this;
159    }
160
161    /**
162     * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both.
163     */
164    public NxQueryBuilder esQuery(QueryBuilder queryBuilder) {
165        esQueryBuilder = addSecurityFilter(queryBuilder);
166        nxql = null;
167        return this;
168    }
169
170    /**
171     * Ask for the Elasticsearch _source field, use it to build documents.
172     */
173    public NxQueryBuilder fetchFromElasticsearch() {
174        fetchFromElasticsearch = true;
175        return this;
176    }
177
178    /**
179     * If search results are found, use this SearchHit and DocumentModel consumer on each hit.
180     *
181     * @since 10.2
182     */
183    public NxQueryBuilder hitDocConsumer(EsFetcher.HitDocConsumer consumer) {
184        hitDocConsumer = consumer;
185        return this;
186    }
187
188    /**
189     * @since 11.1
190     */
191    public NxQueryBuilder useUnrestrictedSession(boolean useUnrestrictedSession) {
192        this.useUnrestrictedSession = useUnrestrictedSession;
193        return this;
194    }
195
196    /**
197     * Fetch the documents using VCS (database) engine. This is done by default
198     */
199    public NxQueryBuilder fetchFromDatabase() {
200        fetchFromElasticsearch = false;
201        return this;
202    }
203
204    /**
205     * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from
206     * {@link EsResult#getElasticsearchResponse()}
207     *
208     * @since 7.3
209     */
210    public NxQueryBuilder onlyElasticsearchResponse() {
211        esOnly = true;
212        return this;
213    }
214
215    @SuppressWarnings("unchecked")
216    public NxQueryBuilder addAggregate(AggregateEsBase<? extends Aggregation, ? extends Bucket> aggregate) {
217        aggregates.add((AggregateEsBase<Aggregation, Bucket>) aggregate);
218        return this;
219    }
220
221    public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Aggregation, ? extends Bucket>> aggregates) {
222        if (aggregates != null && !aggregates.isEmpty()) {
223            aggregates.forEach(this::addAggregate);
224        }
225        return this;
226    }
227
228    /**
229     * @since 9.1
230     */
231    public NxQueryBuilder highlight(List<String> highlightFields) {
232        this.highlightFields = highlightFields;
233        return this;
234    }
235
236    public int getLimit() {
237        return limit;
238    }
239
240    public int getOffset() {
241        return offset;
242    }
243
244    public List<SortInfo> getSortInfos() {
245        return sortInfos;
246    }
247
248    public String getNxql() {
249        return nxql;
250    }
251
252    public boolean isFetchFromElasticsearch() {
253        return fetchFromElasticsearch;
254    }
255
256    public CoreSession getSession() {
257        return session;
258    }
259
260    /**
261     * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor
262     * aggregates, use the udpateRequest to get the full request.
263     */
264    public QueryBuilder makeQuery() {
265        if (esQueryBuilder == null) {
266            if (nxql != null) {
267                esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session);
268                // handle the built-in order by clause
269                if (nxql.toLowerCase().contains("order by")) {
270                    List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql);
271                    sortInfos.addAll(builtInSortInfos);
272                }
273                if (nxqlHasSelectClause(nxql)) {
274                    selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql);
275                    Set<String> keySet = selectFieldsAndTypes.keySet();
276                    selectFields = keySet.toArray(new String[0]);
277                    returnsDocuments = false;
278                }
279                esQueryBuilder = addSecurityFilter(esQueryBuilder);
280            }
281        }
282        return esQueryBuilder;
283    }
284
285    protected boolean nxqlHasSelectClause(String nxql) {
286        String lowerNxql = nxql.toLowerCase();
287        return lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from");
288    }
289
290    public SortBuilder<?>[] getSortBuilders() {
291        SortBuilder<?>[] ret;
292        if (sortInfos.isEmpty()) {
293            return new SortBuilder[0];
294        }
295        ret = new SortBuilder[sortInfos.size()];
296        int i = 0;
297        for (SortInfo sortInfo : sortInfos) {
298            String fieldType = guessFieldType(sortInfo.getSortColumn());
299            ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn())
300                                                                     .order(sortInfo.getSortAscending() ? SortOrder.ASC
301                                                                             : SortOrder.DESC)
302                                                                     .unmappedType(fieldType);
303        }
304        return ret;
305    }
306
307    protected String guessFieldType(String field) {
308        String fieldType;
309        if (ES_SCORE_FIELD.equals(field)) {
310            // this special field should not have an unmappedType
311            return null;
312        }
313        try {
314            SchemaManager schemaManager = Framework.getService(SchemaManager.class);
315            fieldType = schemaManager.getField(field).getType().getName();
316        } catch (NullPointerException e) {
317            // probably an internal field without schema
318            fieldType = "keyword";
319        }
320        switch (fieldType) {
321        case "integer":
322        case "long":
323        case "boolean":
324        case "date":
325            return fieldType;
326        }
327        return "keyword";
328    }
329
330    protected QueryBuilder getAggregateFilter() {
331        BoolQueryBuilder ret = QueryBuilders.boolQuery();
332        for (AggregateEsBase<?, ?> agg : aggregates) {
333            QueryBuilder filter = agg.getEsFilter();
334            if (filter != null) {
335                ret.must(filter);
336            }
337        }
338        if (!ret.hasClauses()) {
339            return null;
340        }
341        return ret;
342    }
343
344    protected QueryBuilder getAggregateFilterExceptFor(String id) {
345        BoolQueryBuilder ret = QueryBuilders.boolQuery();
346        for (AggregateEsBase<?, ?> agg : aggregates) {
347            if (!agg.getId().equals(id)) {
348                QueryBuilder filter = agg.getEsFilter();
349                if (filter != null) {
350                    ret.must(filter);
351                }
352            }
353        }
354        if (!ret.hasClauses()) {
355            return QueryBuilders.matchAllQuery();
356        }
357        return ret;
358    }
359
360    public List<AggregateEsBase<Aggregation, Bucket>> getAggregates() {
361        return aggregates;
362    }
363
364    public List<FilterAggregationBuilder> getEsAggregates() {
365        List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size());
366        for (AggregateEsBase<?, ?> agg : aggregates) {
367            FilterAggregationBuilder fagg = null;
368            fagg = new FilterAggregationBuilder(getAggregateFilterId(agg), getAggregateFilterExceptFor(agg.getId()));
369            fagg.subAggregation(agg.getEsAggregate());
370            ret.add(fagg);
371        }
372        return ret;
373    }
374
375    public void updateRequest(SearchSourceBuilder request) {
376        // Set limits
377        request.from(getOffset()).size(getLimit());
378        // Build query with security checks
379        request.query(makeQuery());
380        // Add sort
381        for (SortBuilder<?> sortBuilder : getSortBuilders()) {
382            request.sort(sortBuilder);
383        }
384        // Add Aggregate
385        for (AbstractAggregationBuilder<?> aggregate : getEsAggregates()) {
386            request.aggregation(aggregate);
387        }
388        // Add Aggregate post filter
389        QueryBuilder aggFilter = getAggregateFilter();
390        if (aggFilter != null) {
391            request.postFilter(aggFilter);
392        }
393
394        // Add highlighting
395        if (highlightFields != null && !highlightFields.isEmpty()) {
396            HighlightBuilder hb = new HighlightBuilder();
397            for (String field : highlightFields) {
398                hb.field(field);
399            }
400            hb.requireFieldMatch(false);
401            request.highlighter(hb);
402        }
403        // Fields selection
404        if (!isFetchFromElasticsearch()) {
405            request.fetchSource(getSelectFields(), null);
406        }
407
408    }
409
410    protected QueryBuilder addSecurityFilter(QueryBuilder query) {
411        NuxeoPrincipal principal = session.getPrincipal();
412        if (principal == null || principal.isAdministrator() || useUnrestrictedSession) {
413            return query;
414        }
415        String[] principals = SecurityService.getPrincipalsToCheck(principal);
416        // we want an ACL that match principals but we discard
417        // unsupported ACE that contains negative ACE
418        QueryBuilder aclFilter = QueryBuilders.boolQuery()
419                                              .must(QueryBuilders.termsQuery(ACL_FIELD, principals))
420                                              .mustNot(QueryBuilders.termsQuery(ACL_FIELD, UNSUPPORTED_ACL));
421        return QueryBuilders.boolQuery().must(query).filter(aclFilter);
422    }
423
424    /**
425     * Add a specific repository to search. Default search is done on the session repository only.
426     *
427     * @since 6.0
428     */
429    public NxQueryBuilder addSearchRepository(String repositoryName) {
430        repositories.add(repositoryName);
431        return this;
432    }
433
434    /**
435     * Search on all available repositories.
436     *
437     * @since 6.0
438     */
439    public NxQueryBuilder searchOnAllRepositories() {
440        searchOnAllRepo = true;
441        return this;
442    }
443
444    /**
445     * Return the list of repositories to search, or an empty list to search on all available repositories;
446     *
447     * @since 6.0
448     */
449    public List<String> getSearchRepositories() {
450        if (searchOnAllRepo) {
451            return Collections.emptyList();
452        }
453        return repositories;
454    }
455
456    /**
457     * @since 6.0
458     */
459    public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) {
460        if (useUnrestrictedSession) {
461            return CoreInstance.doPrivileged(session, s -> {
462                return getFetcher(s, response, repoNames);
463            });
464        }
465        return getFetcher(session, response, repoNames);
466    }
467
468    protected Fetcher getFetcher(CoreSession session, SearchResponse response, Map<String, String> repoNames) {
469        if (isFetchFromElasticsearch()) {
470            return new EsFetcher(session, response, repoNames, hitDocConsumer);
471        }
472        return new VcsFetcher(session, response, repoNames);
473    }
474
475    /**
476     * @since 7.2
477     */
478    public String[] getSelectFields() {
479        return selectFields;
480    }
481
482    /**
483     * @since 7.2
484     */
485    public Map<String, Type> getSelectFieldsAndTypes() {
486        return selectFieldsAndTypes;
487    }
488
489    /**
490     * @since 7.2
491     */
492    public boolean returnsDocuments() {
493        if (esOnly) {
494            return false;
495        }
496        return returnsDocuments;
497    }
498
499    public boolean returnsRows() {
500        if (esOnly) {
501            return false;
502        }
503        return !returnsDocuments;
504    }
505}