001/*
002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and contributors.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the GNU Lesser General Public License
006 * (LGPL) version 2.1 which accompanies this distribution, and is available at
007 * http://www.gnu.org/licenses/lgpl-2.1.html
008 *
009 * This library is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 * Lesser General Public License for more details.
013 *
014 * Contributors:
015 *     bdelbosc
016 */
017package org.nuxeo.elasticsearch.query;
018
019import org.elasticsearch.action.search.SearchRequestBuilder;
020import org.elasticsearch.action.search.SearchResponse;
021import org.elasticsearch.index.query.AndFilterBuilder;
022import org.elasticsearch.index.query.FilterBuilder;
023import org.elasticsearch.index.query.FilterBuilders;
024import org.elasticsearch.index.query.QueryBuilder;
025import org.elasticsearch.index.query.QueryBuilders;
026import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
027import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
028import org.elasticsearch.search.sort.FieldSortBuilder;
029import org.elasticsearch.search.sort.SortBuilder;
030import org.elasticsearch.search.sort.SortOrder;
031import org.nuxeo.ecm.core.api.CoreSession;
032import org.nuxeo.ecm.core.api.NuxeoPrincipal;
033import org.nuxeo.ecm.core.api.SortInfo;
034import org.nuxeo.ecm.core.schema.types.Type;
035import org.nuxeo.ecm.core.security.SecurityService;
036import org.nuxeo.ecm.platform.query.api.Aggregate;
037import org.nuxeo.ecm.platform.query.api.Bucket;
038import org.nuxeo.elasticsearch.ElasticSearchConstants;
039import org.nuxeo.elasticsearch.aggregate.AggregateEsBase;
040import org.nuxeo.elasticsearch.api.EsResult;
041import org.nuxeo.elasticsearch.fetcher.EsFetcher;
042import org.nuxeo.elasticsearch.fetcher.Fetcher;
043import org.nuxeo.elasticsearch.fetcher.VcsFetcher;
044import org.nuxeo.runtime.api.Framework;
045
046import java.security.Principal;
047import java.util.ArrayList;
048import java.util.Collections;
049import java.util.List;
050import java.util.Map;
051import java.util.Set;
052
053import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL;
054import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD;
055import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY;
056
057/**
058 * Elasticsearch query buidler for the Nuxeo ES api.
059 *
060 * @since 5.9.5
061 */
062public class NxQueryBuilder {
063
064    private static final int DEFAULT_LIMIT = 10;
065
066    private int limit = DEFAULT_LIMIT;
067
068    private static final String AGG_FILTER_SUFFIX = "_filter";
069
070    private final CoreSession session;
071
072    private final List<SortInfo> sortInfos = new ArrayList<>();
073
074    private final List<String> repositories = new ArrayList<>();
075
076    private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>();
077
078    private int offset = 0;
079
080    private String nxql;
081
082    private org.elasticsearch.index.query.QueryBuilder esQueryBuilder;
083
084    private boolean fetchFromElasticsearch = false;
085
086    private boolean searchOnAllRepo = false;
087
088    private String[] selectFields = {ElasticSearchConstants.ID_FIELD};
089
090    private Map<String, Type> selectFieldsAndTypes;
091
092    private boolean returnsDocuments = true;
093
094    private boolean esOnly = false;
095
096    public NxQueryBuilder(CoreSession coreSession) {
097        session = coreSession;
098        repositories.add(coreSession.getRepositoryName());
099        fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false"));
100    }
101
102    public static String getAggregateFilterId(Aggregate agg) {
103        return agg.getId() + AGG_FILTER_SUFFIX;
104    }
105
106    /**
107     * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}, Use -1 to return all documents.
108     */
109    public NxQueryBuilder limit(int limit) {
110        if (limit < 0) {
111            limit = Integer.MAX_VALUE;
112        }
113        this.limit = limit;
114        return this;
115    }
116
117    /**
118     * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then
119     * offset documents are skipped before starting to count the limit documents that are returned.
120     */
121    public NxQueryBuilder offset(int offset) {
122        this.offset = offset;
123        return this;
124    }
125
126    public NxQueryBuilder addSort(SortInfo sortInfo) {
127        sortInfos.add(sortInfo);
128        return this;
129    }
130
131    public NxQueryBuilder addSort(SortInfo[] sortInfos) {
132        if (sortInfos != null && sortInfos.length > 0) {
133            Collections.addAll(this.sortInfos, sortInfos);
134        }
135        return this;
136    }
137
138    /**
139     * Build the query from a NXQL string. You should either use nxql, either esQuery, not both.
140     */
141    public NxQueryBuilder nxql(String nxql) {
142        this.nxql = nxql;
143        this.esQueryBuilder = null;
144        return this;
145    }
146
147    /**
148     * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both.
149     */
150    public NxQueryBuilder esQuery(QueryBuilder queryBuilder) {
151        this.esQueryBuilder = queryBuilder;
152        nxql = null;
153        return this;
154    }
155
156    /**
157     * Ask for the Elasticsearch _source field, use it to build documents.
158     */
159    public NxQueryBuilder fetchFromElasticsearch() {
160        fetchFromElasticsearch = true;
161        return this;
162    }
163
164    /**
165     * Fetch the documents using VCS (database) engine. This is done by default
166     */
167    public NxQueryBuilder fetchFromDatabase() {
168        fetchFromElasticsearch = false;
169        return this;
170    }
171
172    /**
173     * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from
174     * {@link EsResult#getElasticsearchResponse()}
175     *
176     * @since 7.3
177     */
178    public NxQueryBuilder onlyElasticsearchResponse() {
179        esOnly = true;
180        return this;
181    }
182
183    public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) {
184        aggregates.add(aggregate);
185        return this;
186    }
187
188    public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) {
189        if (aggregates != null && !aggregates.isEmpty()) {
190            this.aggregates.addAll(aggregates);
191        }
192        return this;
193    }
194
195    public int getLimit() {
196        return limit;
197    }
198
199    public int getOffset() {
200        return offset;
201    }
202
203    public List<SortInfo> getSortInfos() {
204        return sortInfos;
205    }
206
207    public String getNxql() {
208        return nxql;
209    }
210
211    public boolean isFetchFromElasticsearch() {
212        return fetchFromElasticsearch;
213    }
214
215    public CoreSession getSession() {
216        return session;
217    }
218
219    /**
220     * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor
221     * aggregates, use the udpateRequest to get the full request.
222     */
223    public QueryBuilder makeQuery() {
224        if (esQueryBuilder == null) {
225            if (nxql != null) {
226                esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session);
227                // handle the built-in order by clause
228                if (nxql.toLowerCase().contains("order by")) {
229                    List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql);
230                    sortInfos.addAll(builtInSortInfos);
231                }
232                if (nxqlHasSelectClause(nxql)) {
233                    selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql);
234                    Set<String> keySet = selectFieldsAndTypes.keySet();
235                    selectFields = keySet.toArray(new String[keySet.size()]);
236                    returnsDocuments = false;
237                }
238                esQueryBuilder = addSecurityFilter(esQueryBuilder);
239            }
240        }
241        return esQueryBuilder;
242    }
243
244    protected boolean nxqlHasSelectClause(String nxql) {
245        String lowerNxql = nxql.toLowerCase();
246        if (lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from")) {
247            return true;
248        }
249        return false;
250    }
251
252    public SortBuilder[] getSortBuilders() {
253        SortBuilder[] ret;
254        if (sortInfos.isEmpty()) {
255            return new SortBuilder[0];
256        }
257        ret = new SortBuilder[sortInfos.size()];
258        int i = 0;
259        for (SortInfo sortInfo : sortInfos) {
260            ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()).order(sortInfo.getSortAscending() ? SortOrder.ASC
261                    : SortOrder.DESC);
262        }
263        return ret;
264    }
265
266    protected FilterBuilder getAggregateFilter() {
267        boolean hasFilter = false;
268        AndFilterBuilder ret = FilterBuilders.andFilter();
269        for (AggregateEsBase agg : aggregates) {
270            FilterBuilder filter = agg.getEsFilter();
271            if (filter != null) {
272                ret.add(filter);
273                hasFilter = true;
274            }
275        }
276        if (!hasFilter) {
277            return null;
278        }
279        return ret;
280    }
281
282    protected FilterBuilder getAggregateFilterExceptFor(String id) {
283        boolean hasFilter = false;
284        AndFilterBuilder ret = FilterBuilders.andFilter();
285        for (AggregateEsBase agg : aggregates) {
286            if (!agg.getId().equals(id)) {
287                FilterBuilder filter = agg.getEsFilter();
288                if (filter != null) {
289                    ret.add(filter);
290                    hasFilter = true;
291                }
292            }
293        }
294        if (!hasFilter) {
295            return FilterBuilders.matchAllFilter();
296        }
297        return ret;
298    }
299
300    public List<AggregateEsBase<? extends Bucket>> getAggregates() {
301        return aggregates;
302    }
303
304    public List<FilterAggregationBuilder> getEsAggregates() {
305        List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size());
306        for (AggregateEsBase agg : aggregates) {
307            FilterAggregationBuilder fagg = new FilterAggregationBuilder(getAggregateFilterId(agg));
308            fagg.filter(getAggregateFilterExceptFor(agg.getId()));
309            fagg.subAggregation(agg.getEsAggregate());
310            ret.add(fagg);
311        }
312        return ret;
313    }
314
315    public void updateRequest(SearchRequestBuilder request) {
316        // Set limits
317        request.setFrom(getOffset()).setSize(getLimit());
318        // Build query with security checks
319        request.setQuery(makeQuery());
320        // Add sort
321        for (SortBuilder sortBuilder : getSortBuilders()) {
322            request.addSort(sortBuilder);
323        }
324        // Add Aggregate
325        for (AbstractAggregationBuilder aggregate : getEsAggregates()) {
326            request.addAggregation(aggregate);
327        }
328        // Add Aggregate post filter
329        FilterBuilder aggFilter = getAggregateFilter();
330        if (aggFilter != null) {
331            request.setPostFilter(aggFilter);
332        }
333        // Fields selection
334        if (!isFetchFromElasticsearch()) {
335            request.addFields(getSelectFields());
336        }
337
338    }
339
340    protected QueryBuilder addSecurityFilter(QueryBuilder query) {
341        AndFilterBuilder aclFilter;
342        Principal principal = session.getPrincipal();
343        if (principal == null
344                || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) {
345            return query;
346        }
347        String[] principals = SecurityService.getPrincipalsToCheck(principal);
348        // we want an ACL that match principals but we discard
349        // unsupported ACE that contains negative ACE
350        aclFilter = FilterBuilders.andFilter(FilterBuilders.inFilter(ACL_FIELD, principals),
351                FilterBuilders.notFilter(FilterBuilders.inFilter(ACL_FIELD, UNSUPPORTED_ACL)));
352        return QueryBuilders.filteredQuery(query, aclFilter);
353    }
354
355
356    /**
357     * Add a specific repository to search. Default search is done on the session repository only.
358     *
359     * @since 6.0
360     */
361    public NxQueryBuilder addSearchRepository(String repositoryName) {
362        repositories.add(repositoryName);
363        return this;
364    }
365
366    /**
367     * Search on all available repositories.
368     *
369     * @since 6.0
370     */
371    public NxQueryBuilder searchOnAllRepositories() {
372        searchOnAllRepo = true;
373        return this;
374    }
375
376    /**
377     * Return the list of repositories to search, or an empty list to search on all available repositories;
378     *
379     * @since 6.0
380     */
381    public List<String> getSearchRepositories() {
382        if (searchOnAllRepo) {
383            return Collections.<String>emptyList();
384        }
385        return repositories;
386    }
387
388    /**
389     * @since 6.0
390     */
391    public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) {
392        if (isFetchFromElasticsearch()) {
393            return new EsFetcher(session, response, repoNames);
394        }
395        return new VcsFetcher(session, response, repoNames);
396    }
397
398    /**
399     * @since 7.2
400     */
401    public String[] getSelectFields() {
402        return selectFields;
403    }
404
405    /**
406     * @since 7.2
407     */
408    public Map<String, Type> getSelectFieldsAndTypes() {
409        return selectFieldsAndTypes;
410    }
411
412    /**
413     * @since 7.2
414     */
415    public boolean returnsDocuments() {
416        if (esOnly) {
417            return false;
418        }
419        return returnsDocuments;
420    }
421
422    public boolean returnsRows() {
423        if (esOnly) {
424            return false;
425        }
426        return !returnsDocuments;
427    }
428}