001/*
002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     bdelbosc
018 */
019package org.nuxeo.elasticsearch.query;
020
021import org.elasticsearch.action.search.SearchRequestBuilder;
022import org.elasticsearch.action.search.SearchResponse;
023import org.elasticsearch.index.query.AndFilterBuilder;
024import org.elasticsearch.index.query.FilterBuilder;
025import org.elasticsearch.index.query.FilterBuilders;
026import org.elasticsearch.index.query.QueryBuilder;
027import org.elasticsearch.index.query.QueryBuilders;
028import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
029import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
030import org.elasticsearch.search.sort.FieldSortBuilder;
031import org.elasticsearch.search.sort.SortBuilder;
032import org.elasticsearch.search.sort.SortOrder;
033import org.nuxeo.ecm.core.api.CoreSession;
034import org.nuxeo.ecm.core.api.NuxeoPrincipal;
035import org.nuxeo.ecm.core.api.SortInfo;
036import org.nuxeo.ecm.core.schema.types.Type;
037import org.nuxeo.ecm.core.security.SecurityService;
038import org.nuxeo.ecm.platform.query.api.Aggregate;
039import org.nuxeo.ecm.platform.query.api.Bucket;
040import org.nuxeo.elasticsearch.ElasticSearchConstants;
041import org.nuxeo.elasticsearch.aggregate.AggregateEsBase;
042import org.nuxeo.elasticsearch.api.EsResult;
043import org.nuxeo.elasticsearch.fetcher.EsFetcher;
044import org.nuxeo.elasticsearch.fetcher.Fetcher;
045import org.nuxeo.elasticsearch.fetcher.VcsFetcher;
046import org.nuxeo.runtime.api.Framework;
047
048import java.security.Principal;
049import java.util.ArrayList;
050import java.util.Collections;
051import java.util.List;
052import java.util.Map;
053import java.util.Set;
054
055import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL;
056import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD;
057import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY;
058
059/**
060 * Elasticsearch query buidler for the Nuxeo ES api.
061 *
062 * @since 5.9.5
063 */
064public class NxQueryBuilder {
065
066    private static final int DEFAULT_LIMIT = 10;
067
068    private int limit = DEFAULT_LIMIT;
069
070    private static final String AGG_FILTER_SUFFIX = "_filter";
071
072    private final CoreSession session;
073
074    private final List<SortInfo> sortInfos = new ArrayList<>();
075
076    private final List<String> repositories = new ArrayList<>();
077
078    private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>();
079
080    private int offset = 0;
081
082    private String nxql;
083
084    private org.elasticsearch.index.query.QueryBuilder esQueryBuilder;
085
086    private boolean fetchFromElasticsearch = false;
087
088    private boolean searchOnAllRepo = false;
089
090    private String[] selectFields = {ElasticSearchConstants.ID_FIELD};
091
092    private Map<String, Type> selectFieldsAndTypes;
093
094    private boolean returnsDocuments = true;
095
096    private boolean esOnly = false;
097
098    public NxQueryBuilder(CoreSession coreSession) {
099        session = coreSession;
100        repositories.add(coreSession.getRepositoryName());
101        fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false"));
102    }
103
104    public static String getAggregateFilterId(Aggregate agg) {
105        return agg.getId() + AGG_FILTER_SUFFIX;
106    }
107
108    /**
109     * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}, Use -1 to return all documents.
110     */
111    public NxQueryBuilder limit(int limit) {
112        if (limit < 0) {
113            limit = Integer.MAX_VALUE;
114        }
115        this.limit = limit;
116        return this;
117    }
118
119    /**
120     * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then
121     * offset documents are skipped before starting to count the limit documents that are returned.
122     */
123    public NxQueryBuilder offset(int offset) {
124        this.offset = offset;
125        return this;
126    }
127
128    public NxQueryBuilder addSort(SortInfo sortInfo) {
129        sortInfos.add(sortInfo);
130        return this;
131    }
132
133    public NxQueryBuilder addSort(SortInfo[] sortInfos) {
134        if (sortInfos != null && sortInfos.length > 0) {
135            Collections.addAll(this.sortInfos, sortInfos);
136        }
137        return this;
138    }
139
140    /**
141     * Build the query from a NXQL string. You should either use nxql, either esQuery, not both.
142     */
143    public NxQueryBuilder nxql(String nxql) {
144        this.nxql = nxql;
145        this.esQueryBuilder = null;
146        return this;
147    }
148
149    /**
150     * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both.
151     */
152    public NxQueryBuilder esQuery(QueryBuilder queryBuilder) {
153        this.esQueryBuilder = queryBuilder;
154        nxql = null;
155        return this;
156    }
157
158    /**
159     * Ask for the Elasticsearch _source field, use it to build documents.
160     */
161    public NxQueryBuilder fetchFromElasticsearch() {
162        fetchFromElasticsearch = true;
163        return this;
164    }
165
166    /**
167     * Fetch the documents using VCS (database) engine. This is done by default
168     */
169    public NxQueryBuilder fetchFromDatabase() {
170        fetchFromElasticsearch = false;
171        return this;
172    }
173
174    /**
175     * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from
176     * {@link EsResult#getElasticsearchResponse()}
177     *
178     * @since 7.3
179     */
180    public NxQueryBuilder onlyElasticsearchResponse() {
181        esOnly = true;
182        return this;
183    }
184
185    public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) {
186        aggregates.add(aggregate);
187        return this;
188    }
189
190    public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) {
191        if (aggregates != null && !aggregates.isEmpty()) {
192            this.aggregates.addAll(aggregates);
193        }
194        return this;
195    }
196
197    public int getLimit() {
198        return limit;
199    }
200
201    public int getOffset() {
202        return offset;
203    }
204
205    public List<SortInfo> getSortInfos() {
206        return sortInfos;
207    }
208
209    public String getNxql() {
210        return nxql;
211    }
212
213    public boolean isFetchFromElasticsearch() {
214        return fetchFromElasticsearch;
215    }
216
217    public CoreSession getSession() {
218        return session;
219    }
220
221    /**
222     * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor
223     * aggregates, use the udpateRequest to get the full request.
224     */
225    public QueryBuilder makeQuery() {
226        if (esQueryBuilder == null) {
227            if (nxql != null) {
228                esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session);
229                // handle the built-in order by clause
230                if (nxql.toLowerCase().contains("order by")) {
231                    List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql);
232                    sortInfos.addAll(builtInSortInfos);
233                }
234                if (nxqlHasSelectClause(nxql)) {
235                    selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql);
236                    Set<String> keySet = selectFieldsAndTypes.keySet();
237                    selectFields = keySet.toArray(new String[keySet.size()]);
238                    returnsDocuments = false;
239                }
240                esQueryBuilder = addSecurityFilter(esQueryBuilder);
241            }
242        }
243        return esQueryBuilder;
244    }
245
246    protected boolean nxqlHasSelectClause(String nxql) {
247        String lowerNxql = nxql.toLowerCase();
248        if (lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from")) {
249            return true;
250        }
251        return false;
252    }
253
254    public SortBuilder[] getSortBuilders() {
255        SortBuilder[] ret;
256        if (sortInfos.isEmpty()) {
257            return new SortBuilder[0];
258        }
259        ret = new SortBuilder[sortInfos.size()];
260        int i = 0;
261        for (SortInfo sortInfo : sortInfos) {
262            ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()).order(sortInfo.getSortAscending() ? SortOrder.ASC
263                    : SortOrder.DESC);
264        }
265        return ret;
266    }
267
268    protected FilterBuilder getAggregateFilter() {
269        boolean hasFilter = false;
270        AndFilterBuilder ret = FilterBuilders.andFilter();
271        for (AggregateEsBase agg : aggregates) {
272            FilterBuilder filter = agg.getEsFilter();
273            if (filter != null) {
274                ret.add(filter);
275                hasFilter = true;
276            }
277        }
278        if (!hasFilter) {
279            return null;
280        }
281        return ret;
282    }
283
284    protected FilterBuilder getAggregateFilterExceptFor(String id) {
285        boolean hasFilter = false;
286        AndFilterBuilder ret = FilterBuilders.andFilter();
287        for (AggregateEsBase agg : aggregates) {
288            if (!agg.getId().equals(id)) {
289                FilterBuilder filter = agg.getEsFilter();
290                if (filter != null) {
291                    ret.add(filter);
292                    hasFilter = true;
293                }
294            }
295        }
296        if (!hasFilter) {
297            return FilterBuilders.matchAllFilter();
298        }
299        return ret;
300    }
301
302    public List<AggregateEsBase<? extends Bucket>> getAggregates() {
303        return aggregates;
304    }
305
306    public List<FilterAggregationBuilder> getEsAggregates() {
307        List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size());
308        for (AggregateEsBase agg : aggregates) {
309            FilterAggregationBuilder fagg = new FilterAggregationBuilder(getAggregateFilterId(agg));
310            fagg.filter(getAggregateFilterExceptFor(agg.getId()));
311            fagg.subAggregation(agg.getEsAggregate());
312            ret.add(fagg);
313        }
314        return ret;
315    }
316
317    public void updateRequest(SearchRequestBuilder request) {
318        // Set limits
319        request.setFrom(getOffset()).setSize(getLimit());
320        // Build query with security checks
321        request.setQuery(makeQuery());
322        // Add sort
323        for (SortBuilder sortBuilder : getSortBuilders()) {
324            request.addSort(sortBuilder);
325        }
326        // Add Aggregate
327        for (AbstractAggregationBuilder aggregate : getEsAggregates()) {
328            request.addAggregation(aggregate);
329        }
330        // Add Aggregate post filter
331        FilterBuilder aggFilter = getAggregateFilter();
332        if (aggFilter != null) {
333            request.setPostFilter(aggFilter);
334        }
335        // Fields selection
336        if (!isFetchFromElasticsearch()) {
337            request.addFields(getSelectFields());
338        }
339
340    }
341
342    protected QueryBuilder addSecurityFilter(QueryBuilder query) {
343        AndFilterBuilder aclFilter;
344        Principal principal = session.getPrincipal();
345        if (principal == null
346                || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) {
347            return query;
348        }
349        String[] principals = SecurityService.getPrincipalsToCheck(principal);
350        // we want an ACL that match principals but we discard
351        // unsupported ACE that contains negative ACE
352        aclFilter = FilterBuilders.andFilter(FilterBuilders.inFilter(ACL_FIELD, principals),
353                FilterBuilders.notFilter(FilterBuilders.inFilter(ACL_FIELD, UNSUPPORTED_ACL)));
354        return QueryBuilders.filteredQuery(query, aclFilter);
355    }
356
357
358    /**
359     * Add a specific repository to search. Default search is done on the session repository only.
360     *
361     * @since 6.0
362     */
363    public NxQueryBuilder addSearchRepository(String repositoryName) {
364        repositories.add(repositoryName);
365        return this;
366    }
367
368    /**
369     * Search on all available repositories.
370     *
371     * @since 6.0
372     */
373    public NxQueryBuilder searchOnAllRepositories() {
374        searchOnAllRepo = true;
375        return this;
376    }
377
378    /**
379     * Return the list of repositories to search, or an empty list to search on all available repositories;
380     *
381     * @since 6.0
382     */
383    public List<String> getSearchRepositories() {
384        if (searchOnAllRepo) {
385            return Collections.<String>emptyList();
386        }
387        return repositories;
388    }
389
390    /**
391     * @since 6.0
392     */
393    public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) {
394        if (isFetchFromElasticsearch()) {
395            return new EsFetcher(session, response, repoNames);
396        }
397        return new VcsFetcher(session, response, repoNames);
398    }
399
400    /**
401     * @since 7.2
402     */
403    public String[] getSelectFields() {
404        return selectFields;
405    }
406
407    /**
408     * @since 7.2
409     */
410    public Map<String, Type> getSelectFieldsAndTypes() {
411        return selectFieldsAndTypes;
412    }
413
414    /**
415     * @since 7.2
416     */
417    public boolean returnsDocuments() {
418        if (esOnly) {
419            return false;
420        }
421        return returnsDocuments;
422    }
423
424    public boolean returnsRows() {
425        if (esOnly) {
426            return false;
427        }
428        return !returnsDocuments;
429    }
430}