001/* 002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019package org.nuxeo.elasticsearch.query; 020 021import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 022import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 023import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 024 025import java.security.Principal; 026import java.util.ArrayList; 027import java.util.Collections; 028import java.util.List; 029import java.util.Map; 030import java.util.Set; 031 032import org.elasticsearch.action.search.SearchRequestBuilder; 033import org.elasticsearch.action.search.SearchResponse; 034import org.elasticsearch.index.query.BoolQueryBuilder; 035import org.elasticsearch.index.query.QueryBuilder; 036import org.elasticsearch.index.query.QueryBuilders; 037import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 038import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 039import org.elasticsearch.search.sort.FieldSortBuilder; 040import org.elasticsearch.search.sort.SortBuilder; 041import org.elasticsearch.search.sort.SortOrder; 042import org.nuxeo.ecm.core.api.CoreSession; 043import org.nuxeo.ecm.core.api.NuxeoPrincipal; 044import org.nuxeo.ecm.core.api.SortInfo; 045import org.nuxeo.ecm.core.schema.types.Type; 046import org.nuxeo.ecm.core.security.SecurityService; 047import org.nuxeo.ecm.platform.query.api.Aggregate; 048import org.nuxeo.ecm.platform.query.api.Bucket; 049import org.nuxeo.elasticsearch.ElasticSearchConstants; 050import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 051import org.nuxeo.elasticsearch.api.EsResult; 052import org.nuxeo.elasticsearch.fetcher.EsFetcher; 053import org.nuxeo.elasticsearch.fetcher.Fetcher; 054import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 055import org.nuxeo.runtime.api.Framework; 056 057/** 058 * Elasticsearch query builder for the Nuxeo ES api. 059 * 060 * @since 5.9.5 061 */ 062public class NxQueryBuilder { 063 064 private static final int DEFAULT_LIMIT = 10; 065 066 private int limit = DEFAULT_LIMIT; 067 068 private static final String AGG_FILTER_SUFFIX = "_filter"; 069 070 private final CoreSession session; 071 072 private final List<SortInfo> sortInfos = new ArrayList<>(); 073 074 private final List<String> repositories = new ArrayList<>(); 075 076 private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>(); 077 078 private int offset = 0; 079 080 private String nxql; 081 082 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 083 084 private boolean fetchFromElasticsearch = false; 085 086 private boolean searchOnAllRepo = false; 087 088 private String[] selectFields = { ElasticSearchConstants.ID_FIELD }; 089 090 private Map<String, Type> selectFieldsAndTypes; 091 092 private boolean returnsDocuments = true; 093 094 private boolean esOnly = false; 095 096 public NxQueryBuilder(CoreSession coreSession) { 097 session = coreSession; 098 repositories.add(coreSession.getRepositoryName()); 099 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 100 } 101 102 public static String getAggregateFilterId(Aggregate agg) { 103 return agg.getId() + AGG_FILTER_SUFFIX; 104 } 105 106 /** 107 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}. Since Nuxeo 8.4 and ES 2.x, we can 108 * not give -1 to this method as the default configuration on ES allows to have a search window of 10000 documents 109 * at maximum. This settings could be changed on ES by changing {index.max_result_window}, but it is preferable to 110 * use the scan & scroll API. 111 */ 112 public NxQueryBuilder limit(int limit) { 113 // For compatibility only, deprecated since 8.4 114 if (limit < 0) { 115 limit = Integer.MAX_VALUE; 116 } 117 this.limit = limit; 118 return this; 119 } 120 121 /** 122 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 123 * offset documents are skipped before starting to count the limit documents that are returned. 124 */ 125 public NxQueryBuilder offset(int offset) { 126 this.offset = offset; 127 return this; 128 } 129 130 public NxQueryBuilder addSort(SortInfo sortInfo) { 131 sortInfos.add(sortInfo); 132 return this; 133 } 134 135 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 136 if (sortInfos != null && sortInfos.length > 0) { 137 Collections.addAll(this.sortInfos, sortInfos); 138 } 139 return this; 140 } 141 142 /** 143 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 144 */ 145 public NxQueryBuilder nxql(String nxql) { 146 this.nxql = nxql; 147 this.esQueryBuilder = null; 148 return this; 149 } 150 151 /** 152 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 153 */ 154 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 155 esQueryBuilder = addSecurityFilter(queryBuilder); 156 nxql = null; 157 return this; 158 } 159 160 /** 161 * Ask for the Elasticsearch _source field, use it to build documents. 162 */ 163 public NxQueryBuilder fetchFromElasticsearch() { 164 fetchFromElasticsearch = true; 165 return this; 166 } 167 168 /** 169 * Fetch the documents using VCS (database) engine. This is done by default 170 */ 171 public NxQueryBuilder fetchFromDatabase() { 172 fetchFromElasticsearch = false; 173 return this; 174 } 175 176 /** 177 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 178 * {@link EsResult#getElasticsearchResponse()} 179 * 180 * @since 7.3 181 */ 182 public NxQueryBuilder onlyElasticsearchResponse() { 183 esOnly = true; 184 return this; 185 } 186 187 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) { 188 aggregates.add(aggregate); 189 return this; 190 } 191 192 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) { 193 if (aggregates != null && !aggregates.isEmpty()) { 194 this.aggregates.addAll(aggregates); 195 } 196 return this; 197 } 198 199 public int getLimit() { 200 return limit; 201 } 202 203 public int getOffset() { 204 return offset; 205 } 206 207 public List<SortInfo> getSortInfos() { 208 return sortInfos; 209 } 210 211 public String getNxql() { 212 return nxql; 213 } 214 215 public boolean isFetchFromElasticsearch() { 216 return fetchFromElasticsearch; 217 } 218 219 public CoreSession getSession() { 220 return session; 221 } 222 223 /** 224 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 225 * aggregates, use the udpateRequest to get the full request. 226 */ 227 public QueryBuilder makeQuery() { 228 if (esQueryBuilder == null) { 229 if (nxql != null) { 230 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 231 // handle the built-in order by clause 232 if (nxql.toLowerCase().contains("order by")) { 233 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 234 sortInfos.addAll(builtInSortInfos); 235 } 236 if (nxqlHasSelectClause(nxql)) { 237 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 238 Set<String> keySet = selectFieldsAndTypes.keySet(); 239 selectFields = keySet.toArray(new String[keySet.size()]); 240 returnsDocuments = false; 241 } 242 esQueryBuilder = addSecurityFilter(esQueryBuilder); 243 } 244 } 245 return esQueryBuilder; 246 } 247 248 protected boolean nxqlHasSelectClause(String nxql) { 249 String lowerNxql = nxql.toLowerCase(); 250 return lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from"); 251 } 252 253 public SortBuilder[] getSortBuilders() { 254 SortBuilder[] ret; 255 if (sortInfos.isEmpty()) { 256 return new SortBuilder[0]; 257 } 258 ret = new SortBuilder[sortInfos.size()]; 259 int i = 0; 260 for (SortInfo sortInfo : sortInfos) { 261 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()).order( 262 sortInfo.getSortAscending() ? SortOrder.ASC : SortOrder.DESC); 263 } 264 return ret; 265 } 266 267 protected QueryBuilder getAggregateFilter() { 268 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 269 for (AggregateEsBase agg : aggregates) { 270 QueryBuilder filter = agg.getEsFilter(); 271 if (filter != null) { 272 ret.must(filter); 273 } 274 } 275 if (!ret.hasClauses()) { 276 return null; 277 } 278 return ret; 279 } 280 281 protected QueryBuilder getAggregateFilterExceptFor(String id) { 282 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 283 for (AggregateEsBase agg : aggregates) { 284 if (!agg.getId().equals(id)) { 285 QueryBuilder filter = agg.getEsFilter(); 286 if (filter != null) { 287 ret.must(filter); 288 } 289 } 290 } 291 if (!ret.hasClauses()) { 292 return QueryBuilders.matchAllQuery(); 293 } 294 return ret; 295 } 296 297 public List<AggregateEsBase<? extends Bucket>> getAggregates() { 298 return aggregates; 299 } 300 301 public List<FilterAggregationBuilder> getEsAggregates() { 302 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 303 for (AggregateEsBase agg : aggregates) { 304 FilterAggregationBuilder fagg = new FilterAggregationBuilder(getAggregateFilterId(agg)); 305 fagg.filter(getAggregateFilterExceptFor(agg.getId())); 306 fagg.subAggregation(agg.getEsAggregate()); 307 ret.add(fagg); 308 } 309 return ret; 310 } 311 312 public void updateRequest(SearchRequestBuilder request) { 313 // Set limits 314 request.setFrom(getOffset()).setSize(getLimit()); 315 // Build query with security checks 316 request.setQuery(makeQuery()); 317 // Add sort 318 for (SortBuilder sortBuilder : getSortBuilders()) { 319 request.addSort(sortBuilder); 320 } 321 // Add Aggregate 322 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 323 request.addAggregation(aggregate); 324 } 325 // Add Aggregate post filter 326 QueryBuilder aggFilter = getAggregateFilter(); 327 if (aggFilter != null) { 328 request.setPostFilter(aggFilter); 329 } 330 // Fields selection 331 if (!isFetchFromElasticsearch()) { 332 request.addFields(getSelectFields()); 333 } 334 335 } 336 337 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 338 Principal principal = session.getPrincipal(); 339 if (principal == null 340 || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) { 341 return query; 342 } 343 String[] principals = SecurityService.getPrincipalsToCheck(principal); 344 // we want an ACL that match principals but we discard 345 // unsupported ACE that contains negative ACE 346 QueryBuilder aclFilter = QueryBuilders.boolQuery() 347 .must(QueryBuilders.termsQuery(ACL_FIELD, principals)) 348 .mustNot(QueryBuilders.termsQuery(ACL_FIELD, UNSUPPORTED_ACL)); 349 return QueryBuilders.boolQuery().must(query).filter(aclFilter); 350 } 351 352 /** 353 * Add a specific repository to search. Default search is done on the session repository only. 354 * 355 * @since 6.0 356 */ 357 public NxQueryBuilder addSearchRepository(String repositoryName) { 358 repositories.add(repositoryName); 359 return this; 360 } 361 362 /** 363 * Search on all available repositories. 364 * 365 * @since 6.0 366 */ 367 public NxQueryBuilder searchOnAllRepositories() { 368 searchOnAllRepo = true; 369 return this; 370 } 371 372 /** 373 * Return the list of repositories to search, or an empty list to search on all available repositories; 374 * 375 * @since 6.0 376 */ 377 public List<String> getSearchRepositories() { 378 if (searchOnAllRepo) { 379 return Collections.<String> emptyList(); 380 } 381 return repositories; 382 } 383 384 /** 385 * @since 6.0 386 */ 387 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 388 if (isFetchFromElasticsearch()) { 389 return new EsFetcher(session, response, repoNames); 390 } 391 return new VcsFetcher(session, response, repoNames); 392 } 393 394 /** 395 * @since 7.2 396 */ 397 public String[] getSelectFields() { 398 return selectFields; 399 } 400 401 /** 402 * @since 7.2 403 */ 404 public Map<String, Type> getSelectFieldsAndTypes() { 405 return selectFieldsAndTypes; 406 } 407 408 /** 409 * @since 7.2 410 */ 411 public boolean returnsDocuments() { 412 if (esOnly) { 413 return false; 414 } 415 return returnsDocuments; 416 } 417 418 public boolean returnsRows() { 419 if (esOnly) { 420 return false; 421 } 422 return !returnsDocuments; 423 } 424}