001/* 002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019package org.nuxeo.elasticsearch.query; 020 021import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 022import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 023import static org.nuxeo.elasticsearch.ElasticSearchConstants.ES_SCORE_FIELD; 024import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 025 026import java.util.ArrayList; 027import java.util.Collections; 028import java.util.List; 029import java.util.Map; 030import java.util.Set; 031 032import org.elasticsearch.action.search.SearchResponse; 033import org.elasticsearch.index.query.BoolQueryBuilder; 034import org.elasticsearch.index.query.QueryBuilder; 035import org.elasticsearch.index.query.QueryBuilders; 036import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 037import org.elasticsearch.search.aggregations.Aggregation; 038import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 039import org.elasticsearch.search.builder.SearchSourceBuilder; 040import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; 041import org.elasticsearch.search.sort.FieldSortBuilder; 042import org.elasticsearch.search.sort.SortBuilder; 043import org.elasticsearch.search.sort.SortOrder; 044import org.nuxeo.ecm.core.api.CoreSession; 045import org.nuxeo.ecm.core.api.NuxeoPrincipal; 046import org.nuxeo.ecm.core.api.SortInfo; 047import org.nuxeo.ecm.core.schema.SchemaManager; 048import org.nuxeo.ecm.core.schema.types.Type; 049import org.nuxeo.ecm.core.security.SecurityService; 050import org.nuxeo.ecm.platform.query.api.Aggregate; 051import org.nuxeo.ecm.platform.query.api.Bucket; 052import org.nuxeo.elasticsearch.ElasticSearchConstants; 053import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 054import org.nuxeo.elasticsearch.api.EsResult; 055import org.nuxeo.elasticsearch.fetcher.EsFetcher; 056import org.nuxeo.elasticsearch.fetcher.Fetcher; 057import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 058import org.nuxeo.runtime.api.Framework; 059 060/** 061 * Elasticsearch query builder for the Nuxeo ES api. 062 * 063 * @since 5.9.5 064 */ 065public class NxQueryBuilder { 066 067 private static final int DEFAULT_LIMIT = 10; 068 069 private static final String AGG_FILTER_SUFFIX = "_filter"; 070 071 private final CoreSession session; 072 073 private final List<SortInfo> sortInfos = new ArrayList<>(); 074 075 private final List<String> repositories = new ArrayList<>(); 076 077 private final List<AggregateEsBase<Aggregation, Bucket>> aggregates = new ArrayList<>(); 078 079 private int limit = DEFAULT_LIMIT; 080 081 private int offset = 0; 082 083 private String nxql; 084 085 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 086 087 private boolean fetchFromElasticsearch = false; 088 089 private boolean searchOnAllRepo = false; 090 091 private String[] selectFields = { ElasticSearchConstants.ID_FIELD }; 092 093 private Map<String, Type> selectFieldsAndTypes; 094 095 private boolean returnsDocuments = true; 096 097 private boolean esOnly = false; 098 099 private List<String> highlightFields; 100 101 private EsFetcher.HitDocConsumer hitDocConsumer; 102 103 public NxQueryBuilder(CoreSession coreSession) { 104 session = coreSession; 105 repositories.add(coreSession.getRepositoryName()); 106 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 107 } 108 109 public static String getAggregateFilterId(Aggregate agg) { 110 return agg.getId() + AGG_FILTER_SUFFIX; 111 } 112 113 /** 114 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}. Since Nuxeo 8.4 and ES 2.x, we can 115 * not give -1 to this method as the default configuration on ES allows to have a search window of 10000 documents 116 * at maximum. This settings could be changed on ES by changing {index.max_result_window}, but it is preferable to 117 * use the scan & scroll API. 118 */ 119 public NxQueryBuilder limit(int limit) { 120 // For compatibility only, deprecated since 8.4 121 if (limit < 0) { 122 limit = Integer.MAX_VALUE; 123 } 124 this.limit = limit; 125 return this; 126 } 127 128 /** 129 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 130 * offset documents are skipped before starting to count the limit documents that are returned. 131 */ 132 public NxQueryBuilder offset(int offset) { 133 this.offset = offset; 134 return this; 135 } 136 137 public NxQueryBuilder addSort(SortInfo sortInfo) { 138 sortInfos.add(sortInfo); 139 return this; 140 } 141 142 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 143 if (sortInfos != null && sortInfos.length > 0) { 144 Collections.addAll(this.sortInfos, sortInfos); 145 } 146 return this; 147 } 148 149 /** 150 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 151 */ 152 public NxQueryBuilder nxql(String nxql) { 153 this.nxql = nxql; 154 this.esQueryBuilder = null; 155 return this; 156 } 157 158 /** 159 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 160 */ 161 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 162 esQueryBuilder = addSecurityFilter(queryBuilder); 163 nxql = null; 164 return this; 165 } 166 167 /** 168 * Ask for the Elasticsearch _source field, use it to build documents. 169 */ 170 public NxQueryBuilder fetchFromElasticsearch() { 171 fetchFromElasticsearch = true; 172 return this; 173 } 174 175 /** 176 * If search results are found, use this SearchHit and DocumentModel consumer on each hit. 177 * 178 * @since 10.2 179 */ 180 public NxQueryBuilder hitDocConsumer(EsFetcher.HitDocConsumer consumer) { 181 hitDocConsumer = consumer; 182 return this; 183 } 184 185 /** 186 * Fetch the documents using VCS (database) engine. This is done by default 187 */ 188 public NxQueryBuilder fetchFromDatabase() { 189 fetchFromElasticsearch = false; 190 return this; 191 } 192 193 /** 194 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 195 * {@link EsResult#getElasticsearchResponse()} 196 * 197 * @since 7.3 198 */ 199 public NxQueryBuilder onlyElasticsearchResponse() { 200 esOnly = true; 201 return this; 202 } 203 204 @SuppressWarnings("unchecked") 205 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Aggregation, ? extends Bucket> aggregate) { 206 aggregates.add((AggregateEsBase<Aggregation, Bucket>) aggregate); 207 return this; 208 } 209 210 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Aggregation, ? extends Bucket>> aggregates) { 211 if (aggregates != null && !aggregates.isEmpty()) { 212 aggregates.forEach(this::addAggregate); 213 } 214 return this; 215 } 216 217 /** 218 * @since 9.1 219 */ 220 public NxQueryBuilder highlight(List<String> highlightFields) { 221 this.highlightFields = highlightFields; 222 return this; 223 } 224 225 public int getLimit() { 226 return limit; 227 } 228 229 public int getOffset() { 230 return offset; 231 } 232 233 public List<SortInfo> getSortInfos() { 234 return sortInfos; 235 } 236 237 public String getNxql() { 238 return nxql; 239 } 240 241 public boolean isFetchFromElasticsearch() { 242 return fetchFromElasticsearch; 243 } 244 245 public CoreSession getSession() { 246 return session; 247 } 248 249 /** 250 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 251 * aggregates, use the udpateRequest to get the full request. 252 */ 253 public QueryBuilder makeQuery() { 254 if (esQueryBuilder == null) { 255 if (nxql != null) { 256 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 257 // handle the built-in order by clause 258 if (nxql.toLowerCase().contains("order by")) { 259 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 260 sortInfos.addAll(builtInSortInfos); 261 } 262 if (nxqlHasSelectClause(nxql)) { 263 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 264 Set<String> keySet = selectFieldsAndTypes.keySet(); 265 selectFields = keySet.toArray(new String[keySet.size()]); 266 returnsDocuments = false; 267 } 268 esQueryBuilder = addSecurityFilter(esQueryBuilder); 269 } 270 } 271 return esQueryBuilder; 272 } 273 274 protected boolean nxqlHasSelectClause(String nxql) { 275 String lowerNxql = nxql.toLowerCase(); 276 return lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from"); 277 } 278 279 public SortBuilder[] getSortBuilders() { 280 SortBuilder[] ret; 281 if (sortInfos.isEmpty()) { 282 return new SortBuilder[0]; 283 } 284 ret = new SortBuilder[sortInfos.size()]; 285 int i = 0; 286 for (SortInfo sortInfo : sortInfos) { 287 String fieldType = guessFieldType(sortInfo.getSortColumn()); 288 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()) 289 .order(sortInfo.getSortAscending() ? SortOrder.ASC 290 : SortOrder.DESC) 291 .unmappedType(fieldType); 292 } 293 return ret; 294 } 295 296 protected String guessFieldType(String field) { 297 String fieldType; 298 if (ES_SCORE_FIELD.equals(field)) { 299 // this special field should not have an unmappedType 300 return null; 301 } 302 try { 303 SchemaManager schemaManager = Framework.getService(SchemaManager.class); 304 fieldType = schemaManager.getField(field).getType().getName(); 305 } catch (NullPointerException e) { 306 // probably an internal field without schema 307 fieldType = "keyword"; 308 } 309 switch (fieldType) { 310 case "integer": 311 case "long": 312 case "boolean": 313 case "date": 314 return fieldType; 315 } 316 return "keyword"; 317 } 318 319 protected QueryBuilder getAggregateFilter() { 320 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 321 for (AggregateEsBase agg : aggregates) { 322 QueryBuilder filter = agg.getEsFilter(); 323 if (filter != null) { 324 ret.must(filter); 325 } 326 } 327 if (!ret.hasClauses()) { 328 return null; 329 } 330 return ret; 331 } 332 333 protected QueryBuilder getAggregateFilterExceptFor(String id) { 334 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 335 for (AggregateEsBase agg : aggregates) { 336 if (!agg.getId().equals(id)) { 337 QueryBuilder filter = agg.getEsFilter(); 338 if (filter != null) { 339 ret.must(filter); 340 } 341 } 342 } 343 if (!ret.hasClauses()) { 344 return QueryBuilders.matchAllQuery(); 345 } 346 return ret; 347 } 348 349 public List<AggregateEsBase<Aggregation, Bucket>> getAggregates() { 350 return aggregates; 351 } 352 353 public List<FilterAggregationBuilder> getEsAggregates() { 354 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 355 for (AggregateEsBase agg : aggregates) { 356 FilterAggregationBuilder fagg = null; 357 fagg = new FilterAggregationBuilder(getAggregateFilterId(agg), getAggregateFilterExceptFor(agg.getId())); 358 fagg.subAggregation(agg.getEsAggregate()); 359 ret.add(fagg); 360 } 361 return ret; 362 } 363 364 public void updateRequest(SearchSourceBuilder request) { 365 // Set limits 366 request.from(getOffset()).size(getLimit()); 367 // Build query with security checks 368 request.query(makeQuery()); 369 // Add sort 370 for (SortBuilder sortBuilder : getSortBuilders()) { 371 request.sort(sortBuilder); 372 } 373 // Add Aggregate 374 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 375 request.aggregation(aggregate); 376 } 377 // Add Aggregate post filter 378 QueryBuilder aggFilter = getAggregateFilter(); 379 if (aggFilter != null) { 380 request.postFilter(aggFilter); 381 } 382 383 // Add highlighting 384 if (highlightFields != null && !highlightFields.isEmpty()) { 385 HighlightBuilder hb = new HighlightBuilder(); 386 for (String field : highlightFields) { 387 hb.field(field); 388 } 389 hb.requireFieldMatch(false); 390 request.highlighter(hb); 391 } 392 // Fields selection 393 if (!isFetchFromElasticsearch()) { 394 request.fetchSource(getSelectFields(), null); 395 } 396 397 } 398 399 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 400 NuxeoPrincipal principal = session.getPrincipal(); 401 if (principal == null || principal.isAdministrator()) { 402 return query; 403 } 404 String[] principals = SecurityService.getPrincipalsToCheck(principal); 405 // we want an ACL that match principals but we discard 406 // unsupported ACE that contains negative ACE 407 QueryBuilder aclFilter = QueryBuilders.boolQuery() 408 .must(QueryBuilders.termsQuery(ACL_FIELD, principals)) 409 .mustNot(QueryBuilders.termsQuery(ACL_FIELD, UNSUPPORTED_ACL)); 410 return QueryBuilders.boolQuery().must(query).filter(aclFilter); 411 } 412 413 /** 414 * Add a specific repository to search. Default search is done on the session repository only. 415 * 416 * @since 6.0 417 */ 418 public NxQueryBuilder addSearchRepository(String repositoryName) { 419 repositories.add(repositoryName); 420 return this; 421 } 422 423 /** 424 * Search on all available repositories. 425 * 426 * @since 6.0 427 */ 428 public NxQueryBuilder searchOnAllRepositories() { 429 searchOnAllRepo = true; 430 return this; 431 } 432 433 /** 434 * Return the list of repositories to search, or an empty list to search on all available repositories; 435 * 436 * @since 6.0 437 */ 438 public List<String> getSearchRepositories() { 439 if (searchOnAllRepo) { 440 return Collections.<String> emptyList(); 441 } 442 return repositories; 443 } 444 445 /** 446 * @since 6.0 447 */ 448 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 449 if (isFetchFromElasticsearch()) { 450 return new EsFetcher(session, response, repoNames, hitDocConsumer); 451 } 452 return new VcsFetcher(session, response, repoNames); 453 } 454 455 /** 456 * @since 7.2 457 */ 458 public String[] getSelectFields() { 459 return selectFields; 460 } 461 462 /** 463 * @since 7.2 464 */ 465 public Map<String, Type> getSelectFieldsAndTypes() { 466 return selectFieldsAndTypes; 467 } 468 469 /** 470 * @since 7.2 471 */ 472 public boolean returnsDocuments() { 473 if (esOnly) { 474 return false; 475 } 476 return returnsDocuments; 477 } 478 479 public boolean returnsRows() { 480 if (esOnly) { 481 return false; 482 } 483 return !returnsDocuments; 484 } 485}