001/* 002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019package org.nuxeo.elasticsearch.query; 020 021import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 022import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 023import static org.nuxeo.elasticsearch.ElasticSearchConstants.ES_SCORE_FIELD; 024import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 025 026import java.security.Principal; 027import java.util.ArrayList; 028import java.util.Collections; 029import java.util.List; 030import java.util.Map; 031import java.util.Set; 032 033import org.elasticsearch.action.search.SearchResponse; 034import org.elasticsearch.index.query.BoolQueryBuilder; 035import org.elasticsearch.index.query.QueryBuilder; 036import org.elasticsearch.index.query.QueryBuilders; 037import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 038import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 039import org.elasticsearch.search.builder.SearchSourceBuilder; 040import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; 041import org.elasticsearch.search.sort.FieldSortBuilder; 042import org.elasticsearch.search.sort.SortBuilder; 043import org.elasticsearch.search.sort.SortOrder; 044import org.nuxeo.ecm.core.api.CoreSession; 045import org.nuxeo.ecm.core.api.NuxeoPrincipal; 046import org.nuxeo.ecm.core.api.SortInfo; 047import org.nuxeo.ecm.core.schema.SchemaManager; 048import org.nuxeo.ecm.core.schema.types.Type; 049import org.nuxeo.ecm.core.security.SecurityService; 050import org.nuxeo.ecm.platform.query.api.Aggregate; 051import org.nuxeo.ecm.platform.query.api.Bucket; 052import org.nuxeo.elasticsearch.ElasticSearchConstants; 053import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 054import org.nuxeo.elasticsearch.api.EsResult; 055import org.nuxeo.elasticsearch.fetcher.EsFetcher; 056import org.nuxeo.elasticsearch.fetcher.Fetcher; 057import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 058import org.nuxeo.runtime.api.Framework; 059 060/** 061 * Elasticsearch query builder for the Nuxeo ES api. 062 * 063 * @since 5.9.5 064 */ 065public class NxQueryBuilder { 066 067 private static final int DEFAULT_LIMIT = 10; 068 069 private static final String AGG_FILTER_SUFFIX = "_filter"; 070 071 private final CoreSession session; 072 073 private final List<SortInfo> sortInfos = new ArrayList<>(); 074 075 private final List<String> repositories = new ArrayList<>(); 076 077 private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>(); 078 079 private int limit = DEFAULT_LIMIT; 080 081 private int offset = 0; 082 083 private String nxql; 084 085 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 086 087 private boolean fetchFromElasticsearch = false; 088 089 private boolean searchOnAllRepo = false; 090 091 private String[] selectFields = { ElasticSearchConstants.ID_FIELD }; 092 093 private Map<String, Type> selectFieldsAndTypes; 094 095 private boolean returnsDocuments = true; 096 097 private boolean esOnly = false; 098 099 private List<String> highlightFields; 100 101 private EsFetcher.HitDocConsumer hitDocConsumer; 102 103 public NxQueryBuilder(CoreSession coreSession) { 104 session = coreSession; 105 repositories.add(coreSession.getRepositoryName()); 106 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 107 } 108 109 public static String getAggregateFilterId(Aggregate agg) { 110 return agg.getId() + AGG_FILTER_SUFFIX; 111 } 112 113 /** 114 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}. Since Nuxeo 8.4 and ES 2.x, we can 115 * not give -1 to this method as the default configuration on ES allows to have a search window of 10000 documents 116 * at maximum. This settings could be changed on ES by changing {index.max_result_window}, but it is preferable to 117 * use the scan & scroll API. 118 */ 119 public NxQueryBuilder limit(int limit) { 120 // For compatibility only, deprecated since 8.4 121 if (limit < 0) { 122 limit = Integer.MAX_VALUE; 123 } 124 this.limit = limit; 125 return this; 126 } 127 128 /** 129 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 130 * offset documents are skipped before starting to count the limit documents that are returned. 131 */ 132 public NxQueryBuilder offset(int offset) { 133 this.offset = offset; 134 return this; 135 } 136 137 public NxQueryBuilder addSort(SortInfo sortInfo) { 138 sortInfos.add(sortInfo); 139 return this; 140 } 141 142 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 143 if (sortInfos != null && sortInfos.length > 0) { 144 Collections.addAll(this.sortInfos, sortInfos); 145 } 146 return this; 147 } 148 149 /** 150 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 151 */ 152 public NxQueryBuilder nxql(String nxql) { 153 this.nxql = nxql; 154 this.esQueryBuilder = null; 155 return this; 156 } 157 158 /** 159 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 160 */ 161 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 162 esQueryBuilder = addSecurityFilter(queryBuilder); 163 nxql = null; 164 return this; 165 } 166 167 /** 168 * Ask for the Elasticsearch _source field, use it to build documents. 169 */ 170 public NxQueryBuilder fetchFromElasticsearch() { 171 fetchFromElasticsearch = true; 172 return this; 173 } 174 175 /** 176 * If search results are found, use this SearchHit and DocumentModel consumer on each hit. 177 * @since 10.2 178 */ 179 public NxQueryBuilder hitDocConsumer(EsFetcher.HitDocConsumer consumer) { 180 hitDocConsumer = consumer; 181 return this; 182 } 183 184 /** 185 * Fetch the documents using VCS (database) engine. This is done by default 186 */ 187 public NxQueryBuilder fetchFromDatabase() { 188 fetchFromElasticsearch = false; 189 return this; 190 } 191 192 /** 193 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 194 * {@link EsResult#getElasticsearchResponse()} 195 * 196 * @since 7.3 197 */ 198 public NxQueryBuilder onlyElasticsearchResponse() { 199 esOnly = true; 200 return this; 201 } 202 203 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) { 204 aggregates.add(aggregate); 205 return this; 206 } 207 208 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) { 209 if (aggregates != null && !aggregates.isEmpty()) { 210 this.aggregates.addAll(aggregates); 211 } 212 return this; 213 } 214 215 /** 216 * @since 9.1 217 */ 218 public NxQueryBuilder highlight(List<String> highlightFields) { 219 this.highlightFields = highlightFields; 220 return this; 221 } 222 223 public int getLimit() { 224 return limit; 225 } 226 227 public int getOffset() { 228 return offset; 229 } 230 231 public List<SortInfo> getSortInfos() { 232 return sortInfos; 233 } 234 235 public String getNxql() { 236 return nxql; 237 } 238 239 public boolean isFetchFromElasticsearch() { 240 return fetchFromElasticsearch; 241 } 242 243 public CoreSession getSession() { 244 return session; 245 } 246 247 /** 248 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 249 * aggregates, use the udpateRequest to get the full request. 250 */ 251 public QueryBuilder makeQuery() { 252 if (esQueryBuilder == null) { 253 if (nxql != null) { 254 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 255 // handle the built-in order by clause 256 if (nxql.toLowerCase().contains("order by")) { 257 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 258 sortInfos.addAll(builtInSortInfos); 259 } 260 if (nxqlHasSelectClause(nxql)) { 261 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 262 Set<String> keySet = selectFieldsAndTypes.keySet(); 263 selectFields = keySet.toArray(new String[keySet.size()]); 264 returnsDocuments = false; 265 } 266 esQueryBuilder = addSecurityFilter(esQueryBuilder); 267 } 268 } 269 return esQueryBuilder; 270 } 271 272 protected boolean nxqlHasSelectClause(String nxql) { 273 String lowerNxql = nxql.toLowerCase(); 274 return lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from"); 275 } 276 277 public SortBuilder[] getSortBuilders() { 278 SortBuilder[] ret; 279 if (sortInfos.isEmpty()) { 280 return new SortBuilder[0]; 281 } 282 ret = new SortBuilder[sortInfos.size()]; 283 int i = 0; 284 for (SortInfo sortInfo : sortInfos) { 285 String fieldType = guessFieldType(sortInfo.getSortColumn()); 286 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()) 287 .order(sortInfo.getSortAscending() ? SortOrder.ASC 288 : SortOrder.DESC) 289 .unmappedType(fieldType); 290 } 291 return ret; 292 } 293 294 protected String guessFieldType(String field) { 295 String fieldType; 296 if (ES_SCORE_FIELD.equals(field)) { 297 // this special field should not have an unmappedType 298 return null; 299 } 300 try { 301 SchemaManager schemaManager = Framework.getService(SchemaManager.class); 302 fieldType = schemaManager.getField(field).getType().getName(); 303 } catch (NullPointerException e) { 304 // probably an internal field without schema 305 fieldType = "keyword"; 306 } 307 switch (fieldType) { 308 case "integer": 309 case "long": 310 case "boolean": 311 case "date": 312 return fieldType; 313 } 314 return "keyword"; 315 } 316 317 protected QueryBuilder getAggregateFilter() { 318 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 319 for (AggregateEsBase agg : aggregates) { 320 QueryBuilder filter = agg.getEsFilter(); 321 if (filter != null) { 322 ret.must(filter); 323 } 324 } 325 if (!ret.hasClauses()) { 326 return null; 327 } 328 return ret; 329 } 330 331 protected QueryBuilder getAggregateFilterExceptFor(String id) { 332 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 333 for (AggregateEsBase agg : aggregates) { 334 if (!agg.getId().equals(id)) { 335 QueryBuilder filter = agg.getEsFilter(); 336 if (filter != null) { 337 ret.must(filter); 338 } 339 } 340 } 341 if (!ret.hasClauses()) { 342 return QueryBuilders.matchAllQuery(); 343 } 344 return ret; 345 } 346 347 public List<AggregateEsBase<? extends Bucket>> getAggregates() { 348 return aggregates; 349 } 350 351 public List<FilterAggregationBuilder> getEsAggregates() { 352 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 353 for (AggregateEsBase agg : aggregates) { 354 FilterAggregationBuilder fagg = null; 355 fagg = new FilterAggregationBuilder(getAggregateFilterId(agg), getAggregateFilterExceptFor(agg.getId())); 356 fagg.subAggregation(agg.getEsAggregate()); 357 ret.add(fagg); 358 } 359 return ret; 360 } 361 362 public void updateRequest(SearchSourceBuilder request) { 363 // Set limits 364 request.from(getOffset()).size(getLimit()); 365 // Build query with security checks 366 request.query(makeQuery()); 367 // Add sort 368 for (SortBuilder sortBuilder : getSortBuilders()) { 369 request.sort(sortBuilder); 370 } 371 // Add Aggregate 372 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 373 request.aggregation(aggregate); 374 } 375 // Add Aggregate post filter 376 QueryBuilder aggFilter = getAggregateFilter(); 377 if (aggFilter != null) { 378 request.postFilter(aggFilter); 379 } 380 381 // Add highlighting 382 if (highlightFields != null && !highlightFields.isEmpty()) { 383 HighlightBuilder hb = new HighlightBuilder(); 384 for (String field : highlightFields) { 385 hb.field(field); 386 } 387 hb.requireFieldMatch(false); 388 request.highlighter(hb); 389 } 390 // Fields selection 391 if (!isFetchFromElasticsearch()) { 392 request.fetchSource(getSelectFields(), null); 393 } 394 395 } 396 397 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 398 Principal principal = session.getPrincipal(); 399 if (principal == null 400 || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) { 401 return query; 402 } 403 String[] principals = SecurityService.getPrincipalsToCheck(principal); 404 // we want an ACL that match principals but we discard 405 // unsupported ACE that contains negative ACE 406 QueryBuilder aclFilter = QueryBuilders.boolQuery() 407 .must(QueryBuilders.termsQuery(ACL_FIELD, principals)) 408 .mustNot(QueryBuilders.termsQuery(ACL_FIELD, UNSUPPORTED_ACL)); 409 return QueryBuilders.boolQuery().must(query).filter(aclFilter); 410 } 411 412 /** 413 * Add a specific repository to search. Default search is done on the session repository only. 414 * 415 * @since 6.0 416 */ 417 public NxQueryBuilder addSearchRepository(String repositoryName) { 418 repositories.add(repositoryName); 419 return this; 420 } 421 422 /** 423 * Search on all available repositories. 424 * 425 * @since 6.0 426 */ 427 public NxQueryBuilder searchOnAllRepositories() { 428 searchOnAllRepo = true; 429 return this; 430 } 431 432 /** 433 * Return the list of repositories to search, or an empty list to search on all available repositories; 434 * 435 * @since 6.0 436 */ 437 public List<String> getSearchRepositories() { 438 if (searchOnAllRepo) { 439 return Collections.<String> emptyList(); 440 } 441 return repositories; 442 } 443 444 /** 445 * @since 6.0 446 */ 447 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 448 if (isFetchFromElasticsearch()) { 449 return new EsFetcher(session, response, repoNames, hitDocConsumer); 450 } 451 return new VcsFetcher(session, response, repoNames); 452 } 453 454 /** 455 * @since 7.2 456 */ 457 public String[] getSelectFields() { 458 return selectFields; 459 } 460 461 /** 462 * @since 7.2 463 */ 464 public Map<String, Type> getSelectFieldsAndTypes() { 465 return selectFieldsAndTypes; 466 } 467 468 /** 469 * @since 7.2 470 */ 471 public boolean returnsDocuments() { 472 if (esOnly) { 473 return false; 474 } 475 return returnsDocuments; 476 } 477 478 public boolean returnsRows() { 479 if (esOnly) { 480 return false; 481 } 482 return !returnsDocuments; 483 } 484}