001/* 002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019package org.nuxeo.elasticsearch.query; 020 021import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 022import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 023import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 024 025import java.security.Principal; 026import java.util.ArrayList; 027import java.util.Collections; 028import java.util.List; 029import java.util.Map; 030import java.util.Set; 031 032import org.elasticsearch.action.search.SearchResponse; 033import org.elasticsearch.index.query.BoolQueryBuilder; 034import org.elasticsearch.index.query.QueryBuilder; 035import org.elasticsearch.index.query.QueryBuilders; 036import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 037import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 038import org.elasticsearch.search.builder.SearchSourceBuilder; 039import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; 040import org.elasticsearch.search.sort.FieldSortBuilder; 041import org.elasticsearch.search.sort.SortBuilder; 042import org.elasticsearch.search.sort.SortOrder; 043import org.nuxeo.ecm.core.api.CoreSession; 044import org.nuxeo.ecm.core.api.NuxeoPrincipal; 045import org.nuxeo.ecm.core.api.SortInfo; 046import org.nuxeo.ecm.core.schema.SchemaManager; 047import org.nuxeo.ecm.core.schema.types.Type; 048import org.nuxeo.ecm.core.security.SecurityService; 049import org.nuxeo.ecm.platform.query.api.Aggregate; 050import org.nuxeo.ecm.platform.query.api.Bucket; 051import org.nuxeo.elasticsearch.ElasticSearchConstants; 052import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 053import org.nuxeo.elasticsearch.api.EsResult; 054import org.nuxeo.elasticsearch.fetcher.EsFetcher; 055import org.nuxeo.elasticsearch.fetcher.Fetcher; 056import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 057import org.nuxeo.runtime.api.Framework; 058 059/** 060 * Elasticsearch query builder for the Nuxeo ES api. 061 * 062 * @since 5.9.5 063 */ 064public class NxQueryBuilder { 065 066 private static final int DEFAULT_LIMIT = 10; 067 068 private static final String AGG_FILTER_SUFFIX = "_filter"; 069 070 private final CoreSession session; 071 072 private final List<SortInfo> sortInfos = new ArrayList<>(); 073 074 private final List<String> repositories = new ArrayList<>(); 075 076 private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>(); 077 078 private int limit = DEFAULT_LIMIT; 079 080 private int offset = 0; 081 082 private String nxql; 083 084 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 085 086 private boolean fetchFromElasticsearch = false; 087 088 private boolean searchOnAllRepo = false; 089 090 private String[] selectFields = { ElasticSearchConstants.ID_FIELD }; 091 092 private Map<String, Type> selectFieldsAndTypes; 093 094 private boolean returnsDocuments = true; 095 096 private boolean esOnly = false; 097 098 private List<String> highlightFields; 099 100 public NxQueryBuilder(CoreSession coreSession) { 101 session = coreSession; 102 repositories.add(coreSession.getRepositoryName()); 103 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 104 } 105 106 public static String getAggregateFilterId(Aggregate agg) { 107 return agg.getId() + AGG_FILTER_SUFFIX; 108 } 109 110 /** 111 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}. Since Nuxeo 8.4 and ES 2.x, we can 112 * not give -1 to this method as the default configuration on ES allows to have a search window of 10000 documents 113 * at maximum. This settings could be changed on ES by changing {index.max_result_window}, but it is preferable to 114 * use the scan & scroll API. 115 */ 116 public NxQueryBuilder limit(int limit) { 117 // For compatibility only, deprecated since 8.4 118 if (limit < 0) { 119 limit = Integer.MAX_VALUE; 120 } 121 this.limit = limit; 122 return this; 123 } 124 125 /** 126 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 127 * offset documents are skipped before starting to count the limit documents that are returned. 128 */ 129 public NxQueryBuilder offset(int offset) { 130 this.offset = offset; 131 return this; 132 } 133 134 public NxQueryBuilder addSort(SortInfo sortInfo) { 135 sortInfos.add(sortInfo); 136 return this; 137 } 138 139 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 140 if (sortInfos != null && sortInfos.length > 0) { 141 Collections.addAll(this.sortInfos, sortInfos); 142 } 143 return this; 144 } 145 146 /** 147 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 148 */ 149 public NxQueryBuilder nxql(String nxql) { 150 this.nxql = nxql; 151 this.esQueryBuilder = null; 152 return this; 153 } 154 155 /** 156 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 157 */ 158 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 159 esQueryBuilder = addSecurityFilter(queryBuilder); 160 nxql = null; 161 return this; 162 } 163 164 /** 165 * Ask for the Elasticsearch _source field, use it to build documents. 166 */ 167 public NxQueryBuilder fetchFromElasticsearch() { 168 fetchFromElasticsearch = true; 169 return this; 170 } 171 172 /** 173 * Fetch the documents using VCS (database) engine. This is done by default 174 */ 175 public NxQueryBuilder fetchFromDatabase() { 176 fetchFromElasticsearch = false; 177 return this; 178 } 179 180 /** 181 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 182 * {@link EsResult#getElasticsearchResponse()} 183 * 184 * @since 7.3 185 */ 186 public NxQueryBuilder onlyElasticsearchResponse() { 187 esOnly = true; 188 return this; 189 } 190 191 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) { 192 aggregates.add(aggregate); 193 return this; 194 } 195 196 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) { 197 if (aggregates != null && !aggregates.isEmpty()) { 198 this.aggregates.addAll(aggregates); 199 } 200 return this; 201 } 202 203 /** 204 * @since 9.1 205 */ 206 public NxQueryBuilder highlight(List<String> highlightFields) { 207 this.highlightFields = highlightFields; 208 return this; 209 } 210 211 public int getLimit() { 212 return limit; 213 } 214 215 public int getOffset() { 216 return offset; 217 } 218 219 public List<SortInfo> getSortInfos() { 220 return sortInfos; 221 } 222 223 public String getNxql() { 224 return nxql; 225 } 226 227 public boolean isFetchFromElasticsearch() { 228 return fetchFromElasticsearch; 229 } 230 231 public CoreSession getSession() { 232 return session; 233 } 234 235 /** 236 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 237 * aggregates, use the udpateRequest to get the full request. 238 */ 239 public QueryBuilder makeQuery() { 240 if (esQueryBuilder == null) { 241 if (nxql != null) { 242 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 243 // handle the built-in order by clause 244 if (nxql.toLowerCase().contains("order by")) { 245 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 246 sortInfos.addAll(builtInSortInfos); 247 } 248 if (nxqlHasSelectClause(nxql)) { 249 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 250 Set<String> keySet = selectFieldsAndTypes.keySet(); 251 selectFields = keySet.toArray(new String[keySet.size()]); 252 returnsDocuments = false; 253 } 254 esQueryBuilder = addSecurityFilter(esQueryBuilder); 255 } 256 } 257 return esQueryBuilder; 258 } 259 260 protected boolean nxqlHasSelectClause(String nxql) { 261 String lowerNxql = nxql.toLowerCase(); 262 return lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from"); 263 } 264 265 public SortBuilder[] getSortBuilders() { 266 SortBuilder[] ret; 267 if (sortInfos.isEmpty()) { 268 return new SortBuilder[0]; 269 } 270 ret = new SortBuilder[sortInfos.size()]; 271 int i = 0; 272 for (SortInfo sortInfo : sortInfos) { 273 String fieldType = guessFieldType(sortInfo.getSortColumn()); 274 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()) 275 .order(sortInfo.getSortAscending() ? SortOrder.ASC 276 : SortOrder.DESC) 277 .unmappedType(fieldType); 278 } 279 return ret; 280 } 281 282 protected String guessFieldType(String field) { 283 String fieldType; 284 try { 285 SchemaManager schemaManager = Framework.getService(SchemaManager.class); 286 fieldType = schemaManager.getField(field).getType().getName(); 287 } catch (NullPointerException e) { 288 // probably an internal field without schema 289 fieldType = "string"; 290 } 291 switch (fieldType) { 292 case "integer": 293 case "long": 294 case "boolean": 295 case "date": 296 case "string": 297 return fieldType; 298 } 299 return "string"; 300 } 301 302 protected QueryBuilder getAggregateFilter() { 303 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 304 for (AggregateEsBase agg : aggregates) { 305 QueryBuilder filter = agg.getEsFilter(); 306 if (filter != null) { 307 ret.must(filter); 308 } 309 } 310 if (!ret.hasClauses()) { 311 return null; 312 } 313 return ret; 314 } 315 316 protected QueryBuilder getAggregateFilterExceptFor(String id) { 317 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 318 for (AggregateEsBase agg : aggregates) { 319 if (!agg.getId().equals(id)) { 320 QueryBuilder filter = agg.getEsFilter(); 321 if (filter != null) { 322 ret.must(filter); 323 } 324 } 325 } 326 if (!ret.hasClauses()) { 327 return QueryBuilders.matchAllQuery(); 328 } 329 return ret; 330 } 331 332 public List<AggregateEsBase<? extends Bucket>> getAggregates() { 333 return aggregates; 334 } 335 336 public List<FilterAggregationBuilder> getEsAggregates() { 337 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 338 for (AggregateEsBase agg : aggregates) { 339 FilterAggregationBuilder fagg = null; 340 fagg = new FilterAggregationBuilder(getAggregateFilterId(agg), getAggregateFilterExceptFor(agg.getId())); 341 fagg.subAggregation(agg.getEsAggregate()); 342 ret.add(fagg); 343 } 344 return ret; 345 } 346 347 public void updateRequest(SearchSourceBuilder request) { 348 // Set limits 349 request.from(getOffset()).size(getLimit()); 350 // Build query with security checks 351 request.query(makeQuery()); 352 // Add sort 353 for (SortBuilder sortBuilder : getSortBuilders()) { 354 request.sort(sortBuilder); 355 } 356 // Add Aggregate 357 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 358 request.aggregation(aggregate); 359 } 360 // Add Aggregate post filter 361 QueryBuilder aggFilter = getAggregateFilter(); 362 if (aggFilter != null) { 363 request.postFilter(aggFilter); 364 } 365 366 // Add highlighting 367 if (highlightFields != null && !highlightFields.isEmpty()) { 368 HighlightBuilder hb = new HighlightBuilder(); 369 for (String field : highlightFields) { 370 hb.field(field); 371 } 372 hb.requireFieldMatch(false); 373 request.highlighter(hb); 374 } 375 // Fields selection 376 if (!isFetchFromElasticsearch()) { 377 request.fetchSource(getSelectFields(), null); 378 } 379 380 } 381 382 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 383 Principal principal = session.getPrincipal(); 384 if (principal == null 385 || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) { 386 return query; 387 } 388 String[] principals = SecurityService.getPrincipalsToCheck(principal); 389 // we want an ACL that match principals but we discard 390 // unsupported ACE that contains negative ACE 391 QueryBuilder aclFilter = QueryBuilders.boolQuery() 392 .must(QueryBuilders.termsQuery(ACL_FIELD, principals)) 393 .mustNot(QueryBuilders.termsQuery(ACL_FIELD, UNSUPPORTED_ACL)); 394 return QueryBuilders.boolQuery().must(query).filter(aclFilter); 395 } 396 397 /** 398 * Add a specific repository to search. Default search is done on the session repository only. 399 * 400 * @since 6.0 401 */ 402 public NxQueryBuilder addSearchRepository(String repositoryName) { 403 repositories.add(repositoryName); 404 return this; 405 } 406 407 /** 408 * Search on all available repositories. 409 * 410 * @since 6.0 411 */ 412 public NxQueryBuilder searchOnAllRepositories() { 413 searchOnAllRepo = true; 414 return this; 415 } 416 417 /** 418 * Return the list of repositories to search, or an empty list to search on all available repositories; 419 * 420 * @since 6.0 421 */ 422 public List<String> getSearchRepositories() { 423 if (searchOnAllRepo) { 424 return Collections.<String> emptyList(); 425 } 426 return repositories; 427 } 428 429 /** 430 * @since 6.0 431 */ 432 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 433 if (isFetchFromElasticsearch()) { 434 return new EsFetcher(session, response, repoNames); 435 } 436 return new VcsFetcher(session, response, repoNames); 437 } 438 439 /** 440 * @since 7.2 441 */ 442 public String[] getSelectFields() { 443 return selectFields; 444 } 445 446 /** 447 * @since 7.2 448 */ 449 public Map<String, Type> getSelectFieldsAndTypes() { 450 return selectFieldsAndTypes; 451 } 452 453 /** 454 * @since 7.2 455 */ 456 public boolean returnsDocuments() { 457 if (esOnly) { 458 return false; 459 } 460 return returnsDocuments; 461 } 462 463 public boolean returnsRows() { 464 if (esOnly) { 465 return false; 466 } 467 return !returnsDocuments; 468 } 469}