001/* 002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019package org.nuxeo.elasticsearch.query; 020 021import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 022import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 023import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 024 025import java.security.Principal; 026import java.util.ArrayList; 027import java.util.Collections; 028import java.util.List; 029import java.util.Map; 030import java.util.Set; 031 032import org.elasticsearch.action.search.SearchRequestBuilder; 033import org.elasticsearch.action.search.SearchResponse; 034import org.elasticsearch.index.query.BoolQueryBuilder; 035import org.elasticsearch.index.query.QueryBuilder; 036import org.elasticsearch.index.query.QueryBuilders; 037import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 038import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 039import org.elasticsearch.search.sort.FieldSortBuilder; 040import org.elasticsearch.search.sort.SortBuilder; 041import org.elasticsearch.search.sort.SortOrder; 042import org.nuxeo.ecm.core.api.CoreSession; 043import org.nuxeo.ecm.core.api.NuxeoPrincipal; 044import org.nuxeo.ecm.core.api.SortInfo; 045import org.nuxeo.ecm.core.schema.SchemaManager; 046import org.nuxeo.ecm.core.schema.types.Type; 047import org.nuxeo.ecm.core.security.SecurityService; 048import org.nuxeo.ecm.platform.query.api.Aggregate; 049import org.nuxeo.ecm.platform.query.api.Bucket; 050import org.nuxeo.elasticsearch.ElasticSearchConstants; 051import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 052import org.nuxeo.elasticsearch.api.EsResult; 053import org.nuxeo.elasticsearch.fetcher.EsFetcher; 054import org.nuxeo.elasticsearch.fetcher.Fetcher; 055import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 056import org.nuxeo.runtime.api.Framework; 057 058/** 059 * Elasticsearch query builder for the Nuxeo ES api. 060 * 061 * @since 5.9.5 062 */ 063public class NxQueryBuilder { 064 065 private static final int DEFAULT_LIMIT = 10; 066 067 private int limit = DEFAULT_LIMIT; 068 069 private static final String AGG_FILTER_SUFFIX = "_filter"; 070 071 private final CoreSession session; 072 073 private final List<SortInfo> sortInfos = new ArrayList<>(); 074 075 private final List<String> repositories = new ArrayList<>(); 076 077 private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>(); 078 079 private int offset = 0; 080 081 private String nxql; 082 083 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 084 085 private boolean fetchFromElasticsearch = false; 086 087 private boolean searchOnAllRepo = false; 088 089 private String[] selectFields = { ElasticSearchConstants.ID_FIELD }; 090 091 private Map<String, Type> selectFieldsAndTypes; 092 093 private boolean returnsDocuments = true; 094 095 private boolean esOnly = false; 096 097 private List<String> highlightFields; 098 099 public NxQueryBuilder(CoreSession coreSession) { 100 session = coreSession; 101 repositories.add(coreSession.getRepositoryName()); 102 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 103 } 104 105 public static String getAggregateFilterId(Aggregate agg) { 106 return agg.getId() + AGG_FILTER_SUFFIX; 107 } 108 109 /** 110 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}. Since Nuxeo 8.4 and ES 2.x, we can 111 * not give -1 to this method as the default configuration on ES allows to have a search window of 10000 documents 112 * at maximum. This settings could be changed on ES by changing {index.max_result_window}, but it is preferable to 113 * use the scan & scroll API. 114 */ 115 public NxQueryBuilder limit(int limit) { 116 // For compatibility only, deprecated since 8.4 117 if (limit < 0) { 118 limit = Integer.MAX_VALUE; 119 } 120 this.limit = limit; 121 return this; 122 } 123 124 /** 125 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 126 * offset documents are skipped before starting to count the limit documents that are returned. 127 */ 128 public NxQueryBuilder offset(int offset) { 129 this.offset = offset; 130 return this; 131 } 132 133 public NxQueryBuilder addSort(SortInfo sortInfo) { 134 sortInfos.add(sortInfo); 135 return this; 136 } 137 138 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 139 if (sortInfos != null && sortInfos.length > 0) { 140 Collections.addAll(this.sortInfos, sortInfos); 141 } 142 return this; 143 } 144 145 /** 146 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 147 */ 148 public NxQueryBuilder nxql(String nxql) { 149 this.nxql = nxql; 150 this.esQueryBuilder = null; 151 return this; 152 } 153 154 /** 155 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 156 */ 157 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 158 esQueryBuilder = addSecurityFilter(queryBuilder); 159 nxql = null; 160 return this; 161 } 162 163 /** 164 * Ask for the Elasticsearch _source field, use it to build documents. 165 */ 166 public NxQueryBuilder fetchFromElasticsearch() { 167 fetchFromElasticsearch = true; 168 return this; 169 } 170 171 /** 172 * Fetch the documents using VCS (database) engine. This is done by default 173 */ 174 public NxQueryBuilder fetchFromDatabase() { 175 fetchFromElasticsearch = false; 176 return this; 177 } 178 179 /** 180 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 181 * {@link EsResult#getElasticsearchResponse()} 182 * 183 * @since 7.3 184 */ 185 public NxQueryBuilder onlyElasticsearchResponse() { 186 esOnly = true; 187 return this; 188 } 189 190 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) { 191 aggregates.add(aggregate); 192 return this; 193 } 194 195 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) { 196 if (aggregates != null && !aggregates.isEmpty()) { 197 this.aggregates.addAll(aggregates); 198 } 199 return this; 200 } 201 202 /** 203 * @since 9.1 204 */ 205 public NxQueryBuilder highlight(List<String> highlightFields) { 206 this.highlightFields = highlightFields; 207 return this; 208 } 209 210 public int getLimit() { 211 return limit; 212 } 213 214 public int getOffset() { 215 return offset; 216 } 217 218 public List<SortInfo> getSortInfos() { 219 return sortInfos; 220 } 221 222 public String getNxql() { 223 return nxql; 224 } 225 226 public boolean isFetchFromElasticsearch() { 227 return fetchFromElasticsearch; 228 } 229 230 public CoreSession getSession() { 231 return session; 232 } 233 234 /** 235 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 236 * aggregates, use the udpateRequest to get the full request. 237 */ 238 public QueryBuilder makeQuery() { 239 if (esQueryBuilder == null) { 240 if (nxql != null) { 241 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 242 // handle the built-in order by clause 243 if (nxql.toLowerCase().contains("order by")) { 244 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 245 sortInfos.addAll(builtInSortInfos); 246 } 247 if (nxqlHasSelectClause(nxql)) { 248 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 249 Set<String> keySet = selectFieldsAndTypes.keySet(); 250 selectFields = keySet.toArray(new String[keySet.size()]); 251 returnsDocuments = false; 252 } 253 esQueryBuilder = addSecurityFilter(esQueryBuilder); 254 } 255 } 256 return esQueryBuilder; 257 } 258 259 protected boolean nxqlHasSelectClause(String nxql) { 260 String lowerNxql = nxql.toLowerCase(); 261 return lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from"); 262 } 263 264 public SortBuilder[] getSortBuilders() { 265 SortBuilder[] ret; 266 if (sortInfos.isEmpty()) { 267 return new SortBuilder[0]; 268 } 269 ret = new SortBuilder[sortInfos.size()]; 270 int i = 0; 271 for (SortInfo sortInfo : sortInfos) { 272 String fieldType = guessFieldType(sortInfo.getSortColumn()); 273 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()).order( 274 sortInfo.getSortAscending() ? SortOrder.ASC : SortOrder.DESC) 275 .unmappedType(fieldType); 276 } 277 return ret; 278 } 279 280 protected String guessFieldType(String field) { 281 String fieldType; 282 try { 283 SchemaManager schemaManager = Framework.getService(SchemaManager.class); 284 fieldType = schemaManager.getField(field).getType().getName(); 285 } catch (NullPointerException e) { 286 // probably an internal field without schema 287 fieldType = "string"; 288 } 289 switch (fieldType) { 290 case "integer": 291 case "long": 292 case "boolean": 293 case "date": 294 case "string": 295 return fieldType; 296 } 297 return "string"; 298 } 299 300 protected QueryBuilder getAggregateFilter() { 301 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 302 for (AggregateEsBase agg : aggregates) { 303 QueryBuilder filter = agg.getEsFilter(); 304 if (filter != null) { 305 ret.must(filter); 306 } 307 } 308 if (!ret.hasClauses()) { 309 return null; 310 } 311 return ret; 312 } 313 314 protected QueryBuilder getAggregateFilterExceptFor(String id) { 315 BoolQueryBuilder ret = QueryBuilders.boolQuery(); 316 for (AggregateEsBase agg : aggregates) { 317 if (!agg.getId().equals(id)) { 318 QueryBuilder filter = agg.getEsFilter(); 319 if (filter != null) { 320 ret.must(filter); 321 } 322 } 323 } 324 if (!ret.hasClauses()) { 325 return QueryBuilders.matchAllQuery(); 326 } 327 return ret; 328 } 329 330 public List<AggregateEsBase<? extends Bucket>> getAggregates() { 331 return aggregates; 332 } 333 334 public List<FilterAggregationBuilder> getEsAggregates() { 335 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 336 for (AggregateEsBase agg : aggregates) { 337 FilterAggregationBuilder fagg = new FilterAggregationBuilder(getAggregateFilterId(agg)); 338 fagg.filter(getAggregateFilterExceptFor(agg.getId())); 339 fagg.subAggregation(agg.getEsAggregate()); 340 ret.add(fagg); 341 } 342 return ret; 343 } 344 345 public void updateRequest(SearchRequestBuilder request) { 346 // Set limits 347 request.setFrom(getOffset()).setSize(getLimit()); 348 // Build query with security checks 349 request.setQuery(makeQuery()); 350 // Add sort 351 for (SortBuilder sortBuilder : getSortBuilders()) { 352 request.addSort(sortBuilder); 353 } 354 // Add Aggregate 355 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 356 request.addAggregation(aggregate); 357 } 358 // Add Aggregate post filter 359 QueryBuilder aggFilter = getAggregateFilter(); 360 if (aggFilter != null) { 361 request.setPostFilter(aggFilter); 362 } 363 364 // Add highlighting 365 if (highlightFields != null && !highlightFields.isEmpty()) { 366 for (String field : highlightFields) { 367 request.addHighlightedField(field); 368 } 369 request.setHighlighterRequireFieldMatch(false); 370 } 371 372 // Fields selection 373 if (!isFetchFromElasticsearch()) { 374 request.addFields(getSelectFields()); 375 } 376 377 } 378 379 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 380 Principal principal = session.getPrincipal(); 381 if (principal == null 382 || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) { 383 return query; 384 } 385 String[] principals = SecurityService.getPrincipalsToCheck(principal); 386 // we want an ACL that match principals but we discard 387 // unsupported ACE that contains negative ACE 388 QueryBuilder aclFilter = QueryBuilders.boolQuery() 389 .must(QueryBuilders.termsQuery(ACL_FIELD, principals)) 390 .mustNot(QueryBuilders.termsQuery(ACL_FIELD, UNSUPPORTED_ACL)); 391 return QueryBuilders.boolQuery().must(query).filter(aclFilter); 392 } 393 394 /** 395 * Add a specific repository to search. Default search is done on the session repository only. 396 * 397 * @since 6.0 398 */ 399 public NxQueryBuilder addSearchRepository(String repositoryName) { 400 repositories.add(repositoryName); 401 return this; 402 } 403 404 /** 405 * Search on all available repositories. 406 * 407 * @since 6.0 408 */ 409 public NxQueryBuilder searchOnAllRepositories() { 410 searchOnAllRepo = true; 411 return this; 412 } 413 414 /** 415 * Return the list of repositories to search, or an empty list to search on all available repositories; 416 * 417 * @since 6.0 418 */ 419 public List<String> getSearchRepositories() { 420 if (searchOnAllRepo) { 421 return Collections.<String> emptyList(); 422 } 423 return repositories; 424 } 425 426 /** 427 * @since 6.0 428 */ 429 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 430 if (isFetchFromElasticsearch()) { 431 return new EsFetcher(session, response, repoNames); 432 } 433 return new VcsFetcher(session, response, repoNames); 434 } 435 436 /** 437 * @since 7.2 438 */ 439 public String[] getSelectFields() { 440 return selectFields; 441 } 442 443 /** 444 * @since 7.2 445 */ 446 public Map<String, Type> getSelectFieldsAndTypes() { 447 return selectFieldsAndTypes; 448 } 449 450 /** 451 * @since 7.2 452 */ 453 public boolean returnsDocuments() { 454 if (esOnly) { 455 return false; 456 } 457 return returnsDocuments; 458 } 459 460 public boolean returnsRows() { 461 if (esOnly) { 462 return false; 463 } 464 return !returnsDocuments; 465 } 466}