001/* 002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019package org.nuxeo.elasticsearch.query; 020 021import org.elasticsearch.action.search.SearchRequestBuilder; 022import org.elasticsearch.action.search.SearchResponse; 023import org.elasticsearch.index.query.AndFilterBuilder; 024import org.elasticsearch.index.query.FilterBuilder; 025import org.elasticsearch.index.query.FilterBuilders; 026import org.elasticsearch.index.query.QueryBuilder; 027import org.elasticsearch.index.query.QueryBuilders; 028import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 029import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 030import org.elasticsearch.search.sort.FieldSortBuilder; 031import org.elasticsearch.search.sort.SortBuilder; 032import org.elasticsearch.search.sort.SortOrder; 033import org.nuxeo.ecm.core.api.CoreSession; 034import org.nuxeo.ecm.core.api.NuxeoPrincipal; 035import org.nuxeo.ecm.core.api.SortInfo; 036import org.nuxeo.ecm.core.schema.types.Type; 037import org.nuxeo.ecm.core.security.SecurityService; 038import org.nuxeo.ecm.platform.query.api.Aggregate; 039import org.nuxeo.ecm.platform.query.api.Bucket; 040import org.nuxeo.elasticsearch.ElasticSearchConstants; 041import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 042import org.nuxeo.elasticsearch.api.EsResult; 043import org.nuxeo.elasticsearch.fetcher.EsFetcher; 044import org.nuxeo.elasticsearch.fetcher.Fetcher; 045import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 046import org.nuxeo.runtime.api.Framework; 047 048import java.security.Principal; 049import java.util.ArrayList; 050import java.util.Collections; 051import java.util.List; 052import java.util.Map; 053import java.util.Set; 054 055import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 056import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 057import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 058 059/** 060 * Elasticsearch query buidler for the Nuxeo ES api. 061 * 062 * @since 5.9.5 063 */ 064public class NxQueryBuilder { 065 066 private static final int DEFAULT_LIMIT = 10; 067 068 private int limit = DEFAULT_LIMIT; 069 070 private static final String AGG_FILTER_SUFFIX = "_filter"; 071 072 private final CoreSession session; 073 074 private final List<SortInfo> sortInfos = new ArrayList<>(); 075 076 private final List<String> repositories = new ArrayList<>(); 077 078 private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>(); 079 080 private int offset = 0; 081 082 private String nxql; 083 084 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 085 086 private boolean fetchFromElasticsearch = false; 087 088 private boolean searchOnAllRepo = false; 089 090 private String[] selectFields = {ElasticSearchConstants.ID_FIELD}; 091 092 private Map<String, Type> selectFieldsAndTypes; 093 094 private boolean returnsDocuments = true; 095 096 private boolean esOnly = false; 097 098 public NxQueryBuilder(CoreSession coreSession) { 099 session = coreSession; 100 repositories.add(coreSession.getRepositoryName()); 101 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 102 } 103 104 public static String getAggregateFilterId(Aggregate agg) { 105 return agg.getId() + AGG_FILTER_SUFFIX; 106 } 107 108 /** 109 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}, Use -1 to return all documents. 110 */ 111 public NxQueryBuilder limit(int limit) { 112 if (limit < 0) { 113 limit = Integer.MAX_VALUE; 114 } 115 this.limit = limit; 116 return this; 117 } 118 119 /** 120 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 121 * offset documents are skipped before starting to count the limit documents that are returned. 122 */ 123 public NxQueryBuilder offset(int offset) { 124 this.offset = offset; 125 return this; 126 } 127 128 public NxQueryBuilder addSort(SortInfo sortInfo) { 129 sortInfos.add(sortInfo); 130 return this; 131 } 132 133 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 134 if (sortInfos != null && sortInfos.length > 0) { 135 Collections.addAll(this.sortInfos, sortInfos); 136 } 137 return this; 138 } 139 140 /** 141 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 142 */ 143 public NxQueryBuilder nxql(String nxql) { 144 this.nxql = nxql; 145 this.esQueryBuilder = null; 146 return this; 147 } 148 149 /** 150 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 151 */ 152 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 153 this.esQueryBuilder = queryBuilder; 154 nxql = null; 155 return this; 156 } 157 158 /** 159 * Ask for the Elasticsearch _source field, use it to build documents. 160 */ 161 public NxQueryBuilder fetchFromElasticsearch() { 162 fetchFromElasticsearch = true; 163 return this; 164 } 165 166 /** 167 * Fetch the documents using VCS (database) engine. This is done by default 168 */ 169 public NxQueryBuilder fetchFromDatabase() { 170 fetchFromElasticsearch = false; 171 return this; 172 } 173 174 /** 175 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 176 * {@link EsResult#getElasticsearchResponse()} 177 * 178 * @since 7.3 179 */ 180 public NxQueryBuilder onlyElasticsearchResponse() { 181 esOnly = true; 182 return this; 183 } 184 185 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) { 186 aggregates.add(aggregate); 187 return this; 188 } 189 190 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) { 191 if (aggregates != null && !aggregates.isEmpty()) { 192 this.aggregates.addAll(aggregates); 193 } 194 return this; 195 } 196 197 public int getLimit() { 198 return limit; 199 } 200 201 public int getOffset() { 202 return offset; 203 } 204 205 public List<SortInfo> getSortInfos() { 206 return sortInfos; 207 } 208 209 public String getNxql() { 210 return nxql; 211 } 212 213 public boolean isFetchFromElasticsearch() { 214 return fetchFromElasticsearch; 215 } 216 217 public CoreSession getSession() { 218 return session; 219 } 220 221 /** 222 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 223 * aggregates, use the udpateRequest to get the full request. 224 */ 225 public QueryBuilder makeQuery() { 226 if (esQueryBuilder == null) { 227 if (nxql != null) { 228 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 229 // handle the built-in order by clause 230 if (nxql.toLowerCase().contains("order by")) { 231 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 232 sortInfos.addAll(builtInSortInfos); 233 } 234 if (nxqlHasSelectClause(nxql)) { 235 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 236 Set<String> keySet = selectFieldsAndTypes.keySet(); 237 selectFields = keySet.toArray(new String[keySet.size()]); 238 returnsDocuments = false; 239 } 240 esQueryBuilder = addSecurityFilter(esQueryBuilder); 241 } 242 } 243 return esQueryBuilder; 244 } 245 246 protected boolean nxqlHasSelectClause(String nxql) { 247 String lowerNxql = nxql.toLowerCase(); 248 if (lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from")) { 249 return true; 250 } 251 return false; 252 } 253 254 public SortBuilder[] getSortBuilders() { 255 SortBuilder[] ret; 256 if (sortInfos.isEmpty()) { 257 return new SortBuilder[0]; 258 } 259 ret = new SortBuilder[sortInfos.size()]; 260 int i = 0; 261 for (SortInfo sortInfo : sortInfos) { 262 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()).order(sortInfo.getSortAscending() ? SortOrder.ASC 263 : SortOrder.DESC); 264 } 265 return ret; 266 } 267 268 protected FilterBuilder getAggregateFilter() { 269 boolean hasFilter = false; 270 AndFilterBuilder ret = FilterBuilders.andFilter(); 271 for (AggregateEsBase agg : aggregates) { 272 FilterBuilder filter = agg.getEsFilter(); 273 if (filter != null) { 274 ret.add(filter); 275 hasFilter = true; 276 } 277 } 278 if (!hasFilter) { 279 return null; 280 } 281 return ret; 282 } 283 284 protected FilterBuilder getAggregateFilterExceptFor(String id) { 285 boolean hasFilter = false; 286 AndFilterBuilder ret = FilterBuilders.andFilter(); 287 for (AggregateEsBase agg : aggregates) { 288 if (!agg.getId().equals(id)) { 289 FilterBuilder filter = agg.getEsFilter(); 290 if (filter != null) { 291 ret.add(filter); 292 hasFilter = true; 293 } 294 } 295 } 296 if (!hasFilter) { 297 return FilterBuilders.matchAllFilter(); 298 } 299 return ret; 300 } 301 302 public List<AggregateEsBase<? extends Bucket>> getAggregates() { 303 return aggregates; 304 } 305 306 public List<FilterAggregationBuilder> getEsAggregates() { 307 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 308 for (AggregateEsBase agg : aggregates) { 309 FilterAggregationBuilder fagg = new FilterAggregationBuilder(getAggregateFilterId(agg)); 310 fagg.filter(getAggregateFilterExceptFor(agg.getId())); 311 fagg.subAggregation(agg.getEsAggregate()); 312 ret.add(fagg); 313 } 314 return ret; 315 } 316 317 public void updateRequest(SearchRequestBuilder request) { 318 // Set limits 319 request.setFrom(getOffset()).setSize(getLimit()); 320 // Build query with security checks 321 request.setQuery(makeQuery()); 322 // Add sort 323 for (SortBuilder sortBuilder : getSortBuilders()) { 324 request.addSort(sortBuilder); 325 } 326 // Add Aggregate 327 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 328 request.addAggregation(aggregate); 329 } 330 // Add Aggregate post filter 331 FilterBuilder aggFilter = getAggregateFilter(); 332 if (aggFilter != null) { 333 request.setPostFilter(aggFilter); 334 } 335 // Fields selection 336 if (!isFetchFromElasticsearch()) { 337 request.addFields(getSelectFields()); 338 } 339 340 } 341 342 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 343 AndFilterBuilder aclFilter; 344 Principal principal = session.getPrincipal(); 345 if (principal == null 346 || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) { 347 return query; 348 } 349 String[] principals = SecurityService.getPrincipalsToCheck(principal); 350 // we want an ACL that match principals but we discard 351 // unsupported ACE that contains negative ACE 352 aclFilter = FilterBuilders.andFilter(FilterBuilders.inFilter(ACL_FIELD, principals), 353 FilterBuilders.notFilter(FilterBuilders.inFilter(ACL_FIELD, UNSUPPORTED_ACL))); 354 return QueryBuilders.filteredQuery(query, aclFilter); 355 } 356 357 358 /** 359 * Add a specific repository to search. Default search is done on the session repository only. 360 * 361 * @since 6.0 362 */ 363 public NxQueryBuilder addSearchRepository(String repositoryName) { 364 repositories.add(repositoryName); 365 return this; 366 } 367 368 /** 369 * Search on all available repositories. 370 * 371 * @since 6.0 372 */ 373 public NxQueryBuilder searchOnAllRepositories() { 374 searchOnAllRepo = true; 375 return this; 376 } 377 378 /** 379 * Return the list of repositories to search, or an empty list to search on all available repositories; 380 * 381 * @since 6.0 382 */ 383 public List<String> getSearchRepositories() { 384 if (searchOnAllRepo) { 385 return Collections.<String>emptyList(); 386 } 387 return repositories; 388 } 389 390 /** 391 * @since 6.0 392 */ 393 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 394 if (isFetchFromElasticsearch()) { 395 return new EsFetcher(session, response, repoNames); 396 } 397 return new VcsFetcher(session, response, repoNames); 398 } 399 400 /** 401 * @since 7.2 402 */ 403 public String[] getSelectFields() { 404 return selectFields; 405 } 406 407 /** 408 * @since 7.2 409 */ 410 public Map<String, Type> getSelectFieldsAndTypes() { 411 return selectFieldsAndTypes; 412 } 413 414 /** 415 * @since 7.2 416 */ 417 public boolean returnsDocuments() { 418 if (esOnly) { 419 return false; 420 } 421 return returnsDocuments; 422 } 423 424 public boolean returnsRows() { 425 if (esOnly) { 426 return false; 427 } 428 return !returnsDocuments; 429 } 430}