001/* 002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and contributors. 003 * 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the GNU Lesser General Public License 006 * (LGPL) version 2.1 which accompanies this distribution, and is available at 007 * http://www.gnu.org/licenses/lgpl-2.1.html 008 * 009 * This library is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 012 * Lesser General Public License for more details. 013 * 014 * Contributors: 015 * bdelbosc 016 */ 017package org.nuxeo.elasticsearch.query; 018 019import org.elasticsearch.action.search.SearchRequestBuilder; 020import org.elasticsearch.action.search.SearchResponse; 021import org.elasticsearch.index.query.AndFilterBuilder; 022import org.elasticsearch.index.query.FilterBuilder; 023import org.elasticsearch.index.query.FilterBuilders; 024import org.elasticsearch.index.query.QueryBuilder; 025import org.elasticsearch.index.query.QueryBuilders; 026import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; 027import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder; 028import org.elasticsearch.search.sort.FieldSortBuilder; 029import org.elasticsearch.search.sort.SortBuilder; 030import org.elasticsearch.search.sort.SortOrder; 031import org.nuxeo.ecm.core.api.CoreSession; 032import org.nuxeo.ecm.core.api.NuxeoPrincipal; 033import org.nuxeo.ecm.core.api.SortInfo; 034import org.nuxeo.ecm.core.schema.types.Type; 035import org.nuxeo.ecm.core.security.SecurityService; 036import org.nuxeo.ecm.platform.query.api.Aggregate; 037import org.nuxeo.ecm.platform.query.api.Bucket; 038import org.nuxeo.elasticsearch.ElasticSearchConstants; 039import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 040import org.nuxeo.elasticsearch.api.EsResult; 041import org.nuxeo.elasticsearch.fetcher.EsFetcher; 042import org.nuxeo.elasticsearch.fetcher.Fetcher; 043import org.nuxeo.elasticsearch.fetcher.VcsFetcher; 044import org.nuxeo.runtime.api.Framework; 045 046import java.security.Principal; 047import java.util.ArrayList; 048import java.util.Collections; 049import java.util.List; 050import java.util.Map; 051import java.util.Set; 052 053import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 054import static org.nuxeo.elasticsearch.ElasticSearchConstants.ACL_FIELD; 055import static org.nuxeo.elasticsearch.ElasticSearchConstants.FETCH_DOC_FROM_ES_PROPERTY; 056 057/** 058 * Elasticsearch query buidler for the Nuxeo ES api. 059 * 060 * @since 5.9.5 061 */ 062public class NxQueryBuilder { 063 064 private static final int DEFAULT_LIMIT = 10; 065 066 private int limit = DEFAULT_LIMIT; 067 068 private static final String AGG_FILTER_SUFFIX = "_filter"; 069 070 private final CoreSession session; 071 072 private final List<SortInfo> sortInfos = new ArrayList<>(); 073 074 private final List<String> repositories = new ArrayList<>(); 075 076 private final List<AggregateEsBase<? extends Bucket>> aggregates = new ArrayList<>(); 077 078 private int offset = 0; 079 080 private String nxql; 081 082 private org.elasticsearch.index.query.QueryBuilder esQueryBuilder; 083 084 private boolean fetchFromElasticsearch = false; 085 086 private boolean searchOnAllRepo = false; 087 088 private String[] selectFields = {ElasticSearchConstants.ID_FIELD}; 089 090 private Map<String, Type> selectFieldsAndTypes; 091 092 private boolean returnsDocuments = true; 093 094 private boolean esOnly = false; 095 096 public NxQueryBuilder(CoreSession coreSession) { 097 session = coreSession; 098 repositories.add(coreSession.getRepositoryName()); 099 fetchFromElasticsearch = Boolean.parseBoolean(Framework.getProperty(FETCH_DOC_FROM_ES_PROPERTY, "false")); 100 } 101 102 public static String getAggregateFilterId(Aggregate agg) { 103 return agg.getId() + AGG_FILTER_SUFFIX; 104 } 105 106 /** 107 * No more than that many documents will be returned. Default to {DEFAULT_LIMIT}, Use -1 to return all documents. 108 */ 109 public NxQueryBuilder limit(int limit) { 110 if (limit < 0) { 111 limit = Integer.MAX_VALUE; 112 } 113 this.limit = limit; 114 return this; 115 } 116 117 /** 118 * Says to skip that many documents before beginning to return documents. If both offset and limit appear, then 119 * offset documents are skipped before starting to count the limit documents that are returned. 120 */ 121 public NxQueryBuilder offset(int offset) { 122 this.offset = offset; 123 return this; 124 } 125 126 public NxQueryBuilder addSort(SortInfo sortInfo) { 127 sortInfos.add(sortInfo); 128 return this; 129 } 130 131 public NxQueryBuilder addSort(SortInfo[] sortInfos) { 132 if (sortInfos != null && sortInfos.length > 0) { 133 Collections.addAll(this.sortInfos, sortInfos); 134 } 135 return this; 136 } 137 138 /** 139 * Build the query from a NXQL string. You should either use nxql, either esQuery, not both. 140 */ 141 public NxQueryBuilder nxql(String nxql) { 142 this.nxql = nxql; 143 this.esQueryBuilder = null; 144 return this; 145 } 146 147 /** 148 * Build the query using the Elasticsearch QueryBuilder API. You should either use nxql, either esQuery, not both. 149 */ 150 public NxQueryBuilder esQuery(QueryBuilder queryBuilder) { 151 this.esQueryBuilder = queryBuilder; 152 nxql = null; 153 return this; 154 } 155 156 /** 157 * Ask for the Elasticsearch _source field, use it to build documents. 158 */ 159 public NxQueryBuilder fetchFromElasticsearch() { 160 fetchFromElasticsearch = true; 161 return this; 162 } 163 164 /** 165 * Fetch the documents using VCS (database) engine. This is done by default 166 */ 167 public NxQueryBuilder fetchFromDatabase() { 168 fetchFromElasticsearch = false; 169 return this; 170 } 171 172 /** 173 * Don't return document model list, aggregates or rows, only the original Elasticsearch response is accessible from 174 * {@link EsResult#getElasticsearchResponse()} 175 * 176 * @since 7.3 177 */ 178 public NxQueryBuilder onlyElasticsearchResponse() { 179 esOnly = true; 180 return this; 181 } 182 183 public NxQueryBuilder addAggregate(AggregateEsBase<? extends Bucket> aggregate) { 184 aggregates.add(aggregate); 185 return this; 186 } 187 188 public NxQueryBuilder addAggregates(List<AggregateEsBase<? extends Bucket>> aggregates) { 189 if (aggregates != null && !aggregates.isEmpty()) { 190 this.aggregates.addAll(aggregates); 191 } 192 return this; 193 } 194 195 public int getLimit() { 196 return limit; 197 } 198 199 public int getOffset() { 200 return offset; 201 } 202 203 public List<SortInfo> getSortInfos() { 204 return sortInfos; 205 } 206 207 public String getNxql() { 208 return nxql; 209 } 210 211 public boolean isFetchFromElasticsearch() { 212 return fetchFromElasticsearch; 213 } 214 215 public CoreSession getSession() { 216 return session; 217 } 218 219 /** 220 * Get the Elasticsearch queryBuilder. Note that it returns only the query part without order, limits nor 221 * aggregates, use the udpateRequest to get the full request. 222 */ 223 public QueryBuilder makeQuery() { 224 if (esQueryBuilder == null) { 225 if (nxql != null) { 226 esQueryBuilder = NxqlQueryConverter.toESQueryBuilder(nxql, session); 227 // handle the built-in order by clause 228 if (nxql.toLowerCase().contains("order by")) { 229 List<SortInfo> builtInSortInfos = NxqlQueryConverter.getSortInfo(nxql); 230 sortInfos.addAll(builtInSortInfos); 231 } 232 if (nxqlHasSelectClause(nxql)) { 233 selectFieldsAndTypes = NxqlQueryConverter.getSelectClauseFields(nxql); 234 Set<String> keySet = selectFieldsAndTypes.keySet(); 235 selectFields = keySet.toArray(new String[keySet.size()]); 236 returnsDocuments = false; 237 } 238 esQueryBuilder = addSecurityFilter(esQueryBuilder); 239 } 240 } 241 return esQueryBuilder; 242 } 243 244 protected boolean nxqlHasSelectClause(String nxql) { 245 String lowerNxql = nxql.toLowerCase(); 246 if (lowerNxql.startsWith("select") && !lowerNxql.startsWith("select * from")) { 247 return true; 248 } 249 return false; 250 } 251 252 public SortBuilder[] getSortBuilders() { 253 SortBuilder[] ret; 254 if (sortInfos.isEmpty()) { 255 return new SortBuilder[0]; 256 } 257 ret = new SortBuilder[sortInfos.size()]; 258 int i = 0; 259 for (SortInfo sortInfo : sortInfos) { 260 ret[i++] = new FieldSortBuilder(sortInfo.getSortColumn()).order(sortInfo.getSortAscending() ? SortOrder.ASC 261 : SortOrder.DESC); 262 } 263 return ret; 264 } 265 266 protected FilterBuilder getAggregateFilter() { 267 boolean hasFilter = false; 268 AndFilterBuilder ret = FilterBuilders.andFilter(); 269 for (AggregateEsBase agg : aggregates) { 270 FilterBuilder filter = agg.getEsFilter(); 271 if (filter != null) { 272 ret.add(filter); 273 hasFilter = true; 274 } 275 } 276 if (!hasFilter) { 277 return null; 278 } 279 return ret; 280 } 281 282 protected FilterBuilder getAggregateFilterExceptFor(String id) { 283 boolean hasFilter = false; 284 AndFilterBuilder ret = FilterBuilders.andFilter(); 285 for (AggregateEsBase agg : aggregates) { 286 if (!agg.getId().equals(id)) { 287 FilterBuilder filter = agg.getEsFilter(); 288 if (filter != null) { 289 ret.add(filter); 290 hasFilter = true; 291 } 292 } 293 } 294 if (!hasFilter) { 295 return FilterBuilders.matchAllFilter(); 296 } 297 return ret; 298 } 299 300 public List<AggregateEsBase<? extends Bucket>> getAggregates() { 301 return aggregates; 302 } 303 304 public List<FilterAggregationBuilder> getEsAggregates() { 305 List<FilterAggregationBuilder> ret = new ArrayList<>(aggregates.size()); 306 for (AggregateEsBase agg : aggregates) { 307 FilterAggregationBuilder fagg = new FilterAggregationBuilder(getAggregateFilterId(agg)); 308 fagg.filter(getAggregateFilterExceptFor(agg.getId())); 309 fagg.subAggregation(agg.getEsAggregate()); 310 ret.add(fagg); 311 } 312 return ret; 313 } 314 315 public void updateRequest(SearchRequestBuilder request) { 316 // Set limits 317 request.setFrom(getOffset()).setSize(getLimit()); 318 // Build query with security checks 319 request.setQuery(makeQuery()); 320 // Add sort 321 for (SortBuilder sortBuilder : getSortBuilders()) { 322 request.addSort(sortBuilder); 323 } 324 // Add Aggregate 325 for (AbstractAggregationBuilder aggregate : getEsAggregates()) { 326 request.addAggregation(aggregate); 327 } 328 // Add Aggregate post filter 329 FilterBuilder aggFilter = getAggregateFilter(); 330 if (aggFilter != null) { 331 request.setPostFilter(aggFilter); 332 } 333 // Fields selection 334 if (!isFetchFromElasticsearch()) { 335 request.addFields(getSelectFields()); 336 } 337 338 } 339 340 protected QueryBuilder addSecurityFilter(QueryBuilder query) { 341 AndFilterBuilder aclFilter; 342 Principal principal = session.getPrincipal(); 343 if (principal == null 344 || (principal instanceof NuxeoPrincipal && ((NuxeoPrincipal) principal).isAdministrator())) { 345 return query; 346 } 347 String[] principals = SecurityService.getPrincipalsToCheck(principal); 348 // we want an ACL that match principals but we discard 349 // unsupported ACE that contains negative ACE 350 aclFilter = FilterBuilders.andFilter(FilterBuilders.inFilter(ACL_FIELD, principals), 351 FilterBuilders.notFilter(FilterBuilders.inFilter(ACL_FIELD, UNSUPPORTED_ACL))); 352 return QueryBuilders.filteredQuery(query, aclFilter); 353 } 354 355 356 /** 357 * Add a specific repository to search. Default search is done on the session repository only. 358 * 359 * @since 6.0 360 */ 361 public NxQueryBuilder addSearchRepository(String repositoryName) { 362 repositories.add(repositoryName); 363 return this; 364 } 365 366 /** 367 * Search on all available repositories. 368 * 369 * @since 6.0 370 */ 371 public NxQueryBuilder searchOnAllRepositories() { 372 searchOnAllRepo = true; 373 return this; 374 } 375 376 /** 377 * Return the list of repositories to search, or an empty list to search on all available repositories; 378 * 379 * @since 6.0 380 */ 381 public List<String> getSearchRepositories() { 382 if (searchOnAllRepo) { 383 return Collections.<String>emptyList(); 384 } 385 return repositories; 386 } 387 388 /** 389 * @since 6.0 390 */ 391 public Fetcher getFetcher(SearchResponse response, Map<String, String> repoNames) { 392 if (isFetchFromElasticsearch()) { 393 return new EsFetcher(session, response, repoNames); 394 } 395 return new VcsFetcher(session, response, repoNames); 396 } 397 398 /** 399 * @since 7.2 400 */ 401 public String[] getSelectFields() { 402 return selectFields; 403 } 404 405 /** 406 * @since 7.2 407 */ 408 public Map<String, Type> getSelectFieldsAndTypes() { 409 return selectFieldsAndTypes; 410 } 411 412 /** 413 * @since 7.2 414 */ 415 public boolean returnsDocuments() { 416 if (esOnly) { 417 return false; 418 } 419 return returnsDocuments; 420 } 421 422 public boolean returnsRows() { 423 if (esOnly) { 424 return false; 425 } 426 return !returnsDocuments; 427 } 428}