001/* 002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019 020package org.nuxeo.elasticsearch.provider; 021 022import java.io.Serializable; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027 028import org.apache.logging.log4j.LogManager; 029import org.apache.logging.log4j.Logger; 030import org.elasticsearch.index.query.QueryBuilder; 031import org.elasticsearch.search.aggregations.Aggregation; 032import org.nuxeo.ecm.core.api.CoreSession; 033import org.nuxeo.ecm.core.api.DocumentModel; 034import org.nuxeo.ecm.core.api.DocumentModelList; 035import org.nuxeo.ecm.core.api.NuxeoException; 036import org.nuxeo.ecm.core.query.QueryParseException; 037import org.nuxeo.ecm.platform.query.api.Aggregate; 038import org.nuxeo.ecm.platform.query.api.AggregateDefinition; 039import org.nuxeo.ecm.platform.query.api.Bucket; 040import org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider; 041import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 042import org.nuxeo.elasticsearch.aggregate.AggregateFactory; 043import org.nuxeo.elasticsearch.api.ElasticSearchService; 044import org.nuxeo.elasticsearch.api.EsResult; 045import org.nuxeo.elasticsearch.query.NxQueryBuilder; 046import org.nuxeo.elasticsearch.query.NxqlQueryConverter; 047import org.nuxeo.runtime.api.Framework; 048import org.nuxeo.runtime.services.config.ConfigurationService; 049 050/** 051 * Elasticsearch Page provider that converts the NXQL query build by CoreQueryDocumentPageProvider. 052 * 053 * @since 5.9.3 054 */ 055public class ElasticSearchNxqlPageProvider extends CoreQueryDocumentPageProvider { 056 057 public static final String CORE_SESSION_PROPERTY = "coreSession"; 058 059 public static final String SEARCH_ON_ALL_REPOSITORIES_PROPERTY = "searchAllRepositories"; 060 061 // @since 9.2 062 public static final String ES_MAX_RESULT_WINDOW_PROPERTY = "org.nuxeo.elasticsearch.provider.maxResultWindow"; 063 064 // This is the default ES index.max_result_window 065 public static final long DEFAULT_ES_MAX_RESULT_WINDOW_VALUE = 10000; 066 067 protected static final Logger log = LogManager.getLogger(ElasticSearchNxqlPageProvider.class); 068 069 private static final long serialVersionUID = 1L; 070 071 protected HashMap<String, Aggregate<? extends Bucket>> currentAggregates; 072 073 protected Long maxResultWindow; 074 075 @Override 076 public List<DocumentModel> getCurrentPage() { 077 078 long t0 = System.currentTimeMillis(); 079 080 // use a cache 081 if (currentPageDocuments != null) { 082 return currentPageDocuments; 083 } 084 error = null; 085 errorMessage = null; 086 log.debug("Perform query for provider '{}': with pageSize={}, offset={}", this::getName, 087 this::getMinMaxPageSize, this::getCurrentPageOffset); 088 currentPageDocuments = new ArrayList<>(); 089 CoreSession coreSession = getCoreSession(); 090 if (query == null) { 091 buildQuery(coreSession); 092 } 093 if (query == null) { 094 throw new NuxeoException(String.format("Cannot perform null query: check provider '%s'", getName())); 095 } 096 // Build and execute the ES query 097 ElasticSearchService ess = Framework.getService(ElasticSearchService.class); 098 try { 099 NxQueryBuilder nxQuery = new NxQueryBuilder(getCoreSession()).nxql(query) 100 .offset((int) getCurrentPageOffset()) 101 .limit(getLimit()) 102 .addAggregates(buildAggregates()); 103 if (searchOnAllRepositories()) { 104 nxQuery.searchOnAllRepositories(); 105 } 106 nxQuery.useUnrestrictedSession(useUnrestrictedSession()); 107 108 List<String> highlightFields = getHighlights(); 109 if (highlightFields != null && !highlightFields.isEmpty()) { 110 nxQuery.highlight(highlightFields); 111 } 112 113 EsResult ret = ess.queryAndAggregate(nxQuery); 114 DocumentModelList dmList = ret.getDocuments(); 115 currentAggregates = new HashMap<>(ret.getAggregates().size()); 116 for (Aggregate<Bucket> agg : ret.getAggregates()) { 117 currentAggregates.put(agg.getId(), agg); 118 } 119 setResultsCount(dmList.totalSize()); 120 currentPageDocuments = dmList; 121 } catch (QueryParseException e) { 122 error = e; 123 errorMessage = e.getMessage(); 124 log.warn(e.getMessage(), e); 125 } 126 127 // send event for statistics ! 128 fireSearchEvent(getCoreSession().getPrincipal(), query, currentPageDocuments, System.currentTimeMillis() - t0); 129 130 return currentPageDocuments; 131 } 132 133 protected int getLimit() { 134 int ret = (int) getMinMaxPageSize(); 135 if (ret == 0) { 136 ret = (int) Long.min(getMaxResultWindow(), Integer.MAX_VALUE); 137 } 138 return ret; 139 } 140 141 public QueryBuilder getCurrentQueryAsEsBuilder() { 142 String nxql = getCurrentQuery(); 143 return NxqlQueryConverter.toESQueryBuilder(nxql); 144 } 145 146 @Override 147 protected void pageChanged() { 148 currentPageDocuments = null; 149 currentAggregates = null; 150 super.pageChanged(); 151 } 152 153 @Override 154 public void refresh() { 155 currentPageDocuments = null; 156 currentAggregates = null; 157 super.refresh(); 158 } 159 160 @Override 161 protected CoreSession getCoreSession() { 162 Map<String, Serializable> props = getProperties(); 163 CoreSession coreSession = (CoreSession) props.get(CORE_SESSION_PROPERTY); 164 if (coreSession == null) { 165 throw new NuxeoException("cannot find core session"); 166 } 167 return coreSession; 168 } 169 170 private List<AggregateEsBase<? extends Aggregation, ? extends Bucket>> buildAggregates() { 171 ArrayList<AggregateEsBase<? extends Aggregation, ? extends Bucket>> ret = new ArrayList<>( 172 getAggregateDefinitions().size()); 173 boolean skip = isSkipAggregates(); 174 for (AggregateDefinition def : getAggregateDefinitions()) { 175 AggregateEsBase<? extends Aggregation, ? extends Bucket> agg = AggregateFactory.create(def, 176 getSearchDocumentModel()); 177 if (!skip || !agg.getSelection().isEmpty()) { 178 // if we want to skip aggregates but one is selected, it has to be computed to filter the result set 179 ret.add(AggregateFactory.create(def, getSearchDocumentModel())); 180 } 181 } 182 return ret; 183 } 184 185 protected boolean searchOnAllRepositories() { 186 String value = (String) getProperties().get(SEARCH_ON_ALL_REPOSITORIES_PROPERTY); 187 if (value == null) { 188 return false; 189 } 190 return Boolean.parseBoolean(value); 191 } 192 193 @Override 194 public boolean hasAggregateSupport() { 195 return true; 196 } 197 198 @Override 199 public Map<String, Aggregate<? extends Bucket>> getAggregates() { 200 getCurrentPage(); 201 return currentAggregates; 202 } 203 204 /** 205 * Extends the default implementation to add results of aggregates 206 * 207 * @since 7.4 208 */ 209 @Override 210 protected void incorporateAggregates(Map<String, Serializable> eventProps) { 211 212 super.incorporateAggregates(eventProps); 213 if (currentAggregates != null) { 214 HashMap<String, Serializable> aggregateMatches = new HashMap<>(); 215 for (String key : currentAggregates.keySet()) { 216 Aggregate<? extends Bucket> ag = currentAggregates.get(key); 217 ArrayList<HashMap<String, Serializable>> buckets = new ArrayList<>(); 218 for (Bucket bucket : ag.getBuckets()) { 219 HashMap<String, Serializable> b = new HashMap<>(); 220 b.put("key", bucket.getKey()); 221 b.put("count", bucket.getDocCount()); 222 buckets.add(b); 223 } 224 aggregateMatches.put(key, buckets); 225 } 226 eventProps.put("aggregatesMatches", aggregateMatches); 227 } 228 } 229 230 @Override 231 public boolean isLastPageAvailable() { 232 if ((getResultsCount() + getPageSize()) <= getMaxResultWindow()) { 233 return super.isNextPageAvailable(); 234 } 235 return false; 236 } 237 238 @Override 239 public boolean isNextPageAvailable() { 240 if ((getCurrentPageOffset() + 2 * getPageSize()) <= getMaxResultWindow()) { 241 return super.isNextPageAvailable(); 242 } 243 return false; 244 } 245 246 @Override 247 public long getPageLimit() { 248 return getMaxResultWindow() / getPageSize(); 249 } 250 251 /** 252 * Returns the max result window where the PP can navigate without raising Elasticsearch 253 * QueryPhaseExecutionException. {@code from + size} must be less than or equal to this value. 254 * 255 * @since 9.2 256 */ 257 public long getMaxResultWindow() { 258 if (maxResultWindow == null) { 259 ConfigurationService cs = Framework.getService(ConfigurationService.class); 260 maxResultWindow = cs.getLong(ES_MAX_RESULT_WINDOW_PROPERTY, DEFAULT_ES_MAX_RESULT_WINDOW_VALUE); 261 } 262 return maxResultWindow; 263 } 264 265 @Override 266 public long getResultsCountLimit() { 267 return getMaxResultWindow(); 268 } 269 270 /** 271 * Set the max result window where the PP can navigate, for testing purpose. 272 * 273 * @since 9.2 274 */ 275 public void setMaxResultWindow(long maxResultWindow) { 276 this.maxResultWindow = maxResultWindow; 277 } 278 279}