001/* 002 * (C) Copyright 2014 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * bdelbosc 018 */ 019 020package org.nuxeo.elasticsearch.provider; 021 022import java.io.Serializable; 023import java.util.ArrayList; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027 028import org.apache.commons.logging.Log; 029import org.apache.commons.logging.LogFactory; 030import org.elasticsearch.index.query.QueryBuilder; 031import org.nuxeo.ecm.core.api.CoreSession; 032import org.nuxeo.ecm.core.api.DocumentModel; 033import org.nuxeo.ecm.core.api.DocumentModelList; 034import org.nuxeo.ecm.core.api.NuxeoException; 035import org.nuxeo.ecm.core.query.QueryParseException; 036import org.nuxeo.ecm.platform.query.api.Aggregate; 037import org.nuxeo.ecm.platform.query.api.AggregateDefinition; 038import org.nuxeo.ecm.platform.query.api.Bucket; 039import org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider; 040import org.nuxeo.elasticsearch.aggregate.AggregateEsBase; 041import org.nuxeo.elasticsearch.aggregate.AggregateFactory; 042import org.nuxeo.elasticsearch.api.ElasticSearchService; 043import org.nuxeo.elasticsearch.api.EsResult; 044import org.nuxeo.elasticsearch.query.NxQueryBuilder; 045import org.nuxeo.elasticsearch.query.NxqlQueryConverter; 046import org.nuxeo.runtime.api.Framework; 047import org.nuxeo.runtime.services.config.ConfigurationService; 048 049/** 050 * Elasticsearch Page provider that converts the NXQL query build by CoreQueryDocumentPageProvider. 051 * 052 * @since 5.9.3 053 */ 054public class ElasticSearchNxqlPageProvider extends CoreQueryDocumentPageProvider { 055 056 public static final String CORE_SESSION_PROPERTY = "coreSession"; 057 058 public static final String SEARCH_ON_ALL_REPOSITORIES_PROPERTY = "searchAllRepositories"; 059 060 // @since 9.2 061 public static final String ES_MAX_RESULT_WINDOW_PROPERTY = "org.nuxeo.elasticsearch.provider.maxResultWindow"; 062 063 // This is the default ES index.max_result_window 064 public static final String DEFAULT_ES_MAX_RESULT_WINDOW_VALUE = "10000"; 065 066 protected static final Log log = LogFactory.getLog(ElasticSearchNxqlPageProvider.class); 067 068 private static final long serialVersionUID = 1L; 069 070 protected HashMap<String, Aggregate<? extends Bucket>> currentAggregates; 071 072 protected Long maxResultWindow; 073 074 @Override 075 public List<DocumentModel> getCurrentPage() { 076 077 long t0 = System.currentTimeMillis(); 078 079 // use a cache 080 if (currentPageDocuments != null) { 081 return currentPageDocuments; 082 } 083 error = null; 084 errorMessage = null; 085 if (log.isDebugEnabled()) { 086 log.debug(String.format("Perform query for provider '%s': with pageSize=%d, offset=%d", getName(), 087 getMinMaxPageSize(), getCurrentPageOffset())); 088 } 089 currentPageDocuments = new ArrayList<>(); 090 CoreSession coreSession = getCoreSession(); 091 if (query == null) { 092 buildQuery(coreSession); 093 } 094 if (query == null) { 095 throw new NuxeoException(String.format("Cannot perform null query: check provider '%s'", getName())); 096 } 097 // Build and execute the ES query 098 ElasticSearchService ess = Framework.getService(ElasticSearchService.class); 099 try { 100 NxQueryBuilder nxQuery = new NxQueryBuilder(getCoreSession()).nxql(query) 101 .offset((int) getCurrentPageOffset()) 102 .limit(getLimit()) 103 .addAggregates(buildAggregates()); 104 if (searchOnAllRepositories()) { 105 nxQuery.searchOnAllRepositories(); 106 } 107 108 List<String> highlightFields = getHighlights(); 109 if (highlightFields != null && !highlightFields.isEmpty()) { 110 nxQuery.highlight(highlightFields); 111 } 112 113 EsResult ret = ess.queryAndAggregate(nxQuery); 114 DocumentModelList dmList = ret.getDocuments(); 115 currentAggregates = new HashMap<>(ret.getAggregates().size()); 116 for (Aggregate<Bucket> agg : ret.getAggregates()) { 117 currentAggregates.put(agg.getId(), agg); 118 } 119 setResultsCount(dmList.totalSize()); 120 currentPageDocuments = dmList; 121 } catch (QueryParseException e) { 122 error = e; 123 errorMessage = e.getMessage(); 124 log.warn(e.getMessage(), e); 125 } 126 127 // send event for statistics ! 128 fireSearchEvent(getCoreSession().getPrincipal(), query, currentPageDocuments, System.currentTimeMillis() - t0); 129 130 return currentPageDocuments; 131 } 132 133 protected int getLimit() { 134 int ret = (int) getMinMaxPageSize(); 135 if (ret == 0) { 136 ret = -1; 137 } 138 return ret; 139 } 140 141 public QueryBuilder getCurrentQueryAsEsBuilder() { 142 String nxql = getCurrentQuery(); 143 return NxqlQueryConverter.toESQueryBuilder(nxql); 144 } 145 146 @Override 147 protected void pageChanged() { 148 currentPageDocuments = null; 149 currentAggregates = null; 150 super.pageChanged(); 151 } 152 153 @Override 154 public void refresh() { 155 currentPageDocuments = null; 156 currentAggregates = null; 157 super.refresh(); 158 } 159 160 @Override 161 protected CoreSession getCoreSession() { 162 Map<String, Serializable> props = getProperties(); 163 CoreSession coreSession = (CoreSession) props.get(CORE_SESSION_PROPERTY); 164 if (coreSession == null) { 165 throw new NuxeoException("cannot find core session"); 166 } 167 return coreSession; 168 } 169 170 private List<AggregateEsBase<? extends Bucket>> buildAggregates() { 171 ArrayList<AggregateEsBase<? extends Bucket>> ret = new ArrayList<>(getAggregateDefinitions().size()); 172 for (AggregateDefinition def : getAggregateDefinitions()) { 173 ret.add(AggregateFactory.create(def, getSearchDocumentModel())); 174 } 175 return ret; 176 } 177 178 protected boolean searchOnAllRepositories() { 179 String value = (String) getProperties().get(SEARCH_ON_ALL_REPOSITORIES_PROPERTY); 180 if (value == null) { 181 return false; 182 } 183 return Boolean.parseBoolean(value); 184 } 185 186 @Override 187 public boolean hasAggregateSupport() { 188 return true; 189 } 190 191 @Override 192 public Map<String, Aggregate<? extends Bucket>> getAggregates() { 193 getCurrentPage(); 194 return currentAggregates; 195 } 196 197 /** 198 * Extends the default implementation to add results of aggregates 199 * 200 * @since 7.4 201 */ 202 @Override 203 protected void incorporateAggregates(Map<String, Serializable> eventProps) { 204 205 super.incorporateAggregates(eventProps); 206 if (currentAggregates != null) { 207 HashMap<String, Serializable> aggregateMatches = new HashMap<>(); 208 for (String key : currentAggregates.keySet()) { 209 Aggregate<? extends Bucket> ag = currentAggregates.get(key); 210 ArrayList<HashMap<String, Serializable>> buckets = new ArrayList<>(); 211 for (Bucket bucket : ag.getBuckets()) { 212 HashMap<String, Serializable> b = new HashMap<>(); 213 b.put("key", bucket.getKey()); 214 b.put("count", bucket.getDocCount()); 215 buckets.add(b); 216 } 217 aggregateMatches.put(key, buckets); 218 } 219 eventProps.put("aggregatesMatches", aggregateMatches); 220 } 221 } 222 223 @Override 224 public boolean isLastPageAvailable() { 225 if ((getResultsCount() + getPageSize()) <= getMaxResultWindow()) { 226 return super.isNextPageAvailable(); 227 } 228 return false; 229 } 230 231 @Override 232 public boolean isNextPageAvailable() { 233 if ((getCurrentPageOffset() + 2 * getPageSize()) <= getMaxResultWindow()) { 234 return super.isNextPageAvailable(); 235 } 236 return false; 237 } 238 239 @Override 240 public long getPageLimit() { 241 return getMaxResultWindow() / getPageSize(); 242 } 243 244 /** 245 * Returns the max result window where the PP can navigate without raising Elasticsearch 246 * QueryPhaseExecutionException. {@code from + size} must be less than or equal to this value. 247 * 248 * @since 9.2 249 */ 250 public long getMaxResultWindow() { 251 if (maxResultWindow == null) { 252 ConfigurationService cs = Framework.getService(ConfigurationService.class); 253 String maxResultWindowStr = cs.getProperty(ES_MAX_RESULT_WINDOW_PROPERTY, 254 DEFAULT_ES_MAX_RESULT_WINDOW_VALUE); 255 try { 256 maxResultWindow = Long.valueOf(maxResultWindowStr); 257 } catch (NumberFormatException e) { 258 log.warn(String.format( 259 "Invalid maxResultWindow property value: %s for page provider: %s, fallback to default.", 260 maxResultWindowStr, getName())); 261 maxResultWindow = Long.valueOf(DEFAULT_ES_MAX_RESULT_WINDOW_VALUE); 262 } 263 } 264 return maxResultWindow; 265 } 266 267 @Override 268 public long getResultsCountLimit() { 269 return getMaxResultWindow(); 270 } 271 272 /** 273 * Set the max result window where the PP can navigate, for testing purpose. 274 * 275 * @since 9.2 276 */ 277 public void setMaxResultWindow(long maxResultWindow) { 278 this.maxResultWindow = maxResultWindow; 279 } 280 281}