001/* 002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Antoine Taillefer <ataillefer@nuxeo.com> 018 */ 019package org.nuxeo.drive.adapter.impl; 020 021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY; 022 023import java.io.IOException; 024import java.io.Serializable; 025import java.util.ArrayList; 026import java.util.HashMap; 027import java.util.Iterator; 028import java.util.List; 029import java.util.Map; 030import java.util.UUID; 031import java.util.concurrent.Semaphore; 032 033import org.apache.commons.lang.StringUtils; 034import org.apache.commons.logging.Log; 035import org.apache.commons.logging.LogFactory; 036import org.nuxeo.drive.adapter.FileItem; 037import org.nuxeo.drive.adapter.FileSystemItem; 038import org.nuxeo.drive.adapter.FolderItem; 039import org.nuxeo.drive.adapter.RootlessItemException; 040import org.nuxeo.drive.adapter.ScrollFileSystemItemList; 041import org.nuxeo.drive.service.FileSystemItemAdapterService; 042import org.nuxeo.ecm.core.api.Blob; 043import org.nuxeo.ecm.core.api.CoreInstance; 044import org.nuxeo.ecm.core.api.CoreSession; 045import org.nuxeo.ecm.core.api.DocumentModel; 046import org.nuxeo.ecm.core.api.DocumentModelList; 047import org.nuxeo.ecm.core.api.DocumentRef; 048import org.nuxeo.ecm.core.api.DocumentSecurityException; 049import org.nuxeo.ecm.core.api.IdRef; 050import org.nuxeo.ecm.core.api.IterableQueryResult; 051import org.nuxeo.ecm.core.api.NuxeoException; 052import org.nuxeo.ecm.core.api.security.SecurityConstants; 053import org.nuxeo.ecm.core.cache.Cache; 054import org.nuxeo.ecm.core.cache.CacheService; 055import org.nuxeo.ecm.core.query.sql.NXQL; 056import org.nuxeo.ecm.core.schema.FacetNames; 057import org.nuxeo.ecm.platform.filemanager.api.FileManager; 058import org.nuxeo.ecm.platform.query.api.PageProvider; 059import org.nuxeo.ecm.platform.query.api.PageProviderService; 060import org.nuxeo.runtime.api.Framework; 061import org.nuxeo.runtime.services.config.ConfigurationService; 062 063/** 064 * {@link DocumentModel} backed implementation of a {@link FolderItem}. 065 * 066 * @author Antoine Taillefer 067 */ 068public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem { 069 070 private static final Log log = LogFactory.getLog(DocumentBackedFolderItem.class); 071 072 private static final long serialVersionUID = 1L; 073 074 private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN"; 075 076 protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll"; 077 078 protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize"; 079 080 protected static final String MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = "1000"; 081 082 protected static final int VCS_CHUNK_SIZE = 100; 083 084 protected boolean canCreateChild; 085 086 protected boolean canScrollDescendants; 087 088 public DocumentBackedFolderItem(String factoryName, DocumentModel doc) { 089 this(factoryName, doc, false); 090 } 091 092 public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) { 093 this(factoryName, doc, relaxSyncRootConstraint, true); 094 } 095 096 public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint, 097 boolean getLockInfo) { 098 super(factoryName, doc, relaxSyncRootConstraint, getLockInfo); 099 initialize(doc); 100 } 101 102 public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) { 103 this(factoryName, parentItem, doc, false); 104 } 105 106 public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc, 107 boolean relaxSyncRootConstraint) { 108 this(factoryName, parentItem, doc, relaxSyncRootConstraint, true); 109 } 110 111 public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc, 112 boolean relaxSyncRootConstraint, boolean getLockInfo) { 113 super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo); 114 initialize(doc); 115 } 116 117 protected DocumentBackedFolderItem() { 118 // Needed for JSON deserialization 119 } 120 121 /*--------------------- FileSystemItem ---------------------*/ 122 @Override 123 public void rename(String name) { 124 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 125 // Update doc properties 126 DocumentModel doc = getDocument(session); 127 doc.setPropertyValue("dc:title", name); 128 doc = session.saveDocument(doc); 129 session.save(); 130 // Update FileSystemItem attributes 131 this.docTitle = name; 132 this.name = name; 133 updateLastModificationDate(doc); 134 } 135 } 136 137 /*--------------------- FolderItem -----------------*/ 138 @Override 139 @SuppressWarnings("unchecked") 140 public List<FileSystemItem> getChildren() { 141 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 142 PageProviderService pageProviderService = Framework.getLocalService(PageProviderService.class); 143 Map<String, Serializable> props = new HashMap<String, Serializable>(); 144 props.put(CORE_SESSION_PROPERTY, (Serializable) session); 145 PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider( 146 FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId); 147 Long pageSize = childrenPageProvider.getPageSize(); 148 149 List<FileSystemItem> children = new ArrayList<FileSystemItem>(); 150 int nbChildren = 0; 151 boolean reachedPageSize = false; 152 boolean hasNextPage = true; 153 // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size 154 // number of 155 // FileSystemItems 156 while (nbChildren < pageSize && hasNextPage) { 157 List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage(); 158 for (DocumentModel dmChild : dmChildren) { 159 // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo 160 FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false, 161 false, false); 162 if (child != null) { 163 children.add(child); 164 nbChildren++; 165 if (nbChildren == pageSize) { 166 reachedPageSize = true; 167 break; 168 } 169 } 170 } 171 if (!reachedPageSize) { 172 hasNextPage = childrenPageProvider.isNextPageAvailable(); 173 if (hasNextPage) { 174 childrenPageProvider.nextPage(); 175 } 176 } 177 } 178 179 return children; 180 } 181 } 182 183 @Override 184 public boolean getCanScrollDescendants() { 185 return canScrollDescendants; 186 } 187 188 @Override 189 public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) { 190 Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore(); 191 try { 192 if (log.isTraceEnabled()) { 193 log.trace(String.format("Thread [%s] acquiring scroll batch semaphore", Thread.currentThread() 194 .getName())); 195 } 196 semaphore.acquire(); 197 try { 198 if (log.isTraceEnabled()) { 199 log.trace(String.format( 200 "Thread [%s] acquired scroll batch semaphore, available permits reduced to %d", 201 Thread.currentThread().getName(), semaphore.availablePermits())); 202 } 203 return doScrollDescendants(scrollId, batchSize, keepAlive); 204 } finally { 205 semaphore.release(); 206 if (log.isTraceEnabled()) { 207 log.trace(String.format( 208 "Thread [%s] released scroll batch semaphore, available permits increased to %d", 209 Thread.currentThread().getName(), semaphore.availablePermits())); 210 } 211 } 212 } catch (InterruptedException cause) { 213 Thread.currentThread().interrupt(); 214 throw new NuxeoException("Scroll batch interrupted", cause); 215 } 216 } 217 218 @SuppressWarnings("unchecked") 219 protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) { 220 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 221 222 // Limit batch size sent by the client 223 checkBatchSize(batchSize); 224 225 // Scroll through a batch of documents 226 ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive); 227 String newScrollId = descendantDocsBatch.getScrollId(); 228 if (descendantDocsBatch.isEmpty()) { 229 // No more descendants left to return 230 return new ScrollFileSystemItemListImpl(newScrollId, 0); 231 } 232 233 // Adapt documents as FileSystemItems 234 List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session); 235 if (log.isDebugEnabled()) { 236 log.debug(String.format("Retrieved %d descendants of FolderItem %s (batchSize = %d)", 237 descendants.size(), docPath, batchSize)); 238 } 239 return new ScrollFileSystemItemListImpl(newScrollId, descendants); 240 } 241 } 242 243 protected void checkBatchSize(int batchSize) { 244 int maxDescendantsBatchSize = Integer.parseInt(Framework.getService(ConfigurationService.class).getProperty( 245 MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT)); 246 if (batchSize > maxDescendantsBatchSize) { 247 throw new NuxeoException( 248 String.format( 249 "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.", 250 batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY)); 251 } 252 } 253 254 @SuppressWarnings("unchecked") 255 protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session, long keepAlive) { 256 Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE); 257 if (scrollingCache == null) { 258 throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE); 259 } 260 String newScrollId; 261 List<String> descendantIds; 262 if (StringUtils.isEmpty(scrollId)) { 263 // Perform initial query to fetch ids of all the descendant documents and put the result list in a 264 // cache, aka "search context" 265 descendantIds = new ArrayList<>(); 266 StringBuilder sb = new StringBuilder(String.format( 267 "SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId)); 268 sb.append(" AND ecm:currentLifeCycleState != 'deleted'"); 269 sb.append(" AND ecm:mixinType != 'HiddenInNavigation'"); 270 // Don't need to add ecm:isCheckedInVersion = 0 because versions are already excluded by the 271 // ecm:ancestorId clause since they have no path 272 String query = sb.toString(); 273 if (log.isDebugEnabled()) { 274 log.debug(String.format("Executing initial query to scroll through the descendants of %s: %s", docPath, 275 query)); 276 } 277 try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) { 278 Iterator<Map<String, Serializable>> it = res.iterator(); 279 while (it.hasNext()) { 280 descendantIds.add((String) it.next().get(NXQL.ECM_UUID)); 281 } 282 } 283 // Generate a scroll id 284 newScrollId = UUID.randomUUID().toString(); 285 if (log.isDebugEnabled()) { 286 log.debug(String.format( 287 "Put initial query result list (search context) in the %s cache at key (scrollId) %s", 288 DESCENDANTS_SCROLL_CACHE, newScrollId)); 289 } 290 scrollingCache.put(newScrollId, (Serializable) descendantIds); 291 } else { 292 // Get the descendant ids from the cache 293 descendantIds = (List<String>) scrollingCache.get(scrollId); 294 if (descendantIds == null) { 295 throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]", 296 DESCENDANTS_SCROLL_CACHE, scrollId)); 297 } 298 newScrollId = scrollId; 299 } 300 301 if (descendantIds.isEmpty()) { 302 return new ScrollDocumentModelList(newScrollId, 0); 303 } 304 305 // Extract a batch of descendant ids 306 List<String> descendantIdsBatch = getBatch(descendantIds, batchSize); 307 // Fetch documents from VCS 308 DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session); 309 return new ScrollDocumentModelList(newScrollId, descendantDocsBatch); 310 } 311 312 /** 313 * Extracts batchSize elements from the input list. 314 */ 315 protected List<String> getBatch(List<String> ids, int batchSize) { 316 List<String> batch = new ArrayList<>(batchSize); 317 int idCount = 0; 318 Iterator<String> it = ids.iterator(); 319 while (it.hasNext() && idCount < batchSize) { 320 batch.add(it.next()); 321 it.remove(); 322 idCount++; 323 } 324 return batch; 325 } 326 327 protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) { 328 DocumentModelList res = null; 329 int size = ids.size(); 330 int start = 0; 331 int end = Math.min(VCS_CHUNK_SIZE, size); 332 boolean done = false; 333 while (!done) { 334 DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session); 335 if (res == null) { 336 res = docs; 337 } else { 338 res.addAll(docs); 339 } 340 if (end >= ids.size()) { 341 done = true; 342 } else { 343 start = end; 344 end = Math.min(start + VCS_CHUNK_SIZE, size); 345 } 346 } 347 return res; 348 } 349 350 protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) { 351 int docCount = ids.size(); 352 StringBuilder sb = new StringBuilder(); 353 sb.append("SELECT * FROM Document WHERE ecm:uuid IN ("); 354 for (int i = 0; i < docCount; i++) { 355 sb.append(NXQL.escapeString(ids.get(i))); 356 if (i < docCount - 1) { 357 sb.append(", "); 358 } 359 } 360 sb.append(")"); 361 String query = sb.toString(); 362 if (log.isDebugEnabled()) { 363 log.debug(String.format("Fetching %d documents from VCS: %s", docCount, query)); 364 } 365 return session.query(query); 366 } 367 368 /** 369 * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem} 370 * ancestors. 371 */ 372 protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) { 373 Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>(); 374 if (log.isTraceEnabled()) { 375 log.trace(String.format("Caching current FolderItem for doc %s: %s", docPath, getPath())); 376 } 377 ancestorCache.put(new IdRef(docId), this); 378 List<FileSystemItem> descendants = new ArrayList<>(docs.size()); 379 for (DocumentModel doc : docs) { 380 FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false); 381 // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo 382 FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false, 383 false); 384 if (descendant != null) { 385 if (descendant.isFolder()) { 386 if (log.isTraceEnabled()) { 387 log.trace(String.format("Caching descendant FolderItem for doc %s: %s", doc.getPathAsString(), 388 descendant.getPath())); 389 } 390 ancestorCache.put(doc.getRef(), (FolderItem) descendant); 391 } 392 descendants.add(descendant); 393 } 394 } 395 return descendants; 396 } 397 398 protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc, 399 CoreSession session, boolean cacheItem) { 400 // TODO: handle collections 401 DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef()); 402 if (parentDocRef == null) { 403 throw new RootlessItemException("Reached repository root"); 404 } 405 406 FolderItem parentItem = cache.get(parentDocRef); 407 if (parentItem != null) { 408 if (log.isTraceEnabled()) { 409 log.trace(String.format("Found parent FolderItem in cache for doc %s: %s", doc.getPathAsString(), 410 parentItem.getPath())); 411 } 412 return getFolderItem(cache, doc, parentItem, cacheItem); 413 } 414 415 if (log.isTraceEnabled()) { 416 log.trace(String.format("No parent FolderItem found in cache for doc %s, computing ancestor cache", 417 doc.getPathAsString())); 418 } 419 DocumentModel parentDoc = null; 420 try { 421 parentDoc = session.getDocument(parentDocRef); 422 } catch (DocumentSecurityException e) { 423 throw new RootlessItemException(String.format("User %s has no READ access on parent of document %s (%s).", 424 principal.getName(), doc.getPathAsString(), doc.getId()), e); 425 } 426 parentItem = populateAncestorCache(cache, parentDoc, session, true); 427 return getFolderItem(cache, doc, parentItem, cacheItem); 428 } 429 430 protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem, 431 boolean cacheItem) { 432 if (cacheItem) { 433 // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo 434 FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false, 435 false); 436 if (fsItem == null) { 437 throw new RootlessItemException( 438 String.format( 439 "Reached a document %s that cannot be adapted as a (possibly virtual) descendant of the top level folder item.", 440 doc.getPathAsString())); 441 } 442 FolderItem folderItem = (FolderItem) fsItem; 443 if (log.isTraceEnabled()) { 444 log.trace(String.format("Caching FolderItem for doc %s: %s", doc.getPathAsString(), 445 folderItem.getPath())); 446 } 447 cache.put(doc.getRef(), folderItem); 448 return folderItem; 449 } else { 450 return parentItem; 451 } 452 } 453 454 @Override 455 public boolean getCanCreateChild() { 456 return canCreateChild; 457 } 458 459 @Override 460 public FolderItem createFolder(String name) { 461 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 462 DocumentModel folder = getFileManager().createFolder(session, name, docPath); 463 if (folder == null) { 464 throw new NuxeoException( 465 String.format( 466 "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.", 467 name, docPath)); 468 } 469 return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this); 470 } catch (NuxeoException e) { 471 e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath)); 472 throw e; 473 } catch (IOException e) { 474 throw new NuxeoException(String.format("Error while trying to create folder %s as a child of doc %s", name, 475 docPath), e); 476 } 477 } 478 479 @Override 480 public FileItem createFile(Blob blob) { 481 String fileName = blob.getFilename(); 482 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 483 // TODO: manage conflict (overwrite should not necessarily be true) 484 DocumentModel file = getFileManager().createDocumentFromBlob(session, blob, docPath, true, fileName); 485 if (file == null) { 486 throw new NuxeoException( 487 String.format( 488 "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.", 489 fileName, docPath)); 490 } 491 return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this); 492 } catch (NuxeoException e) { 493 e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath)); 494 throw e; 495 } catch (IOException e) { 496 throw new NuxeoException(String.format("Error while trying to create file %s as a child of doc %s", 497 fileName, docPath), e); 498 } 499 } 500 501 /*--------------------- Protected -----------------*/ 502 protected void initialize(DocumentModel doc) { 503 this.name = docTitle; 504 this.folder = true; 505 this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE); 506 if (canCreateChild) { 507 if (Framework.getService(ConfigurationService.class).isBooleanPropertyTrue( 508 PERMISSION_CHECK_OPTIMIZED_PROPERTY)) { 509 // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties 510 // and by default WriteProperties <=> Write <=> AddChildren 511 this.canCreateChild = canRename; 512 } else { 513 // In non optimized mode check AddChildren 514 this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN); 515 } 516 } 517 this.canScrollDescendants = true; 518 } 519 520 protected FileManager getFileManager() { 521 return Framework.getLocalService(FileManager.class); 522 } 523 524 /*---------- Needed for JSON deserialization ----------*/ 525 protected void setCanCreateChild(boolean canCreateChild) { 526 this.canCreateChild = canCreateChild; 527 } 528 529 protected void setCanScrollDescendants(boolean canScrollDescendants) { 530 this.canScrollDescendants = canScrollDescendants; 531 } 532 533}