001/* 002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Antoine Taillefer <ataillefer@nuxeo.com> 018 */ 019package org.nuxeo.drive.adapter.impl; 020 021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY; 022 023import java.io.IOException; 024import java.io.Serializable; 025import java.util.ArrayList; 026import java.util.HashMap; 027import java.util.Iterator; 028import java.util.List; 029import java.util.Map; 030import java.util.UUID; 031import java.util.concurrent.Semaphore; 032 033import org.apache.commons.lang.StringUtils; 034import org.apache.commons.logging.Log; 035import org.apache.commons.logging.LogFactory; 036import org.nuxeo.drive.adapter.FileItem; 037import org.nuxeo.drive.adapter.FileSystemItem; 038import org.nuxeo.drive.adapter.FolderItem; 039import org.nuxeo.drive.adapter.RootlessItemException; 040import org.nuxeo.drive.adapter.ScrollFileSystemItemList; 041import org.nuxeo.drive.service.FileSystemItemAdapterService; 042import org.nuxeo.ecm.core.api.Blob; 043import org.nuxeo.ecm.core.api.CoreInstance; 044import org.nuxeo.ecm.core.api.CoreSession; 045import org.nuxeo.ecm.core.api.DocumentModel; 046import org.nuxeo.ecm.core.api.DocumentModelList; 047import org.nuxeo.ecm.core.api.DocumentRef; 048import org.nuxeo.ecm.core.api.DocumentSecurityException; 049import org.nuxeo.ecm.core.api.IdRef; 050import org.nuxeo.ecm.core.api.IterableQueryResult; 051import org.nuxeo.ecm.core.api.NuxeoException; 052import org.nuxeo.ecm.core.api.security.SecurityConstants; 053import org.nuxeo.ecm.core.cache.Cache; 054import org.nuxeo.ecm.core.cache.CacheService; 055import org.nuxeo.ecm.core.query.sql.NXQL; 056import org.nuxeo.ecm.core.schema.FacetNames; 057import org.nuxeo.ecm.platform.filemanager.api.FileManager; 058import org.nuxeo.ecm.platform.query.api.PageProvider; 059import org.nuxeo.ecm.platform.query.api.PageProviderService; 060import org.nuxeo.runtime.api.Framework; 061import org.nuxeo.runtime.services.config.ConfigurationService; 062 063/** 064 * {@link DocumentModel} backed implementation of a {@link FolderItem}. 065 * 066 * @author Antoine Taillefer 067 */ 068public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem { 069 070 private static final Log log = LogFactory.getLog(DocumentBackedFolderItem.class); 071 072 private static final long serialVersionUID = 1L; 073 074 private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN"; 075 076 protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll"; 077 078 protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize"; 079 080 protected static final String MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = "1000"; 081 082 protected static final int VCS_CHUNK_SIZE = 100; 083 084 protected boolean canCreateChild; 085 086 protected boolean canScrollDescendants; 087 088 public DocumentBackedFolderItem(String factoryName, DocumentModel doc) { 089 this(factoryName, doc, false); 090 } 091 092 public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) { 093 this(factoryName, doc, relaxSyncRootConstraint, true); 094 } 095 096 public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint, 097 boolean getLockInfo) { 098 super(factoryName, doc, relaxSyncRootConstraint, getLockInfo); 099 initialize(doc); 100 } 101 102 public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) { 103 this(factoryName, parentItem, doc, false); 104 } 105 106 public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc, 107 boolean relaxSyncRootConstraint) { 108 this(factoryName, parentItem, doc, relaxSyncRootConstraint, true); 109 } 110 111 public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc, 112 boolean relaxSyncRootConstraint, boolean getLockInfo) { 113 super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo); 114 initialize(doc); 115 } 116 117 protected DocumentBackedFolderItem() { 118 // Needed for JSON deserialization 119 } 120 121 /*--------------------- FileSystemItem ---------------------*/ 122 @Override 123 public void rename(String name) { 124 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 125 // Update doc properties 126 DocumentModel doc = getDocument(session); 127 doc.setPropertyValue("dc:title", name); 128 doc.putContextData(CoreSession.SOURCE, "drive"); 129 doc = session.saveDocument(doc); 130 session.save(); 131 // Update FileSystemItem attributes 132 this.docTitle = name; 133 this.name = name; 134 updateLastModificationDate(doc); 135 } 136 } 137 138 /*--------------------- FolderItem -----------------*/ 139 @Override 140 @SuppressWarnings("unchecked") 141 public List<FileSystemItem> getChildren() { 142 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 143 PageProviderService pageProviderService = Framework.getService(PageProviderService.class); 144 Map<String, Serializable> props = new HashMap<String, Serializable>(); 145 props.put(CORE_SESSION_PROPERTY, (Serializable) session); 146 PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider( 147 FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId); 148 Long pageSize = childrenPageProvider.getPageSize(); 149 150 List<FileSystemItem> children = new ArrayList<FileSystemItem>(); 151 int nbChildren = 0; 152 boolean reachedPageSize = false; 153 boolean hasNextPage = true; 154 // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size 155 // number of 156 // FileSystemItems 157 while (nbChildren < pageSize && hasNextPage) { 158 List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage(); 159 for (DocumentModel dmChild : dmChildren) { 160 // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo 161 FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false, 162 false, false); 163 if (child != null) { 164 children.add(child); 165 nbChildren++; 166 if (nbChildren == pageSize) { 167 reachedPageSize = true; 168 break; 169 } 170 } 171 } 172 if (!reachedPageSize) { 173 hasNextPage = childrenPageProvider.isNextPageAvailable(); 174 if (hasNextPage) { 175 childrenPageProvider.nextPage(); 176 } 177 } 178 } 179 180 return children; 181 } 182 } 183 184 @Override 185 public boolean getCanScrollDescendants() { 186 return canScrollDescendants; 187 } 188 189 @Override 190 public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) { 191 Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore(); 192 try { 193 if (log.isTraceEnabled()) { 194 log.trace(String.format("Thread [%s] acquiring scroll batch semaphore", 195 Thread.currentThread().getName())); 196 } 197 semaphore.acquire(); 198 try { 199 if (log.isTraceEnabled()) { 200 log.trace(String.format( 201 "Thread [%s] acquired scroll batch semaphore, available permits reduced to %d", 202 Thread.currentThread().getName(), semaphore.availablePermits())); 203 } 204 return doScrollDescendants(scrollId, batchSize, keepAlive); 205 } finally { 206 semaphore.release(); 207 if (log.isTraceEnabled()) { 208 log.trace(String.format( 209 "Thread [%s] released scroll batch semaphore, available permits increased to %d", 210 Thread.currentThread().getName(), semaphore.availablePermits())); 211 } 212 } 213 } catch (InterruptedException cause) { 214 Thread.currentThread().interrupt(); 215 throw new NuxeoException("Scroll batch interrupted", cause); 216 } 217 } 218 219 @SuppressWarnings("unchecked") 220 protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) { 221 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 222 223 // Limit batch size sent by the client 224 checkBatchSize(batchSize); 225 226 // Scroll through a batch of documents 227 ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive); 228 String newScrollId = descendantDocsBatch.getScrollId(); 229 if (descendantDocsBatch.isEmpty()) { 230 // No more descendants left to return 231 return new ScrollFileSystemItemListImpl(newScrollId, 0); 232 } 233 234 // Adapt documents as FileSystemItems 235 List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session); 236 if (log.isDebugEnabled()) { 237 log.debug(String.format("Retrieved %d descendants of FolderItem %s (batchSize = %d)", 238 descendants.size(), docPath, batchSize)); 239 } 240 return new ScrollFileSystemItemListImpl(newScrollId, descendants); 241 } 242 } 243 244 protected void checkBatchSize(int batchSize) { 245 int maxDescendantsBatchSize = Integer.parseInt(Framework.getService(ConfigurationService.class).getProperty( 246 MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT)); 247 if (batchSize > maxDescendantsBatchSize) { 248 throw new NuxeoException(String.format( 249 "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.", 250 batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY)); 251 } 252 } 253 254 @SuppressWarnings("unchecked") 255 protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session, 256 long keepAlive) { 257 Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE); 258 if (scrollingCache == null) { 259 throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE); 260 } 261 String newScrollId; 262 List<String> descendantIds; 263 if (StringUtils.isEmpty(scrollId)) { 264 // Perform initial query to fetch ids of all the descendant documents and put the result list in a 265 // cache, aka "search context" 266 descendantIds = new ArrayList<>(); 267 StringBuilder sb = new StringBuilder( 268 String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId)); 269 sb.append(" AND ecm:currentLifeCycleState != 'deleted'"); 270 sb.append(" AND ecm:mixinType != 'HiddenInNavigation'"); 271 // Don't need to add ecm:isCheckedInVersion = 0 because versions are already excluded by the 272 // ecm:ancestorId clause since they have no path 273 String query = sb.toString(); 274 if (log.isDebugEnabled()) { 275 log.debug(String.format("Executing initial query to scroll through the descendants of %s: %s", docPath, 276 query)); 277 } 278 try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) { 279 Iterator<Map<String, Serializable>> it = res.iterator(); 280 while (it.hasNext()) { 281 descendantIds.add((String) it.next().get(NXQL.ECM_UUID)); 282 } 283 } 284 // Generate a scroll id 285 newScrollId = UUID.randomUUID().toString(); 286 if (log.isDebugEnabled()) { 287 log.debug(String.format( 288 "Put initial query result list (search context) in the %s cache at key (scrollId) %s", 289 DESCENDANTS_SCROLL_CACHE, newScrollId)); 290 } 291 scrollingCache.put(newScrollId, (Serializable) descendantIds); 292 } else { 293 // Get the descendant ids from the cache 294 descendantIds = (List<String>) scrollingCache.get(scrollId); 295 if (descendantIds == null) { 296 throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]", 297 DESCENDANTS_SCROLL_CACHE, scrollId)); 298 } 299 newScrollId = scrollId; 300 } 301 302 if (descendantIds.isEmpty()) { 303 return new ScrollDocumentModelList(newScrollId, 0); 304 } 305 306 // Extract a batch of descendant ids 307 List<String> descendantIdsBatch = getBatch(descendantIds, batchSize); 308 // Update descendant ids in the cache 309 scrollingCache.put(newScrollId, (Serializable) descendantIds); 310 // Fetch documents from VCS 311 DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session); 312 return new ScrollDocumentModelList(newScrollId, descendantDocsBatch); 313 } 314 315 /** 316 * Extracts batchSize elements from the input list. 317 */ 318 protected List<String> getBatch(List<String> ids, int batchSize) { 319 List<String> batch = new ArrayList<>(batchSize); 320 int idCount = 0; 321 Iterator<String> it = ids.iterator(); 322 while (it.hasNext() && idCount < batchSize) { 323 batch.add(it.next()); 324 it.remove(); 325 idCount++; 326 } 327 return batch; 328 } 329 330 protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) { 331 DocumentModelList res = null; 332 int size = ids.size(); 333 int start = 0; 334 int end = Math.min(VCS_CHUNK_SIZE, size); 335 boolean done = false; 336 while (!done) { 337 DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session); 338 if (res == null) { 339 res = docs; 340 } else { 341 res.addAll(docs); 342 } 343 if (end >= ids.size()) { 344 done = true; 345 } else { 346 start = end; 347 end = Math.min(start + VCS_CHUNK_SIZE, size); 348 } 349 } 350 return res; 351 } 352 353 protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) { 354 int docCount = ids.size(); 355 StringBuilder sb = new StringBuilder(); 356 sb.append("SELECT * FROM Document WHERE ecm:uuid IN ("); 357 for (int i = 0; i < docCount; i++) { 358 sb.append(NXQL.escapeString(ids.get(i))); 359 if (i < docCount - 1) { 360 sb.append(", "); 361 } 362 } 363 sb.append(")"); 364 String query = sb.toString(); 365 if (log.isDebugEnabled()) { 366 log.debug(String.format("Fetching %d documents from VCS: %s", docCount, query)); 367 } 368 return session.query(query); 369 } 370 371 /** 372 * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem} 373 * ancestors. 374 */ 375 protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) { 376 Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>(); 377 if (log.isTraceEnabled()) { 378 log.trace(String.format("Caching current FolderItem for doc %s: %s", docPath, getPath())); 379 } 380 ancestorCache.put(new IdRef(docId), this); 381 List<FileSystemItem> descendants = new ArrayList<>(docs.size()); 382 for (DocumentModel doc : docs) { 383 FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false); 384 if (parent == null) { 385 if (log.isDebugEnabled()) { 386 log.debug(String.format( 387 "Cannot adapt parent document of %s as a FileSystemItem, skipping descendant document", 388 doc.getPathAsString())); 389 continue; 390 } 391 } 392 // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo 393 FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false, 394 false); 395 if (descendant != null) { 396 if (descendant.isFolder()) { 397 if (log.isTraceEnabled()) { 398 log.trace(String.format("Caching descendant FolderItem for doc %s: %s", doc.getPathAsString(), 399 descendant.getPath())); 400 } 401 ancestorCache.put(doc.getRef(), (FolderItem) descendant); 402 } 403 descendants.add(descendant); 404 } 405 } 406 return descendants; 407 } 408 409 protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc, 410 CoreSession session, boolean cacheItem) { 411 DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef()); 412 if (parentDocRef == null) { 413 throw new RootlessItemException("Reached repository root"); 414 } 415 416 FolderItem parentItem = cache.get(parentDocRef); 417 if (parentItem != null) { 418 if (log.isTraceEnabled()) { 419 log.trace(String.format("Found parent FolderItem in cache for doc %s: %s", doc.getPathAsString(), 420 parentItem.getPath())); 421 } 422 return getFolderItem(cache, doc, parentItem, cacheItem); 423 } 424 425 if (log.isTraceEnabled()) { 426 log.trace(String.format("No parent FolderItem found in cache for doc %s, computing ancestor cache", 427 doc.getPathAsString())); 428 } 429 DocumentModel parentDoc = null; 430 try { 431 parentDoc = session.getDocument(parentDocRef); 432 } catch (DocumentSecurityException e) { 433 throw new RootlessItemException(String.format("User %s has no READ access on parent of document %s (%s).", 434 principal.getName(), doc.getPathAsString(), doc.getId()), e); 435 } 436 parentItem = populateAncestorCache(cache, parentDoc, session, true); 437 if (parentItem == null) { 438 return null; 439 } 440 return getFolderItem(cache, doc, parentItem, cacheItem); 441 } 442 443 protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem, 444 boolean cacheItem) { 445 if (cacheItem) { 446 // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo 447 FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false, 448 false); 449 if (fsItem == null) { 450 if (log.isDebugEnabled()) { 451 log.debug(String.format( 452 "Reached document %s that cannot be adapted as a (possibly virtual) descendant of the top level folder item.", 453 doc.getPathAsString())); 454 } 455 return null; 456 } 457 FolderItem folderItem = (FolderItem) fsItem; 458 if (log.isTraceEnabled()) { 459 log.trace(String.format("Caching FolderItem for doc %s: %s", doc.getPathAsString(), 460 folderItem.getPath())); 461 } 462 cache.put(doc.getRef(), folderItem); 463 return folderItem; 464 } else { 465 return parentItem; 466 } 467 } 468 469 @Override 470 public boolean getCanCreateChild() { 471 return canCreateChild; 472 } 473 474 @Override 475 public FolderItem createFolder(String name, boolean overwrite) { 476 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 477 DocumentModel folder = getFileManager().createFolder(session, name, docPath, overwrite); 478 if (folder == null) { 479 throw new NuxeoException(String.format( 480 "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.", 481 name, docPath)); 482 } 483 return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this); 484 } catch (NuxeoException e) { 485 e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath)); 486 throw e; 487 } catch (IOException e) { 488 throw new NuxeoException( 489 String.format("Error while trying to create folder %s as a child of doc %s", name, docPath), e); 490 } 491 } 492 493 @Override 494 public FileItem createFile(Blob blob, boolean overwrite) { 495 String fileName = blob.getFilename(); 496 try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) { 497 DocumentModel file = getFileManager().createDocumentFromBlob(session, blob, docPath, overwrite, fileName); 498 if (file == null) { 499 throw new NuxeoException(String.format( 500 "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.", 501 fileName, docPath)); 502 } 503 return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this); 504 } catch (NuxeoException e) { 505 e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath)); 506 throw e; 507 } catch (IOException e) { 508 throw new NuxeoException( 509 String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath), e); 510 } 511 } 512 513 /*--------------------- Protected -----------------*/ 514 protected void initialize(DocumentModel doc) { 515 this.name = docTitle; 516 this.folder = true; 517 this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE); 518 if (canCreateChild) { 519 if (Framework.getService(ConfigurationService.class) 520 .isBooleanPropertyTrue(PERMISSION_CHECK_OPTIMIZED_PROPERTY)) { 521 // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties 522 // and by default WriteProperties <=> Write <=> AddChildren 523 this.canCreateChild = canRename; 524 } else { 525 // In non optimized mode check AddChildren 526 this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN); 527 } 528 } 529 this.canScrollDescendants = true; 530 } 531 532 protected FileManager getFileManager() { 533 return Framework.getService(FileManager.class); 534 } 535 536 /*---------- Needed for JSON deserialization ----------*/ 537 protected void setCanCreateChild(boolean canCreateChild) { 538 this.canCreateChild = canCreateChild; 539 } 540 541 protected void setCanScrollDescendants(boolean canScrollDescendants) { 542 this.canScrollDescendants = canScrollDescendants; 543 } 544 545}