Source code

001/*
002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Antoine Taillefer <ataillefer@nuxeo.com>
018 */
019package org.nuxeo.drive.adapter.impl;
020
021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY;
022
023import java.io.IOException;
024import java.io.Serializable;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.Iterator;
028import java.util.List;
029import java.util.Map;
030import java.util.UUID;
031import java.util.concurrent.Semaphore;
032
033import org.apache.commons.lang3.StringUtils;
034import org.apache.logging.log4j.LogManager;
035import org.apache.logging.log4j.Logger;
036import org.nuxeo.drive.adapter.FileItem;
037import org.nuxeo.drive.adapter.FileSystemItem;
038import org.nuxeo.drive.adapter.FolderItem;
039import org.nuxeo.drive.adapter.RootlessItemException;
040import org.nuxeo.drive.adapter.ScrollFileSystemItemList;
041import org.nuxeo.drive.service.FileSystemItemAdapterService;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.CoreInstance;
044import org.nuxeo.ecm.core.api.CoreSession;
045import org.nuxeo.ecm.core.api.DocumentModel;
046import org.nuxeo.ecm.core.api.DocumentModelList;
047import org.nuxeo.ecm.core.api.DocumentRef;
048import org.nuxeo.ecm.core.api.DocumentSecurityException;
049import org.nuxeo.ecm.core.api.IdRef;
050import org.nuxeo.ecm.core.api.IterableQueryResult;
051import org.nuxeo.ecm.core.api.NuxeoException;
052import org.nuxeo.ecm.core.api.security.SecurityConstants;
053import org.nuxeo.ecm.core.cache.Cache;
054import org.nuxeo.ecm.core.cache.CacheService;
055import org.nuxeo.ecm.core.query.sql.NXQL;
056import org.nuxeo.ecm.core.schema.FacetNames;
057import org.nuxeo.ecm.platform.filemanager.api.FileImporterContext;
058import org.nuxeo.ecm.platform.filemanager.api.FileManager;
059import org.nuxeo.ecm.platform.query.api.PageProvider;
060import org.nuxeo.ecm.platform.query.api.PageProviderService;
061import org.nuxeo.runtime.api.Framework;
062import org.nuxeo.runtime.services.config.ConfigurationService;
063
064/**
065 * {@link DocumentModel} backed implementation of a {@link FolderItem}.
066 *
067 * @author Antoine Taillefer
068 */
069public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem {
070
071    private static final Logger log = LogManager.getLogger(DocumentBackedFolderItem.class);
072
073    private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN";
074
075    protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll";
076
077    protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize";
078
079    protected static final int MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = 1000;
080
081    protected static final int VCS_CHUNK_SIZE = 100;
082
083    protected boolean canCreateChild;
084
085    protected boolean canScrollDescendants;
086
087    public DocumentBackedFolderItem(String factoryName, DocumentModel doc) {
088        this(factoryName, doc, false);
089    }
090
091    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) {
092        this(factoryName, doc, relaxSyncRootConstraint, true);
093    }
094
095    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint,
096            boolean getLockInfo) {
097        super(factoryName, doc, relaxSyncRootConstraint, getLockInfo);
098        initialize(doc);
099    }
100
101    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) {
102        this(factoryName, parentItem, doc, false);
103    }
104
105    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
106            boolean relaxSyncRootConstraint) {
107        this(factoryName, parentItem, doc, relaxSyncRootConstraint, true);
108    }
109
110    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
111            boolean relaxSyncRootConstraint, boolean getLockInfo) {
112        super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo);
113        initialize(doc);
114    }
115
116    protected DocumentBackedFolderItem() {
117        // Needed for JSON deserialization
118    }
119
120    /*--------------------- FileSystemItem ---------------------*/
121    @Override
122    public void rename(String name) {
123        CoreSession session = CoreInstance.getCoreSession(repositoryName, principal);
124        // Update doc properties
125        DocumentModel doc = getDocument(session);
126        doc.setPropertyValue("dc:title", name);
127        doc.putContextData(CoreSession.SOURCE, "drive");
128        doc = session.saveDocument(doc);
129        session.save();
130        // Update FileSystemItem attributes
131        this.docTitle = name;
132        this.name = name;
133        updateLastModificationDate(doc);
134    }
135
136    /*--------------------- FolderItem -----------------*/
137    @Override
138    @SuppressWarnings("unchecked")
139    public List<FileSystemItem> getChildren() {
140        CoreSession session = CoreInstance.getCoreSession(repositoryName, principal);
141        PageProviderService pageProviderService = Framework.getService(PageProviderService.class);
142        Map<String, Serializable> props = new HashMap<>();
143        props.put(CORE_SESSION_PROPERTY, (Serializable) session);
144        PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider(
145                FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId);
146        long pageSize = childrenPageProvider.getPageSize();
147
148        List<FileSystemItem> children = new ArrayList<>();
149        int nbChildren = 0;
150        boolean reachedPageSize = false;
151        boolean hasNextPage = true;
152        // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size
153        // number of
154        // FileSystemItems
155        while (nbChildren < pageSize && hasNextPage) {
156            List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage();
157            for (DocumentModel dmChild : dmChildren) {
158                // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
159                FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false, false,
160                        false);
161                if (child != null) {
162                    children.add(child);
163                    nbChildren++;
164                    if (nbChildren == pageSize) {
165                        reachedPageSize = true;
166                        break;
167                    }
168                }
169            }
170            if (!reachedPageSize) {
171                hasNextPage = childrenPageProvider.isNextPageAvailable();
172                if (hasNextPage) {
173                    childrenPageProvider.nextPage();
174                }
175            }
176        }
177
178        return children;
179    }
180
181    @Override
182    public boolean getCanScrollDescendants() {
183        return canScrollDescendants;
184    }
185
186    @Override
187    public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) {
188        Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore();
189        try {
190            log.trace("Thread [{}] acquiring scroll batch semaphore", Thread::currentThread);
191            semaphore.acquire();
192            try {
193                log.trace("Thread [{}] acquired scroll batch semaphore, available permits reduced to {}",
194                        Thread::currentThread, semaphore::availablePermits);
195                return doScrollDescendants(scrollId, batchSize, keepAlive);
196            } finally {
197                semaphore.release();
198                log.trace("Thread [{}] released scroll batch semaphore, available permits increased to {}",
199                        Thread::currentThread, semaphore::availablePermits);
200            }
201        } catch (InterruptedException cause) {
202            Thread.currentThread().interrupt();
203            throw new NuxeoException("Scroll batch interrupted", cause);
204        }
205    }
206
207    protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) {
208        CoreSession session = CoreInstance.getCoreSession(repositoryName, principal);
209
210        // Limit batch size sent by the client
211        checkBatchSize(batchSize);
212
213        // Scroll through a batch of documents
214        ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive);
215        String newScrollId = descendantDocsBatch.getScrollId();
216        if (descendantDocsBatch.isEmpty()) {
217            // No more descendants left to return
218            return new ScrollFileSystemItemListImpl(newScrollId, 0);
219        }
220
221        // Adapt documents as FileSystemItems
222        List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session);
223        log.debug("Retrieved {} descendants of FolderItem {} (batchSize = {})", descendants::size, () -> docPath,
224                () -> batchSize);
225        return new ScrollFileSystemItemListImpl(newScrollId, descendants);
226    }
227
228    protected void checkBatchSize(int batchSize) {
229        int maxDescendantsBatchSize = Framework.getService(ConfigurationService.class).getInteger(
230                MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT);
231        if (batchSize > maxDescendantsBatchSize) {
232            throw new NuxeoException(String.format(
233                    "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.",
234                    batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY));
235        }
236    }
237
238    @SuppressWarnings("unchecked")
239    protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session,
240            long keepAlive) { // NOSONAR
241        Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE);
242        if (scrollingCache == null) {
243            throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE);
244        }
245        String newScrollId;
246        List<String> descendantIds;
247        if (StringUtils.isEmpty(scrollId)) {
248            // Perform initial query to fetch ids of all the descendant documents and put the result list in a
249            // cache, aka "search context"
250            descendantIds = new ArrayList<>();
251            StringBuilder sb = new StringBuilder(
252                    String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId));
253            sb.append(" AND ecm:isTrashed = 0");
254            sb.append(" AND ecm:mixinType != 'HiddenInNavigation'");
255            // Don't need to add ecm:isVersion = 0 because versions are already excluded by the
256            // ecm:ancestorId clause since they have no path
257            String query = sb.toString();
258            log.debug("Executing initial query to scroll through the descendants of {}: {}", docPath, query);
259            try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) {
260                Iterator<Map<String, Serializable>> it = res.iterator();
261                while (it.hasNext()) {
262                    descendantIds.add((String) it.next().get(NXQL.ECM_UUID));
263                }
264            }
265            // Generate a scroll id
266            newScrollId = UUID.randomUUID().toString();
267            log.debug("Put initial query result list (search context) in the {} cache at key (scrollId) {}",
268                    DESCENDANTS_SCROLL_CACHE, newScrollId);
269            scrollingCache.put(newScrollId, (Serializable) descendantIds);
270        } else {
271            // Get the descendant ids from the cache
272            descendantIds = (List<String>) scrollingCache.get(scrollId);
273            if (descendantIds == null) {
274                throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]",
275                        DESCENDANTS_SCROLL_CACHE, scrollId));
276            }
277            newScrollId = scrollId;
278        }
279
280        if (descendantIds.isEmpty()) {
281            return new ScrollDocumentModelList(newScrollId, 0);
282        }
283
284        // Extract a batch of descendant ids
285        List<String> descendantIdsBatch = getBatch(descendantIds, batchSize);
286        // Update descendant ids in the cache
287        scrollingCache.put(newScrollId, (Serializable) descendantIds);
288        // Fetch documents from VCS
289        DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session);
290        return new ScrollDocumentModelList(newScrollId, descendantDocsBatch);
291    }
292
293    /**
294     * Extracts batchSize elements from the input list.
295     */
296    protected List<String> getBatch(List<String> ids, int batchSize) {
297        List<String> batch = new ArrayList<>(batchSize);
298        int idCount = 0;
299        Iterator<String> it = ids.iterator();
300        while (it.hasNext() && idCount < batchSize) {
301            batch.add(it.next());
302            it.remove();
303            idCount++;
304        }
305        return batch;
306    }
307
308    protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) {
309        DocumentModelList res = null;
310        int size = ids.size();
311        int start = 0;
312        int end = Math.min(VCS_CHUNK_SIZE, size);
313        boolean done = false;
314        while (!done) {
315            DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session);
316            if (res == null) {
317                res = docs;
318            } else {
319                res.addAll(docs);
320            }
321            if (end >= ids.size()) {
322                done = true;
323            } else {
324                start = end;
325                end = Math.min(start + VCS_CHUNK_SIZE, size);
326            }
327        }
328        return res;
329    }
330
331    protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) {
332        int docCount = ids.size();
333        StringBuilder sb = new StringBuilder();
334        sb.append("SELECT * FROM Document WHERE ecm:uuid IN (");
335        for (int i = 0; i < docCount; i++) {
336            sb.append(NXQL.escapeString(ids.get(i)));
337            if (i < docCount - 1) {
338                sb.append(", ");
339            }
340        }
341        sb.append(")");
342        String query = sb.toString();
343        log.debug("Fetching {} documents from VCS: {}", docCount, query);
344        return session.query(query);
345    }
346
347    /**
348     * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem}
349     * ancestors.
350     */
351    protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) {
352        Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>();
353        log.trace("Caching current FolderItem for doc {}: {}", () -> docPath, this::getPath);
354        ancestorCache.put(new IdRef(docId), this);
355        List<FileSystemItem> descendants = new ArrayList<>(docs.size());
356        for (DocumentModel doc : docs) {
357            FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false);
358            if (parent == null) {
359                log.debug("Cannot adapt parent document of {} as a FileSystemItem, skipping descendant document",
360                        doc::getPathAsString);
361                continue;
362            }
363            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
364            FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false,
365                    false);
366            if (descendant != null) {
367                if (descendant.isFolder()) {
368                    log.trace("Caching descendant FolderItem for doc {}: {}", doc::getPathAsString,
369                            descendant::getPath);
370                    ancestorCache.put(doc.getRef(), (FolderItem) descendant);
371                }
372                descendants.add(descendant);
373            }
374        }
375        return descendants;
376    }
377
378    protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc,
379            CoreSession session, boolean cacheItem) {
380        DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef());
381        if (parentDocRef == null) {
382            throw new RootlessItemException("Reached repository root");
383        }
384
385        FolderItem parentItem = cache.get(parentDocRef);
386        if (parentItem != null) {
387            log.trace("Found parent FolderItem in cache for doc {}: {}", doc::getPathAsString, parentItem::getPath);
388            return getFolderItem(cache, doc, parentItem, cacheItem);
389        }
390
391        log.trace("No parent FolderItem found in cache for doc {}, computing ancestor cache", doc::getPathAsString);
392        DocumentModel parentDoc = null;
393        try {
394            parentDoc = session.getDocument(parentDocRef);
395        } catch (DocumentSecurityException e) {
396            log.debug("User {} has no READ access on parent of document {} ({}).", principal::getName,
397                    doc::getPathAsString, doc::getId, () -> e);
398            return null;
399        }
400        parentItem = populateAncestorCache(cache, parentDoc, session, true);
401        if (parentItem == null) {
402            return null;
403        }
404        return getFolderItem(cache, doc, parentItem, cacheItem);
405    }
406
407    protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem,
408            boolean cacheItem) {
409        if (cacheItem) {
410            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
411            FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false,
412                    false);
413            if (fsItem == null) {
414                log.debug(
415                        "Reached document {} that cannot be  adapted as a (possibly virtual) descendant of the top level folder item.",
416                        doc::getPathAsString);
417                return null;
418            }
419            FolderItem folderItem = (FolderItem) fsItem;
420            log.trace("Caching FolderItem for doc {}: {}", doc::getPathAsString, folderItem::getPath);
421            cache.put(doc.getRef(), folderItem);
422            return folderItem;
423        } else {
424            return parentItem;
425        }
426    }
427
428    @Override
429    public boolean getCanCreateChild() {
430        return canCreateChild;
431    }
432
433    @Override
434    public FolderItem createFolder(String name, boolean overwrite) {
435        try {
436            CoreSession session = CoreInstance.getCoreSession(repositoryName, principal);
437            DocumentModel folder = getFileManager().createFolder(session, name, docPath, overwrite);
438            if (folder == null) {
439                throw new NuxeoException(String.format(
440                        "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.",
441                        name, docPath));
442            }
443            return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this);
444        } catch (NuxeoException e) {
445            e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath));
446            throw e;
447        } catch (IOException e) {
448            throw new NuxeoException(
449                    String.format("Error while trying to create folder %s as a child of doc %s", name, docPath), e);
450        }
451    }
452
453    @Override
454    public FileItem createFile(Blob blob, boolean overwrite) {
455        String fileName = blob.getFilename();
456        try {
457            CoreSession session = CoreInstance.getCoreSession(repositoryName, principal);
458            FileImporterContext context = FileImporterContext.builder(session, blob, docPath)
459                                                             .overwrite(overwrite)
460                                                             .excludeOneToMany(true)
461                                                             .build();
462            DocumentModel file = getFileManager().createOrUpdateDocument(context);
463            if (file == null) {
464                throw new NuxeoException(String.format(
465                        "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.",
466                        fileName, docPath));
467            }
468            return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this);
469        } catch (NuxeoException e) {
470            e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath));
471            throw e;
472        } catch (IOException e) {
473            throw new NuxeoException(
474                    String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath), e);
475        }
476    }
477
478    /*--------------------- Object -----------------*/
479    // Override equals and hashCode to explicitly show that their implementation rely on the parent class and doesn't
480    // depend on the fields added to this class.
481    @Override
482    public boolean equals(Object obj) {
483        return super.equals(obj);
484    }
485
486    @Override
487    public int hashCode() {
488        return super.hashCode();
489    }
490
491    /*--------------------- Protected -----------------*/
492    protected void initialize(DocumentModel doc) {
493        this.name = docTitle;
494        this.folder = true;
495        this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE);
496        if (canCreateChild) {
497            if (Framework.getService(ConfigurationService.class).isBooleanTrue(PERMISSION_CHECK_OPTIMIZED_PROPERTY)) {
498                // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties
499                // and by default WriteProperties <=> Write <=> AddChildren
500                this.canCreateChild = canRename;
501            } else {
502                // In non optimized mode check AddChildren
503                this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN);
504            }
505        }
506        this.canScrollDescendants = true;
507    }
508
509    protected FileManager getFileManager() {
510        return Framework.getService(FileManager.class);
511    }
512
513    /*---------- Needed for JSON deserialization ----------*/
514    protected void setCanCreateChild(boolean canCreateChild) {
515        this.canCreateChild = canCreateChild;
516    }
517
518    protected void setCanScrollDescendants(boolean canScrollDescendants) {
519        this.canScrollDescendants = canScrollDescendants;
520    }
521
522}