Source code

001/*
002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Antoine Taillefer <ataillefer@nuxeo.com>
018 */
019package org.nuxeo.drive.adapter.impl;
020
021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY;
022
023import java.io.IOException;
024import java.io.Serializable;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.Iterator;
028import java.util.List;
029import java.util.Map;
030import java.util.UUID;
031import java.util.concurrent.Semaphore;
032
033import org.apache.commons.lang3.StringUtils;
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.nuxeo.drive.adapter.FileItem;
037import org.nuxeo.drive.adapter.FileSystemItem;
038import org.nuxeo.drive.adapter.FolderItem;
039import org.nuxeo.drive.adapter.RootlessItemException;
040import org.nuxeo.drive.adapter.ScrollFileSystemItemList;
041import org.nuxeo.drive.service.FileSystemItemAdapterService;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.CloseableCoreSession;
044import org.nuxeo.ecm.core.api.CoreInstance;
045import org.nuxeo.ecm.core.api.CoreSession;
046import org.nuxeo.ecm.core.api.DocumentModel;
047import org.nuxeo.ecm.core.api.DocumentModelList;
048import org.nuxeo.ecm.core.api.DocumentRef;
049import org.nuxeo.ecm.core.api.DocumentSecurityException;
050import org.nuxeo.ecm.core.api.IdRef;
051import org.nuxeo.ecm.core.api.IterableQueryResult;
052import org.nuxeo.ecm.core.api.NuxeoException;
053import org.nuxeo.ecm.core.api.security.SecurityConstants;
054import org.nuxeo.ecm.core.cache.Cache;
055import org.nuxeo.ecm.core.cache.CacheService;
056import org.nuxeo.ecm.core.query.sql.NXQL;
057import org.nuxeo.ecm.core.schema.FacetNames;
058import org.nuxeo.ecm.platform.filemanager.api.FileManager;
059import org.nuxeo.ecm.platform.query.api.PageProvider;
060import org.nuxeo.ecm.platform.query.api.PageProviderService;
061import org.nuxeo.runtime.api.Framework;
062import org.nuxeo.runtime.services.config.ConfigurationService;
063
064/**
065 * {@link DocumentModel} backed implementation of a {@link FolderItem}.
066 *
067 * @author Antoine Taillefer
068 */
069public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem {
070
071    private static final Log log = LogFactory.getLog(DocumentBackedFolderItem.class);
072
073    private static final long serialVersionUID = 1L;
074
075    private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN";
076
077    protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll";
078
079    protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize";
080
081    protected static final String MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = "1000";
082
083    protected static final int VCS_CHUNK_SIZE = 100;
084
085    protected boolean canCreateChild;
086
087    protected boolean canScrollDescendants;
088
089    public DocumentBackedFolderItem(String factoryName, DocumentModel doc) {
090        this(factoryName, doc, false);
091    }
092
093    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) {
094        this(factoryName, doc, relaxSyncRootConstraint, true);
095    }
096
097    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint,
098            boolean getLockInfo) {
099        super(factoryName, doc, relaxSyncRootConstraint, getLockInfo);
100        initialize(doc);
101    }
102
103    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) {
104        this(factoryName, parentItem, doc, false);
105    }
106
107    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
108            boolean relaxSyncRootConstraint) {
109        this(factoryName, parentItem, doc, relaxSyncRootConstraint, true);
110    }
111
112    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
113            boolean relaxSyncRootConstraint, boolean getLockInfo) {
114        super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo);
115        initialize(doc);
116    }
117
118    protected DocumentBackedFolderItem() {
119        // Needed for JSON deserialization
120    }
121
122    /*--------------------- FileSystemItem ---------------------*/
123    @Override
124    public void rename(String name) {
125        try (CloseableCoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
126            // Update doc properties
127            DocumentModel doc = getDocument(session);
128            doc.setPropertyValue("dc:title", name);
129            doc.putContextData(CoreSession.SOURCE, "drive");
130            doc = session.saveDocument(doc);
131            session.save();
132            // Update FileSystemItem attributes
133            this.docTitle = name;
134            this.name = name;
135            updateLastModificationDate(doc);
136        }
137    }
138
139    /*--------------------- FolderItem -----------------*/
140    @Override
141    @SuppressWarnings("unchecked")
142    public List<FileSystemItem> getChildren() {
143        try (CloseableCoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
144            PageProviderService pageProviderService = Framework.getService(PageProviderService.class);
145            Map<String, Serializable> props = new HashMap<>();
146            props.put(CORE_SESSION_PROPERTY, (Serializable) session);
147            PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider(
148                    FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId);
149            long pageSize = childrenPageProvider.getPageSize();
150
151            List<FileSystemItem> children = new ArrayList<>();
152            int nbChildren = 0;
153            boolean reachedPageSize = false;
154            boolean hasNextPage = true;
155            // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size
156            // number of
157            // FileSystemItems
158            while (nbChildren < pageSize && hasNextPage) {
159                List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage();
160                for (DocumentModel dmChild : dmChildren) {
161                    // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
162                    FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false,
163                            false, false);
164                    if (child != null) {
165                        children.add(child);
166                        nbChildren++;
167                        if (nbChildren == pageSize) {
168                            reachedPageSize = true;
169                            break;
170                        }
171                    }
172                }
173                if (!reachedPageSize) {
174                    hasNextPage = childrenPageProvider.isNextPageAvailable();
175                    if (hasNextPage) {
176                        childrenPageProvider.nextPage();
177                    }
178                }
179            }
180
181            return children;
182        }
183    }
184
185    @Override
186    public boolean getCanScrollDescendants() {
187        return canScrollDescendants;
188    }
189
190    @Override
191    public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) {
192        Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore();
193        try {
194            if (log.isTraceEnabled()) {
195                log.trace(String.format("Thread [%s] acquiring scroll batch semaphore",
196                        Thread.currentThread().getName()));
197            }
198            semaphore.acquire();
199            try {
200                if (log.isTraceEnabled()) {
201                    log.trace(String.format(
202                            "Thread [%s] acquired scroll batch semaphore, available permits reduced to %d",
203                            Thread.currentThread().getName(), semaphore.availablePermits()));
204                }
205                return doScrollDescendants(scrollId, batchSize, keepAlive);
206            } finally {
207                semaphore.release();
208                if (log.isTraceEnabled()) {
209                    log.trace(String.format(
210                            "Thread [%s] released scroll batch semaphore, available permits increased to %d",
211                            Thread.currentThread().getName(), semaphore.availablePermits()));
212                }
213            }
214        } catch (InterruptedException cause) {
215            Thread.currentThread().interrupt();
216            throw new NuxeoException("Scroll batch interrupted", cause);
217        }
218    }
219
220    @SuppressWarnings("unchecked")
221    protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) {
222        try (CloseableCoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
223
224            // Limit batch size sent by the client
225            checkBatchSize(batchSize);
226
227            // Scroll through a batch of documents
228            ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive);
229            String newScrollId = descendantDocsBatch.getScrollId();
230            if (descendantDocsBatch.isEmpty()) {
231                // No more descendants left to return
232                return new ScrollFileSystemItemListImpl(newScrollId, 0);
233            }
234
235            // Adapt documents as FileSystemItems
236            List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session);
237            if (log.isDebugEnabled()) {
238                log.debug(String.format("Retrieved %d descendants of FolderItem %s (batchSize = %d)",
239                        descendants.size(), docPath, batchSize));
240            }
241            return new ScrollFileSystemItemListImpl(newScrollId, descendants);
242        }
243    }
244
245    protected void checkBatchSize(int batchSize) {
246        int maxDescendantsBatchSize = Integer.parseInt(Framework.getService(ConfigurationService.class).getProperty(
247                MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT));
248        if (batchSize > maxDescendantsBatchSize) {
249            throw new NuxeoException(String.format(
250                    "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.",
251                    batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY));
252        }
253    }
254
255    @SuppressWarnings("unchecked")
256    protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session,
257            long keepAlive) {
258        Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE);
259        if (scrollingCache == null) {
260            throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE);
261        }
262        String newScrollId;
263        List<String> descendantIds;
264        if (StringUtils.isEmpty(scrollId)) {
265            // Perform initial query to fetch ids of all the descendant documents and put the result list in a
266            // cache, aka "search context"
267            descendantIds = new ArrayList<>();
268            StringBuilder sb = new StringBuilder(
269                    String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId));
270            sb.append(" AND ecm:isTrashed = 0");
271            sb.append(" AND ecm:mixinType != 'HiddenInNavigation'");
272            // Don't need to add ecm:isVersion = 0 because versions are already excluded by the
273            // ecm:ancestorId clause since they have no path
274            String query = sb.toString();
275            if (log.isDebugEnabled()) {
276                log.debug(String.format("Executing initial query to scroll through the descendants of %s: %s", docPath,
277                        query));
278            }
279            try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) {
280                Iterator<Map<String, Serializable>> it = res.iterator();
281                while (it.hasNext()) {
282                    descendantIds.add((String) it.next().get(NXQL.ECM_UUID));
283                }
284            }
285            // Generate a scroll id
286            newScrollId = UUID.randomUUID().toString();
287            if (log.isDebugEnabled()) {
288                log.debug(String.format(
289                        "Put initial query result list (search context) in the %s cache at key (scrollId) %s",
290                        DESCENDANTS_SCROLL_CACHE, newScrollId));
291            }
292            scrollingCache.put(newScrollId, (Serializable) descendantIds);
293        } else {
294            // Get the descendant ids from the cache
295            descendantIds = (List<String>) scrollingCache.get(scrollId);
296            if (descendantIds == null) {
297                throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]",
298                        DESCENDANTS_SCROLL_CACHE, scrollId));
299            }
300            newScrollId = scrollId;
301        }
302
303        if (descendantIds.isEmpty()) {
304            return new ScrollDocumentModelList(newScrollId, 0);
305        }
306
307        // Extract a batch of descendant ids
308        List<String> descendantIdsBatch = getBatch(descendantIds, batchSize);
309        // Update descendant ids in the cache
310        scrollingCache.put(newScrollId, (Serializable) descendantIds);
311        // Fetch documents from VCS
312        DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session);
313        return new ScrollDocumentModelList(newScrollId, descendantDocsBatch);
314    }
315
316    /**
317     * Extracts batchSize elements from the input list.
318     */
319    protected List<String> getBatch(List<String> ids, int batchSize) {
320        List<String> batch = new ArrayList<>(batchSize);
321        int idCount = 0;
322        Iterator<String> it = ids.iterator();
323        while (it.hasNext() && idCount < batchSize) {
324            batch.add(it.next());
325            it.remove();
326            idCount++;
327        }
328        return batch;
329    }
330
331    protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) {
332        DocumentModelList res = null;
333        int size = ids.size();
334        int start = 0;
335        int end = Math.min(VCS_CHUNK_SIZE, size);
336        boolean done = false;
337        while (!done) {
338            DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session);
339            if (res == null) {
340                res = docs;
341            } else {
342                res.addAll(docs);
343            }
344            if (end >= ids.size()) {
345                done = true;
346            } else {
347                start = end;
348                end = Math.min(start + VCS_CHUNK_SIZE, size);
349            }
350        }
351        return res;
352    }
353
354    protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) {
355        int docCount = ids.size();
356        StringBuilder sb = new StringBuilder();
357        sb.append("SELECT * FROM Document WHERE ecm:uuid IN (");
358        for (int i = 0; i < docCount; i++) {
359            sb.append(NXQL.escapeString(ids.get(i)));
360            if (i < docCount - 1) {
361                sb.append(", ");
362            }
363        }
364        sb.append(")");
365        String query = sb.toString();
366        if (log.isDebugEnabled()) {
367            log.debug(String.format("Fetching %d documents from VCS: %s", docCount, query));
368        }
369        return session.query(query);
370    }
371
372    /**
373     * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem}
374     * ancestors.
375     */
376    protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) {
377        Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>();
378        if (log.isTraceEnabled()) {
379            log.trace(String.format("Caching current FolderItem for doc %s: %s", docPath, getPath()));
380        }
381        ancestorCache.put(new IdRef(docId), this);
382        List<FileSystemItem> descendants = new ArrayList<>(docs.size());
383        for (DocumentModel doc : docs) {
384            FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false);
385            if (parent == null) {
386                if (log.isDebugEnabled()) {
387                    log.debug(String.format(
388                            "Cannot adapt parent document of %s as a FileSystemItem, skipping descendant document",
389                            doc.getPathAsString()));
390                    continue;
391                }
392            }
393            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
394            FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false,
395                    false);
396            if (descendant != null) {
397                if (descendant.isFolder()) {
398                    if (log.isTraceEnabled()) {
399                        log.trace(String.format("Caching descendant FolderItem for doc %s: %s", doc.getPathAsString(),
400                                descendant.getPath()));
401                    }
402                    ancestorCache.put(doc.getRef(), (FolderItem) descendant);
403                }
404                descendants.add(descendant);
405            }
406        }
407        return descendants;
408    }
409
410    protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc,
411            CoreSession session, boolean cacheItem) {
412        DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef());
413        if (parentDocRef == null) {
414            throw new RootlessItemException("Reached repository root");
415        }
416
417        FolderItem parentItem = cache.get(parentDocRef);
418        if (parentItem != null) {
419            if (log.isTraceEnabled()) {
420                log.trace(String.format("Found parent FolderItem in cache for doc %s: %s", doc.getPathAsString(),
421                        parentItem.getPath()));
422            }
423            return getFolderItem(cache, doc, parentItem, cacheItem);
424        }
425
426        if (log.isTraceEnabled()) {
427            log.trace(String.format("No parent FolderItem found in cache for doc %s, computing ancestor cache",
428                    doc.getPathAsString()));
429        }
430        DocumentModel parentDoc;
431        try {
432            parentDoc = session.getDocument(parentDocRef);
433        } catch (DocumentSecurityException e) {
434            throw new RootlessItemException(String.format("User %s has no READ access on parent of document %s (%s).",
435                    principal.getName(), doc.getPathAsString(), doc.getId()), e);
436        }
437        parentItem = populateAncestorCache(cache, parentDoc, session, true);
438        if (parentItem == null) {
439            return null;
440        }
441        return getFolderItem(cache, doc, parentItem, cacheItem);
442    }
443
444    protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem,
445            boolean cacheItem) {
446        if (cacheItem) {
447            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
448            FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false,
449                    false);
450            if (fsItem == null) {
451                if (log.isDebugEnabled()) {
452                    log.debug(String.format(
453                            "Reached document %s that cannot be  adapted as a (possibly virtual) descendant of the top level folder item.",
454                            doc.getPathAsString()));
455                }
456                return null;
457            }
458            FolderItem folderItem = (FolderItem) fsItem;
459            if (log.isTraceEnabled()) {
460                log.trace(String.format("Caching FolderItem for doc %s: %s", doc.getPathAsString(),
461                        folderItem.getPath()));
462            }
463            cache.put(doc.getRef(), folderItem);
464            return folderItem;
465        } else {
466            return parentItem;
467        }
468    }
469
470    @Override
471    public boolean getCanCreateChild() {
472        return canCreateChild;
473    }
474
475    @Override
476    public FolderItem createFolder(String name, boolean overwrite) {
477        try (CloseableCoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
478            DocumentModel folder = getFileManager().createFolder(session, name, docPath, overwrite);
479            if (folder == null) {
480                throw new NuxeoException(String.format(
481                        "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.",
482                        name, docPath));
483            }
484            return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this);
485        } catch (NuxeoException e) {
486            e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath));
487            throw e;
488        } catch (IOException e) {
489            throw new NuxeoException(
490                    String.format("Error while trying to create folder %s as a child of doc %s", name, docPath), e);
491        }
492    }
493
494    @Override
495    public FileItem createFile(Blob blob, boolean overwrite) {
496        String fileName = blob.getFilename();
497        try (CloseableCoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
498            DocumentModel file = getFileManager().createDocumentFromBlob(session, blob, docPath, overwrite, fileName);
499            if (file == null) {
500                throw new NuxeoException(String.format(
501                        "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.",
502                        fileName, docPath));
503            }
504            return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this);
505        } catch (NuxeoException e) {
506            e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath));
507            throw e;
508        } catch (IOException e) {
509            throw new NuxeoException(
510                    String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath), e);
511        }
512    }
513
514    /*--------------------- Protected -----------------*/
515    protected void initialize(DocumentModel doc) {
516        this.name = docTitle;
517        this.folder = true;
518        this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE);
519        if (canCreateChild) {
520            if (Framework.getService(ConfigurationService.class)
521                         .isBooleanPropertyTrue(PERMISSION_CHECK_OPTIMIZED_PROPERTY)) {
522                // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties
523                // and by default WriteProperties <=> Write <=> AddChildren
524                this.canCreateChild = canRename;
525            } else {
526                // In non optimized mode check AddChildren
527                this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN);
528            }
529        }
530        this.canScrollDescendants = true;
531    }
532
533    protected FileManager getFileManager() {
534        return Framework.getService(FileManager.class);
535    }
536
537    /*---------- Needed for JSON deserialization ----------*/
538    protected void setCanCreateChild(boolean canCreateChild) {
539        this.canCreateChild = canCreateChild;
540    }
541
542    protected void setCanScrollDescendants(boolean canScrollDescendants) {
543        this.canScrollDescendants = canScrollDescendants;
544    }
545
546}