Source code

001/*
002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Antoine Taillefer <ataillefer@nuxeo.com>
018 */
019package org.nuxeo.drive.adapter.impl;
020
021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY;
022
023import java.io.IOException;
024import java.io.Serializable;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.Iterator;
028import java.util.List;
029import java.util.Map;
030import java.util.UUID;
031import java.util.concurrent.Semaphore;
032
033import org.apache.commons.lang.StringUtils;
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.nuxeo.drive.adapter.FileItem;
037import org.nuxeo.drive.adapter.FileSystemItem;
038import org.nuxeo.drive.adapter.FolderItem;
039import org.nuxeo.drive.adapter.RootlessItemException;
040import org.nuxeo.drive.adapter.ScrollFileSystemItemList;
041import org.nuxeo.drive.service.FileSystemItemAdapterService;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.CoreInstance;
044import org.nuxeo.ecm.core.api.CoreSession;
045import org.nuxeo.ecm.core.api.DocumentModel;
046import org.nuxeo.ecm.core.api.DocumentModelList;
047import org.nuxeo.ecm.core.api.DocumentRef;
048import org.nuxeo.ecm.core.api.DocumentSecurityException;
049import org.nuxeo.ecm.core.api.IdRef;
050import org.nuxeo.ecm.core.api.IterableQueryResult;
051import org.nuxeo.ecm.core.api.NuxeoException;
052import org.nuxeo.ecm.core.api.security.SecurityConstants;
053import org.nuxeo.ecm.core.cache.Cache;
054import org.nuxeo.ecm.core.cache.CacheService;
055import org.nuxeo.ecm.core.query.sql.NXQL;
056import org.nuxeo.ecm.core.schema.FacetNames;
057import org.nuxeo.ecm.platform.filemanager.api.FileManager;
058import org.nuxeo.ecm.platform.query.api.PageProvider;
059import org.nuxeo.ecm.platform.query.api.PageProviderService;
060import org.nuxeo.runtime.api.Framework;
061import org.nuxeo.runtime.services.config.ConfigurationService;
062
063/**
064 * {@link DocumentModel} backed implementation of a {@link FolderItem}.
065 *
066 * @author Antoine Taillefer
067 */
068public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem {
069
070    private static final Log log = LogFactory.getLog(DocumentBackedFolderItem.class);
071
072    private static final long serialVersionUID = 1L;
073
074    private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN";
075
076    protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll";
077
078    protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize";
079
080    protected static final String MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = "1000";
081
082    protected static final int VCS_CHUNK_SIZE = 100;
083
084    protected boolean canCreateChild;
085
086    protected boolean canScrollDescendants;
087
088    public DocumentBackedFolderItem(String factoryName, DocumentModel doc) {
089        this(factoryName, doc, false);
090    }
091
092    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) {
093        this(factoryName, doc, relaxSyncRootConstraint, true);
094    }
095
096    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint,
097            boolean getLockInfo) {
098        super(factoryName, doc, relaxSyncRootConstraint, getLockInfo);
099        initialize(doc);
100    }
101
102    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) {
103        this(factoryName, parentItem, doc, false);
104    }
105
106    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
107            boolean relaxSyncRootConstraint) {
108        this(factoryName, parentItem, doc, relaxSyncRootConstraint, true);
109    }
110
111    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
112            boolean relaxSyncRootConstraint, boolean getLockInfo) {
113        super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo);
114        initialize(doc);
115    }
116
117    protected DocumentBackedFolderItem() {
118        // Needed for JSON deserialization
119    }
120
121    /*--------------------- FileSystemItem ---------------------*/
122    @Override
123    public void rename(String name) {
124        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
125            // Update doc properties
126            DocumentModel doc = getDocument(session);
127            doc.setPropertyValue("dc:title", name);
128            doc.putContextData(CoreSession.SOURCE, "drive");
129            doc = session.saveDocument(doc);
130            session.save();
131            // Update FileSystemItem attributes
132            this.docTitle = name;
133            this.name = name;
134            updateLastModificationDate(doc);
135        }
136    }
137
138    /*--------------------- FolderItem -----------------*/
139    @Override
140    @SuppressWarnings("unchecked")
141    public List<FileSystemItem> getChildren() {
142        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
143            PageProviderService pageProviderService = Framework.getLocalService(PageProviderService.class);
144            Map<String, Serializable> props = new HashMap<String, Serializable>();
145            props.put(CORE_SESSION_PROPERTY, (Serializable) session);
146            PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider(
147                    FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId);
148            Long pageSize = childrenPageProvider.getPageSize();
149
150            List<FileSystemItem> children = new ArrayList<FileSystemItem>();
151            int nbChildren = 0;
152            boolean reachedPageSize = false;
153            boolean hasNextPage = true;
154            // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size
155            // number of
156            // FileSystemItems
157            while (nbChildren < pageSize && hasNextPage) {
158                List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage();
159                for (DocumentModel dmChild : dmChildren) {
160                    // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
161                    FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false,
162                            false, false);
163                    if (child != null) {
164                        children.add(child);
165                        nbChildren++;
166                        if (nbChildren == pageSize) {
167                            reachedPageSize = true;
168                            break;
169                        }
170                    }
171                }
172                if (!reachedPageSize) {
173                    hasNextPage = childrenPageProvider.isNextPageAvailable();
174                    if (hasNextPage) {
175                        childrenPageProvider.nextPage();
176                    }
177                }
178            }
179
180            return children;
181        }
182    }
183
184    @Override
185    public boolean getCanScrollDescendants() {
186        return canScrollDescendants;
187    }
188
189    @Override
190    public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) {
191        Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore();
192        try {
193            if (log.isTraceEnabled()) {
194                log.trace(String.format("Thread [%s] acquiring scroll batch semaphore",
195                        Thread.currentThread().getName()));
196            }
197            semaphore.acquire();
198            try {
199                if (log.isTraceEnabled()) {
200                    log.trace(String.format(
201                            "Thread [%s] acquired scroll batch semaphore, available permits reduced to %d",
202                            Thread.currentThread().getName(), semaphore.availablePermits()));
203                }
204                return doScrollDescendants(scrollId, batchSize, keepAlive);
205            } finally {
206                semaphore.release();
207                if (log.isTraceEnabled()) {
208                    log.trace(String.format(
209                            "Thread [%s] released scroll batch semaphore, available permits increased to %d",
210                            Thread.currentThread().getName(), semaphore.availablePermits()));
211                }
212            }
213        } catch (InterruptedException cause) {
214            Thread.currentThread().interrupt();
215            throw new NuxeoException("Scroll batch interrupted", cause);
216        }
217    }
218
219    @SuppressWarnings("unchecked")
220    protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) {
221        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
222
223            // Limit batch size sent by the client
224            checkBatchSize(batchSize);
225
226            // Scroll through a batch of documents
227            ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive);
228            String newScrollId = descendantDocsBatch.getScrollId();
229            if (descendantDocsBatch.isEmpty()) {
230                // No more descendants left to return
231                return new ScrollFileSystemItemListImpl(newScrollId, 0);
232            }
233
234            // Adapt documents as FileSystemItems
235            List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session);
236            if (log.isDebugEnabled()) {
237                log.debug(String.format("Retrieved %d descendants of FolderItem %s (batchSize = %d)",
238                        descendants.size(), docPath, batchSize));
239            }
240            return new ScrollFileSystemItemListImpl(newScrollId, descendants);
241        }
242    }
243
244    protected void checkBatchSize(int batchSize) {
245        int maxDescendantsBatchSize = Integer.parseInt(Framework.getService(ConfigurationService.class).getProperty(
246                MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT));
247        if (batchSize > maxDescendantsBatchSize) {
248            throw new NuxeoException(String.format(
249                    "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.",
250                    batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY));
251        }
252    }
253
254    @SuppressWarnings("unchecked")
255    protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session,
256            long keepAlive) {
257        Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE);
258        if (scrollingCache == null) {
259            throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE);
260        }
261        String newScrollId;
262        List<String> descendantIds;
263        if (StringUtils.isEmpty(scrollId)) {
264            // Perform initial query to fetch ids of all the descendant documents and put the result list in a
265            // cache, aka "search context"
266            descendantIds = new ArrayList<>();
267            StringBuilder sb = new StringBuilder(
268                    String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId));
269            sb.append(" AND ecm:currentLifeCycleState != 'deleted'");
270            sb.append(" AND ecm:mixinType != 'HiddenInNavigation'");
271            // Don't need to add ecm:isCheckedInVersion = 0 because versions are already excluded by the
272            // ecm:ancestorId clause since they have no path
273            String query = sb.toString();
274            if (log.isDebugEnabled()) {
275                log.debug(String.format("Executing initial query to scroll through the descendants of %s: %s", docPath,
276                        query));
277            }
278            try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) {
279                Iterator<Map<String, Serializable>> it = res.iterator();
280                while (it.hasNext()) {
281                    descendantIds.add((String) it.next().get(NXQL.ECM_UUID));
282                }
283            }
284            // Generate a scroll id
285            newScrollId = UUID.randomUUID().toString();
286            if (log.isDebugEnabled()) {
287                log.debug(String.format(
288                        "Put initial query result list (search context) in the %s cache at key (scrollId) %s",
289                        DESCENDANTS_SCROLL_CACHE, newScrollId));
290            }
291            scrollingCache.put(newScrollId, (Serializable) descendantIds);
292        } else {
293            // Get the descendant ids from the cache
294            descendantIds = (List<String>) scrollingCache.get(scrollId);
295            if (descendantIds == null) {
296                throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]",
297                        DESCENDANTS_SCROLL_CACHE, scrollId));
298            }
299            newScrollId = scrollId;
300        }
301
302        if (descendantIds.isEmpty()) {
303            return new ScrollDocumentModelList(newScrollId, 0);
304        }
305
306        // Extract a batch of descendant ids
307        List<String> descendantIdsBatch = getBatch(descendantIds, batchSize);
308        // Update descendant ids in the cache
309        scrollingCache.put(newScrollId, (Serializable) descendantIds);
310        // Fetch documents from VCS
311        DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session);
312        return new ScrollDocumentModelList(newScrollId, descendantDocsBatch);
313    }
314
315    /**
316     * Extracts batchSize elements from the input list.
317     */
318    protected List<String> getBatch(List<String> ids, int batchSize) {
319        List<String> batch = new ArrayList<>(batchSize);
320        int idCount = 0;
321        Iterator<String> it = ids.iterator();
322        while (it.hasNext() && idCount < batchSize) {
323            batch.add(it.next());
324            it.remove();
325            idCount++;
326        }
327        return batch;
328    }
329
330    protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) {
331        DocumentModelList res = null;
332        int size = ids.size();
333        int start = 0;
334        int end = Math.min(VCS_CHUNK_SIZE, size);
335        boolean done = false;
336        while (!done) {
337            DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session);
338            if (res == null) {
339                res = docs;
340            } else {
341                res.addAll(docs);
342            }
343            if (end >= ids.size()) {
344                done = true;
345            } else {
346                start = end;
347                end = Math.min(start + VCS_CHUNK_SIZE, size);
348            }
349        }
350        return res;
351    }
352
353    protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) {
354        int docCount = ids.size();
355        StringBuilder sb = new StringBuilder();
356        sb.append("SELECT * FROM Document WHERE ecm:uuid IN (");
357        for (int i = 0; i < docCount; i++) {
358            sb.append(NXQL.escapeString(ids.get(i)));
359            if (i < docCount - 1) {
360                sb.append(", ");
361            }
362        }
363        sb.append(")");
364        String query = sb.toString();
365        if (log.isDebugEnabled()) {
366            log.debug(String.format("Fetching %d documents from VCS: %s", docCount, query));
367        }
368        return session.query(query);
369    }
370
371    /**
372     * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem}
373     * ancestors.
374     */
375    protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) {
376        Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>();
377        if (log.isTraceEnabled()) {
378            log.trace(String.format("Caching current FolderItem for doc %s: %s", docPath, getPath()));
379        }
380        ancestorCache.put(new IdRef(docId), this);
381        List<FileSystemItem> descendants = new ArrayList<>(docs.size());
382        for (DocumentModel doc : docs) {
383            FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false);
384            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
385            FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false,
386                    false);
387            if (descendant != null) {
388                if (descendant.isFolder()) {
389                    if (log.isTraceEnabled()) {
390                        log.trace(String.format("Caching descendant FolderItem for doc %s: %s", doc.getPathAsString(),
391                                descendant.getPath()));
392                    }
393                    ancestorCache.put(doc.getRef(), (FolderItem) descendant);
394                }
395                descendants.add(descendant);
396            }
397        }
398        return descendants;
399    }
400
401    protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc,
402            CoreSession session, boolean cacheItem) {
403        // TODO: handle collections
404        DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef());
405        if (parentDocRef == null) {
406            throw new RootlessItemException("Reached repository root");
407        }
408
409        FolderItem parentItem = cache.get(parentDocRef);
410        if (parentItem != null) {
411            if (log.isTraceEnabled()) {
412                log.trace(String.format("Found parent FolderItem in cache for doc %s: %s", doc.getPathAsString(),
413                        parentItem.getPath()));
414            }
415            return getFolderItem(cache, doc, parentItem, cacheItem);
416        }
417
418        if (log.isTraceEnabled()) {
419            log.trace(String.format("No parent FolderItem found in cache for doc %s, computing ancestor cache",
420                    doc.getPathAsString()));
421        }
422        DocumentModel parentDoc = null;
423        try {
424            parentDoc = session.getDocument(parentDocRef);
425        } catch (DocumentSecurityException e) {
426            throw new RootlessItemException(String.format("User %s has no READ access on parent of document %s (%s).",
427                    principal.getName(), doc.getPathAsString(), doc.getId()), e);
428        }
429        parentItem = populateAncestorCache(cache, parentDoc, session, true);
430        return getFolderItem(cache, doc, parentItem, cacheItem);
431    }
432
433    protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem,
434            boolean cacheItem) {
435        if (cacheItem) {
436            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
437            FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false,
438                    false);
439            if (fsItem == null) {
440                throw new RootlessItemException(String.format(
441                        "Reached a document %s that cannot be  adapted as a (possibly virtual) descendant of the top level folder item.",
442                        doc.getPathAsString()));
443            }
444            FolderItem folderItem = (FolderItem) fsItem;
445            if (log.isTraceEnabled()) {
446                log.trace(String.format("Caching FolderItem for doc %s: %s", doc.getPathAsString(),
447                        folderItem.getPath()));
448            }
449            cache.put(doc.getRef(), folderItem);
450            return folderItem;
451        } else {
452            return parentItem;
453        }
454    }
455
456    @Override
457    public boolean getCanCreateChild() {
458        return canCreateChild;
459    }
460
461    @Override
462    public FolderItem createFolder(String name, boolean overwrite) {
463        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
464            DocumentModel folder = getFileManager().createFolder(session, name, docPath, overwrite);
465            if (folder == null) {
466                throw new NuxeoException(String.format(
467                        "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.",
468                        name, docPath));
469            }
470            return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this);
471        } catch (NuxeoException e) {
472            e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath));
473            throw e;
474        } catch (IOException e) {
475            throw new NuxeoException(
476                    String.format("Error while trying to create folder %s as a child of doc %s", name, docPath), e);
477        }
478    }
479
480    @Override
481    public FileItem createFile(Blob blob, boolean overwrite) {
482        String fileName = blob.getFilename();
483        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
484            DocumentModel file = getFileManager().createDocumentFromBlob(session, blob, docPath, overwrite, fileName);
485            if (file == null) {
486                throw new NuxeoException(String.format(
487                        "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.",
488                        fileName, docPath));
489            }
490            return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this);
491        } catch (NuxeoException e) {
492            e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath));
493            throw e;
494        } catch (IOException e) {
495            throw new NuxeoException(
496                    String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath), e);
497        }
498    }
499
500    /*--------------------- Protected -----------------*/
501    protected void initialize(DocumentModel doc) {
502        this.name = docTitle;
503        this.folder = true;
504        this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE);
505        if (canCreateChild) {
506            if (Framework.getService(ConfigurationService.class)
507                         .isBooleanPropertyTrue(PERMISSION_CHECK_OPTIMIZED_PROPERTY)) {
508                // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties
509                // and by default WriteProperties <=> Write <=> AddChildren
510                this.canCreateChild = canRename;
511            } else {
512                // In non optimized mode check AddChildren
513                this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN);
514            }
515        }
516        this.canScrollDescendants = true;
517    }
518
519    protected FileManager getFileManager() {
520        return Framework.getLocalService(FileManager.class);
521    }
522
523    /*---------- Needed for JSON deserialization ----------*/
524    protected void setCanCreateChild(boolean canCreateChild) {
525        this.canCreateChild = canCreateChild;
526    }
527
528    protected void setCanScrollDescendants(boolean canScrollDescendants) {
529        this.canScrollDescendants = canScrollDescendants;
530    }
531
532}