Source code

001/*
002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Antoine Taillefer <ataillefer@nuxeo.com>
018 */
019package org.nuxeo.drive.adapter.impl;
020
021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY;
022
023import java.io.IOException;
024import java.io.Serializable;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.Iterator;
028import java.util.List;
029import java.util.Map;
030import java.util.UUID;
031import java.util.concurrent.Semaphore;
032
033import org.apache.commons.lang.StringUtils;
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.nuxeo.drive.adapter.FileItem;
037import org.nuxeo.drive.adapter.FileSystemItem;
038import org.nuxeo.drive.adapter.FolderItem;
039import org.nuxeo.drive.adapter.RootlessItemException;
040import org.nuxeo.drive.adapter.ScrollFileSystemItemList;
041import org.nuxeo.drive.service.FileSystemItemAdapterService;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.CoreInstance;
044import org.nuxeo.ecm.core.api.CoreSession;
045import org.nuxeo.ecm.core.api.DocumentModel;
046import org.nuxeo.ecm.core.api.DocumentModelList;
047import org.nuxeo.ecm.core.api.DocumentRef;
048import org.nuxeo.ecm.core.api.DocumentSecurityException;
049import org.nuxeo.ecm.core.api.IdRef;
050import org.nuxeo.ecm.core.api.IterableQueryResult;
051import org.nuxeo.ecm.core.api.NuxeoException;
052import org.nuxeo.ecm.core.api.security.SecurityConstants;
053import org.nuxeo.ecm.core.cache.Cache;
054import org.nuxeo.ecm.core.cache.CacheService;
055import org.nuxeo.ecm.core.query.sql.NXQL;
056import org.nuxeo.ecm.core.schema.FacetNames;
057import org.nuxeo.ecm.platform.filemanager.api.FileManager;
058import org.nuxeo.ecm.platform.query.api.PageProvider;
059import org.nuxeo.ecm.platform.query.api.PageProviderService;
060import org.nuxeo.runtime.api.Framework;
061import org.nuxeo.runtime.services.config.ConfigurationService;
062
063/**
064 * {@link DocumentModel} backed implementation of a {@link FolderItem}.
065 *
066 * @author Antoine Taillefer
067 */
068public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem {
069
070    private static final Log log = LogFactory.getLog(DocumentBackedFolderItem.class);
071
072    private static final long serialVersionUID = 1L;
073
074    private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN";
075
076    protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll";
077
078    protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize";
079
080    protected static final String MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = "1000";
081
082    protected static final int VCS_CHUNK_SIZE = 100;
083
084    protected boolean canCreateChild;
085
086    protected boolean canScrollDescendants;
087
088    public DocumentBackedFolderItem(String factoryName, DocumentModel doc) {
089        this(factoryName, doc, false);
090    }
091
092    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) {
093        this(factoryName, doc, relaxSyncRootConstraint, true);
094    }
095
096    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint,
097            boolean getLockInfo) {
098        super(factoryName, doc, relaxSyncRootConstraint, getLockInfo);
099        initialize(doc);
100    }
101
102    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) {
103        this(factoryName, parentItem, doc, false);
104    }
105
106    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
107            boolean relaxSyncRootConstraint) {
108        this(factoryName, parentItem, doc, relaxSyncRootConstraint, true);
109    }
110
111    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
112            boolean relaxSyncRootConstraint, boolean getLockInfo) {
113        super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo);
114        initialize(doc);
115    }
116
117    protected DocumentBackedFolderItem() {
118        // Needed for JSON deserialization
119    }
120
121    /*--------------------- FileSystemItem ---------------------*/
122    @Override
123    public void rename(String name) {
124        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
125            // Update doc properties
126            DocumentModel doc = getDocument(session);
127            doc.setPropertyValue("dc:title", name);
128            doc = session.saveDocument(doc);
129            session.save();
130            // Update FileSystemItem attributes
131            this.docTitle = name;
132            this.name = name;
133            updateLastModificationDate(doc);
134        }
135    }
136
137    /*--------------------- FolderItem -----------------*/
138    @Override
139    @SuppressWarnings("unchecked")
140    public List<FileSystemItem> getChildren() {
141        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
142            PageProviderService pageProviderService = Framework.getLocalService(PageProviderService.class);
143            Map<String, Serializable> props = new HashMap<String, Serializable>();
144            props.put(CORE_SESSION_PROPERTY, (Serializable) session);
145            PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider(
146                    FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId);
147            Long pageSize = childrenPageProvider.getPageSize();
148
149            List<FileSystemItem> children = new ArrayList<FileSystemItem>();
150            int nbChildren = 0;
151            boolean reachedPageSize = false;
152            boolean hasNextPage = true;
153            // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size
154            // number of
155            // FileSystemItems
156            while (nbChildren < pageSize && hasNextPage) {
157                List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage();
158                for (DocumentModel dmChild : dmChildren) {
159                    // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
160                    FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false,
161                            false, false);
162                    if (child != null) {
163                        children.add(child);
164                        nbChildren++;
165                        if (nbChildren == pageSize) {
166                            reachedPageSize = true;
167                            break;
168                        }
169                    }
170                }
171                if (!reachedPageSize) {
172                    hasNextPage = childrenPageProvider.isNextPageAvailable();
173                    if (hasNextPage) {
174                        childrenPageProvider.nextPage();
175                    }
176                }
177            }
178
179            return children;
180        }
181    }
182
183    @Override
184    public boolean getCanScrollDescendants() {
185        return canScrollDescendants;
186    }
187
188    @Override
189    public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) {
190        Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore();
191        try {
192            if (log.isTraceEnabled()) {
193                log.trace(String.format("Thread [%s] acquiring scroll batch semaphore",
194                        Thread.currentThread().getName()));
195            }
196            semaphore.acquire();
197            try {
198                if (log.isTraceEnabled()) {
199                    log.trace(String.format(
200                            "Thread [%s] acquired scroll batch semaphore, available permits reduced to %d",
201                            Thread.currentThread().getName(), semaphore.availablePermits()));
202                }
203                return doScrollDescendants(scrollId, batchSize, keepAlive);
204            } finally {
205                semaphore.release();
206                if (log.isTraceEnabled()) {
207                    log.trace(String.format(
208                            "Thread [%s] released scroll batch semaphore, available permits increased to %d",
209                            Thread.currentThread().getName(), semaphore.availablePermits()));
210                }
211            }
212        } catch (InterruptedException cause) {
213            Thread.currentThread().interrupt();
214            throw new NuxeoException("Scroll batch interrupted", cause);
215        }
216    }
217
218    @SuppressWarnings("unchecked")
219    protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) {
220        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
221
222            // Limit batch size sent by the client
223            checkBatchSize(batchSize);
224
225            // Scroll through a batch of documents
226            ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive);
227            String newScrollId = descendantDocsBatch.getScrollId();
228            if (descendantDocsBatch.isEmpty()) {
229                // No more descendants left to return
230                return new ScrollFileSystemItemListImpl(newScrollId, 0);
231            }
232
233            // Adapt documents as FileSystemItems
234            List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session);
235            if (log.isDebugEnabled()) {
236                log.debug(String.format("Retrieved %d descendants of FolderItem %s (batchSize = %d)",
237                        descendants.size(), docPath, batchSize));
238            }
239            return new ScrollFileSystemItemListImpl(newScrollId, descendants);
240        }
241    }
242
243    protected void checkBatchSize(int batchSize) {
244        int maxDescendantsBatchSize = Integer.parseInt(Framework.getService(ConfigurationService.class).getProperty(
245                MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT));
246        if (batchSize > maxDescendantsBatchSize) {
247            throw new NuxeoException(String.format(
248                    "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.",
249                    batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY));
250        }
251    }
252
253    @SuppressWarnings("unchecked")
254    protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session,
255            long keepAlive) {
256        Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE);
257        if (scrollingCache == null) {
258            throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE);
259        }
260        String newScrollId;
261        List<String> descendantIds;
262        if (StringUtils.isEmpty(scrollId)) {
263            // Perform initial query to fetch ids of all the descendant documents and put the result list in a
264            // cache, aka "search context"
265            descendantIds = new ArrayList<>();
266            StringBuilder sb = new StringBuilder(
267                    String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId));
268            sb.append(" AND ecm:currentLifeCycleState != 'deleted'");
269            sb.append(" AND ecm:mixinType != 'HiddenInNavigation'");
270            // Don't need to add ecm:isCheckedInVersion = 0 because versions are already excluded by the
271            // ecm:ancestorId clause since they have no path
272            String query = sb.toString();
273            if (log.isDebugEnabled()) {
274                log.debug(String.format("Executing initial query to scroll through the descendants of %s: %s", docPath,
275                        query));
276            }
277            try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) {
278                Iterator<Map<String, Serializable>> it = res.iterator();
279                while (it.hasNext()) {
280                    descendantIds.add((String) it.next().get(NXQL.ECM_UUID));
281                }
282            }
283            // Generate a scroll id
284            newScrollId = UUID.randomUUID().toString();
285            if (log.isDebugEnabled()) {
286                log.debug(String.format(
287                        "Put initial query result list (search context) in the %s cache at key (scrollId) %s",
288                        DESCENDANTS_SCROLL_CACHE, newScrollId));
289            }
290            scrollingCache.put(newScrollId, (Serializable) descendantIds);
291        } else {
292            // Get the descendant ids from the cache
293            descendantIds = (List<String>) scrollingCache.get(scrollId);
294            if (descendantIds == null) {
295                throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]",
296                        DESCENDANTS_SCROLL_CACHE, scrollId));
297            }
298            newScrollId = scrollId;
299        }
300
301        if (descendantIds.isEmpty()) {
302            return new ScrollDocumentModelList(newScrollId, 0);
303        }
304
305        // Extract a batch of descendant ids
306        List<String> descendantIdsBatch = getBatch(descendantIds, batchSize);
307        // Update descendant ids in the cache
308        scrollingCache.put(newScrollId, (Serializable) descendantIds);
309        // Fetch documents from VCS
310        DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session);
311        return new ScrollDocumentModelList(newScrollId, descendantDocsBatch);
312    }
313
314    /**
315     * Extracts batchSize elements from the input list.
316     */
317    protected List<String> getBatch(List<String> ids, int batchSize) {
318        List<String> batch = new ArrayList<>(batchSize);
319        int idCount = 0;
320        Iterator<String> it = ids.iterator();
321        while (it.hasNext() && idCount < batchSize) {
322            batch.add(it.next());
323            it.remove();
324            idCount++;
325        }
326        return batch;
327    }
328
329    protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) {
330        DocumentModelList res = null;
331        int size = ids.size();
332        int start = 0;
333        int end = Math.min(VCS_CHUNK_SIZE, size);
334        boolean done = false;
335        while (!done) {
336            DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session);
337            if (res == null) {
338                res = docs;
339            } else {
340                res.addAll(docs);
341            }
342            if (end >= ids.size()) {
343                done = true;
344            } else {
345                start = end;
346                end = Math.min(start + VCS_CHUNK_SIZE, size);
347            }
348        }
349        return res;
350    }
351
352    protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) {
353        int docCount = ids.size();
354        StringBuilder sb = new StringBuilder();
355        sb.append("SELECT * FROM Document WHERE ecm:uuid IN (");
356        for (int i = 0; i < docCount; i++) {
357            sb.append(NXQL.escapeString(ids.get(i)));
358            if (i < docCount - 1) {
359                sb.append(", ");
360            }
361        }
362        sb.append(")");
363        String query = sb.toString();
364        if (log.isDebugEnabled()) {
365            log.debug(String.format("Fetching %d documents from VCS: %s", docCount, query));
366        }
367        return session.query(query);
368    }
369
370    /**
371     * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem}
372     * ancestors.
373     */
374    protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) {
375        Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>();
376        if (log.isTraceEnabled()) {
377            log.trace(String.format("Caching current FolderItem for doc %s: %s", docPath, getPath()));
378        }
379        ancestorCache.put(new IdRef(docId), this);
380        List<FileSystemItem> descendants = new ArrayList<>(docs.size());
381        for (DocumentModel doc : docs) {
382            FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false);
383            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
384            FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false,
385                    false);
386            if (descendant != null) {
387                if (descendant.isFolder()) {
388                    if (log.isTraceEnabled()) {
389                        log.trace(String.format("Caching descendant FolderItem for doc %s: %s", doc.getPathAsString(),
390                                descendant.getPath()));
391                    }
392                    ancestorCache.put(doc.getRef(), (FolderItem) descendant);
393                }
394                descendants.add(descendant);
395            }
396        }
397        return descendants;
398    }
399
400    protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc,
401            CoreSession session, boolean cacheItem) {
402        // TODO: handle collections
403        DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef());
404        if (parentDocRef == null) {
405            throw new RootlessItemException("Reached repository root");
406        }
407
408        FolderItem parentItem = cache.get(parentDocRef);
409        if (parentItem != null) {
410            if (log.isTraceEnabled()) {
411                log.trace(String.format("Found parent FolderItem in cache for doc %s: %s", doc.getPathAsString(),
412                        parentItem.getPath()));
413            }
414            return getFolderItem(cache, doc, parentItem, cacheItem);
415        }
416
417        if (log.isTraceEnabled()) {
418            log.trace(String.format("No parent FolderItem found in cache for doc %s, computing ancestor cache",
419                    doc.getPathAsString()));
420        }
421        DocumentModel parentDoc = null;
422        try {
423            parentDoc = session.getDocument(parentDocRef);
424        } catch (DocumentSecurityException e) {
425            throw new RootlessItemException(String.format("User %s has no READ access on parent of document %s (%s).",
426                    principal.getName(), doc.getPathAsString(), doc.getId()), e);
427        }
428        parentItem = populateAncestorCache(cache, parentDoc, session, true);
429        return getFolderItem(cache, doc, parentItem, cacheItem);
430    }
431
432    protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem,
433            boolean cacheItem) {
434        if (cacheItem) {
435            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
436            FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false,
437                    false);
438            if (fsItem == null) {
439                throw new RootlessItemException(String.format(
440                        "Reached a document %s that cannot be  adapted as a (possibly virtual) descendant of the top level folder item.",
441                        doc.getPathAsString()));
442            }
443            FolderItem folderItem = (FolderItem) fsItem;
444            if (log.isTraceEnabled()) {
445                log.trace(String.format("Caching FolderItem for doc %s: %s", doc.getPathAsString(),
446                        folderItem.getPath()));
447            }
448            cache.put(doc.getRef(), folderItem);
449            return folderItem;
450        } else {
451            return parentItem;
452        }
453    }
454
455    @Override
456    public boolean getCanCreateChild() {
457        return canCreateChild;
458    }
459
460    @Override
461    public FolderItem createFolder(String name) {
462        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
463            DocumentModel folder = getFileManager().createFolder(session, name, docPath);
464            if (folder == null) {
465                throw new NuxeoException(String.format(
466                        "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.",
467                        name, docPath));
468            }
469            return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this);
470        } catch (NuxeoException e) {
471            e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath));
472            throw e;
473        } catch (IOException e) {
474            throw new NuxeoException(
475                    String.format("Error while trying to create folder %s as a child of doc %s", name, docPath), e);
476        }
477    }
478
479    @Override
480    public FileItem createFile(Blob blob) {
481        String fileName = blob.getFilename();
482        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
483            // TODO: manage conflict (overwrite should not necessarily be true)
484            DocumentModel file = getFileManager().createDocumentFromBlob(session, blob, docPath, true, fileName);
485            if (file == null) {
486                throw new NuxeoException(String.format(
487                        "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.",
488                        fileName, docPath));
489            }
490            return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this);
491        } catch (NuxeoException e) {
492            e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath));
493            throw e;
494        } catch (IOException e) {
495            throw new NuxeoException(
496                    String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath), e);
497        }
498    }
499
500    /*--------------------- Protected -----------------*/
501    protected void initialize(DocumentModel doc) {
502        this.name = docTitle;
503        this.folder = true;
504        this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE);
505        if (canCreateChild) {
506            if (Framework.getService(ConfigurationService.class)
507                         .isBooleanPropertyTrue(PERMISSION_CHECK_OPTIMIZED_PROPERTY)) {
508                // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties
509                // and by default WriteProperties <=> Write <=> AddChildren
510                this.canCreateChild = canRename;
511            } else {
512                // In non optimized mode check AddChildren
513                this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN);
514            }
515        }
516        this.canScrollDescendants = true;
517    }
518
519    protected FileManager getFileManager() {
520        return Framework.getLocalService(FileManager.class);
521    }
522
523    /*---------- Needed for JSON deserialization ----------*/
524    protected void setCanCreateChild(boolean canCreateChild) {
525        this.canCreateChild = canCreateChild;
526    }
527
528    protected void setCanScrollDescendants(boolean canScrollDescendants) {
529        this.canScrollDescendants = canScrollDescendants;
530    }
531
532}