001/*
002 * (C) Copyright 2012 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Antoine Taillefer <ataillefer@nuxeo.com>
018 */
019package org.nuxeo.drive.adapter.impl;
020
021import static org.nuxeo.ecm.platform.query.nxql.CoreQueryDocumentPageProvider.CORE_SESSION_PROPERTY;
022
023import java.io.IOException;
024import java.io.Serializable;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.Iterator;
028import java.util.List;
029import java.util.Map;
030import java.util.UUID;
031import java.util.concurrent.Semaphore;
032
033import org.apache.commons.lang.StringUtils;
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.nuxeo.drive.adapter.FileItem;
037import org.nuxeo.drive.adapter.FileSystemItem;
038import org.nuxeo.drive.adapter.FolderItem;
039import org.nuxeo.drive.adapter.RootlessItemException;
040import org.nuxeo.drive.adapter.ScrollFileSystemItemList;
041import org.nuxeo.drive.service.FileSystemItemAdapterService;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.CoreInstance;
044import org.nuxeo.ecm.core.api.CoreSession;
045import org.nuxeo.ecm.core.api.DocumentModel;
046import org.nuxeo.ecm.core.api.DocumentModelList;
047import org.nuxeo.ecm.core.api.DocumentRef;
048import org.nuxeo.ecm.core.api.DocumentSecurityException;
049import org.nuxeo.ecm.core.api.IdRef;
050import org.nuxeo.ecm.core.api.IterableQueryResult;
051import org.nuxeo.ecm.core.api.NuxeoException;
052import org.nuxeo.ecm.core.api.security.SecurityConstants;
053import org.nuxeo.ecm.core.cache.Cache;
054import org.nuxeo.ecm.core.cache.CacheService;
055import org.nuxeo.ecm.core.query.sql.NXQL;
056import org.nuxeo.ecm.core.schema.FacetNames;
057import org.nuxeo.ecm.platform.filemanager.api.FileManager;
058import org.nuxeo.ecm.platform.query.api.PageProvider;
059import org.nuxeo.ecm.platform.query.api.PageProviderService;
060import org.nuxeo.runtime.api.Framework;
061import org.nuxeo.runtime.services.config.ConfigurationService;
062
063/**
064 * {@link DocumentModel} backed implementation of a {@link FolderItem}.
065 *
066 * @author Antoine Taillefer
067 */
068public class DocumentBackedFolderItem extends AbstractDocumentBackedFileSystemItem implements FolderItem {
069
070    private static final Log log = LogFactory.getLog(DocumentBackedFolderItem.class);
071
072    private static final long serialVersionUID = 1L;
073
074    private static final String FOLDER_ITEM_CHILDREN_PAGE_PROVIDER = "FOLDER_ITEM_CHILDREN";
075
076    protected static final String DESCENDANTS_SCROLL_CACHE = "driveDescendantsScroll";
077
078    protected static final String MAX_DESCENDANTS_BATCH_SIZE_PROPERTY = "org.nuxeo.drive.maxDescendantsBatchSize";
079
080    protected static final String MAX_DESCENDANTS_BATCH_SIZE_DEFAULT = "1000";
081
082    protected static final int VCS_CHUNK_SIZE = 100;
083
084    protected boolean canCreateChild;
085
086    protected boolean canScrollDescendants;
087
088    public DocumentBackedFolderItem(String factoryName, DocumentModel doc) {
089        this(factoryName, doc, false);
090    }
091
092    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint) {
093        this(factoryName, doc, relaxSyncRootConstraint, true);
094    }
095
096    public DocumentBackedFolderItem(String factoryName, DocumentModel doc, boolean relaxSyncRootConstraint,
097            boolean getLockInfo) {
098        super(factoryName, doc, relaxSyncRootConstraint, getLockInfo);
099        initialize(doc);
100    }
101
102    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc) {
103        this(factoryName, parentItem, doc, false);
104    }
105
106    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
107            boolean relaxSyncRootConstraint) {
108        this(factoryName, parentItem, doc, relaxSyncRootConstraint, true);
109    }
110
111    public DocumentBackedFolderItem(String factoryName, FolderItem parentItem, DocumentModel doc,
112            boolean relaxSyncRootConstraint, boolean getLockInfo) {
113        super(factoryName, parentItem, doc, relaxSyncRootConstraint, getLockInfo);
114        initialize(doc);
115    }
116
117    protected DocumentBackedFolderItem() {
118        // Needed for JSON deserialization
119    }
120
121    /*--------------------- FileSystemItem ---------------------*/
122    @Override
123    public void rename(String name) {
124        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
125            // Update doc properties
126            DocumentModel doc = getDocument(session);
127            doc.setPropertyValue("dc:title", name);
128            doc = session.saveDocument(doc);
129            session.save();
130            // Update FileSystemItem attributes
131            this.docTitle = name;
132            this.name = name;
133            updateLastModificationDate(doc);
134        }
135    }
136
137    /*--------------------- FolderItem -----------------*/
138    @Override
139    @SuppressWarnings("unchecked")
140    public List<FileSystemItem> getChildren() {
141        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
142            PageProviderService pageProviderService = Framework.getLocalService(PageProviderService.class);
143            Map<String, Serializable> props = new HashMap<String, Serializable>();
144            props.put(CORE_SESSION_PROPERTY, (Serializable) session);
145            PageProvider<DocumentModel> childrenPageProvider = (PageProvider<DocumentModel>) pageProviderService.getPageProvider(
146                    FOLDER_ITEM_CHILDREN_PAGE_PROVIDER, null, null, 0L, props, docId);
147            Long pageSize = childrenPageProvider.getPageSize();
148
149            List<FileSystemItem> children = new ArrayList<FileSystemItem>();
150            int nbChildren = 0;
151            boolean reachedPageSize = false;
152            boolean hasNextPage = true;
153            // Since query results are filtered, make sure we iterate on PageProvider to get at most its page size
154            // number of
155            // FileSystemItems
156            while (nbChildren < pageSize && hasNextPage) {
157                List<DocumentModel> dmChildren = childrenPageProvider.getCurrentPage();
158                for (DocumentModel dmChild : dmChildren) {
159                    // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
160                    FileSystemItem child = getFileSystemItemAdapterService().getFileSystemItem(dmChild, this, false,
161                            false, false);
162                    if (child != null) {
163                        children.add(child);
164                        nbChildren++;
165                        if (nbChildren == pageSize) {
166                            reachedPageSize = true;
167                            break;
168                        }
169                    }
170                }
171                if (!reachedPageSize) {
172                    hasNextPage = childrenPageProvider.isNextPageAvailable();
173                    if (hasNextPage) {
174                        childrenPageProvider.nextPage();
175                    }
176                }
177            }
178
179            return children;
180        }
181    }
182
183    @Override
184    public boolean getCanScrollDescendants() {
185        return canScrollDescendants;
186    }
187
188    @Override
189    public ScrollFileSystemItemList scrollDescendants(String scrollId, int batchSize, long keepAlive) {
190        Semaphore semaphore = Framework.getService(FileSystemItemAdapterService.class).getScrollBatchSemaphore();
191        try {
192            if (log.isTraceEnabled()) {
193                log.trace(String.format("Thread [%s] acquiring scroll batch semaphore", Thread.currentThread()
194                                                                                              .getName()));
195            }
196            semaphore.acquire();
197            try {
198                if (log.isTraceEnabled()) {
199                    log.trace(String.format(
200                            "Thread [%s] acquired scroll batch semaphore, available permits reduced to %d",
201                            Thread.currentThread().getName(), semaphore.availablePermits()));
202                }
203                return doScrollDescendants(scrollId, batchSize, keepAlive);
204            } finally {
205                semaphore.release();
206                if (log.isTraceEnabled()) {
207                    log.trace(String.format(
208                            "Thread [%s] released scroll batch semaphore, available permits increased to %d",
209                            Thread.currentThread().getName(), semaphore.availablePermits()));
210                }
211            }
212        } catch (InterruptedException cause) {
213            Thread.currentThread().interrupt();
214            throw new NuxeoException("Scroll batch interrupted", cause);
215        }
216    }
217
218    @SuppressWarnings("unchecked")
219    protected ScrollFileSystemItemList doScrollDescendants(String scrollId, int batchSize, long keepAlive) {
220        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
221
222            // Limit batch size sent by the client
223            checkBatchSize(batchSize);
224
225            // Scroll through a batch of documents
226            ScrollDocumentModelList descendantDocsBatch = getScrollBatch(scrollId, batchSize, session, keepAlive);
227            String newScrollId = descendantDocsBatch.getScrollId();
228            if (descendantDocsBatch.isEmpty()) {
229                // No more descendants left to return
230                return new ScrollFileSystemItemListImpl(newScrollId, 0);
231            }
232
233            // Adapt documents as FileSystemItems
234            List<FileSystemItem> descendants = adaptDocuments(descendantDocsBatch, session);
235            if (log.isDebugEnabled()) {
236                log.debug(String.format("Retrieved %d descendants of FolderItem %s (batchSize = %d)",
237                        descendants.size(), docPath, batchSize));
238            }
239            return new ScrollFileSystemItemListImpl(newScrollId, descendants);
240        }
241    }
242
243    protected void checkBatchSize(int batchSize) {
244        int maxDescendantsBatchSize = Integer.parseInt(Framework.getService(ConfigurationService.class).getProperty(
245                MAX_DESCENDANTS_BATCH_SIZE_PROPERTY, MAX_DESCENDANTS_BATCH_SIZE_DEFAULT));
246        if (batchSize > maxDescendantsBatchSize) {
247            throw new NuxeoException(
248                    String.format(
249                            "Batch size %d is greater than the maximum batch size allowed %d. If you need to increase this limit you can set the %s configuration property but this is not recommended for performance reasons.",
250                            batchSize, maxDescendantsBatchSize, MAX_DESCENDANTS_BATCH_SIZE_PROPERTY));
251        }
252    }
253
254    @SuppressWarnings("unchecked")
255    protected ScrollDocumentModelList getScrollBatch(String scrollId, int batchSize, CoreSession session, long keepAlive) {
256        Cache scrollingCache = Framework.getService(CacheService.class).getCache(DESCENDANTS_SCROLL_CACHE);
257        if (scrollingCache == null) {
258            throw new NuxeoException("Cache not found: " + DESCENDANTS_SCROLL_CACHE);
259        }
260        String newScrollId;
261        List<String> descendantIds;
262        if (StringUtils.isEmpty(scrollId)) {
263            // Perform initial query to fetch ids of all the descendant documents and put the result list in a
264            // cache, aka "search context"
265            descendantIds = new ArrayList<>();
266            StringBuilder sb = new StringBuilder(String.format(
267                    "SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", docId));
268            sb.append(" AND ecm:currentLifeCycleState != 'deleted'");
269            sb.append(" AND ecm:mixinType != 'HiddenInNavigation'");
270            // Don't need to add ecm:isCheckedInVersion = 0 because versions are already excluded by the
271            // ecm:ancestorId clause since they have no path
272            String query = sb.toString();
273            if (log.isDebugEnabled()) {
274                log.debug(String.format("Executing initial query to scroll through the descendants of %s: %s", docPath,
275                        query));
276            }
277            try (IterableQueryResult res = session.queryAndFetch(sb.toString(), NXQL.NXQL)) {
278                Iterator<Map<String, Serializable>> it = res.iterator();
279                while (it.hasNext()) {
280                    descendantIds.add((String) it.next().get(NXQL.ECM_UUID));
281                }
282            }
283            // Generate a scroll id
284            newScrollId = UUID.randomUUID().toString();
285            if (log.isDebugEnabled()) {
286                log.debug(String.format(
287                        "Put initial query result list (search context) in the %s cache at key (scrollId) %s",
288                        DESCENDANTS_SCROLL_CACHE, newScrollId));
289            }
290            scrollingCache.put(newScrollId, (Serializable) descendantIds);
291        } else {
292            // Get the descendant ids from the cache
293            descendantIds = (List<String>) scrollingCache.get(scrollId);
294            if (descendantIds == null) {
295                throw new NuxeoException(String.format("No search context found in the %s cache for scrollId [%s]",
296                        DESCENDANTS_SCROLL_CACHE, scrollId));
297            }
298            newScrollId = scrollId;
299        }
300
301        if (descendantIds.isEmpty()) {
302            return new ScrollDocumentModelList(newScrollId, 0);
303        }
304
305        // Extract a batch of descendant ids
306        List<String> descendantIdsBatch = getBatch(descendantIds, batchSize);
307        // Fetch documents from VCS
308        DocumentModelList descendantDocsBatch = fetchFromVCS(descendantIdsBatch, session);
309        return new ScrollDocumentModelList(newScrollId, descendantDocsBatch);
310    }
311
312    /**
313     * Extracts batchSize elements from the input list.
314     */
315    protected List<String> getBatch(List<String> ids, int batchSize) {
316        List<String> batch = new ArrayList<>(batchSize);
317        int idCount = 0;
318        Iterator<String> it = ids.iterator();
319        while (it.hasNext() && idCount < batchSize) {
320            batch.add(it.next());
321            it.remove();
322            idCount++;
323        }
324        return batch;
325    }
326
327    protected DocumentModelList fetchFromVCS(List<String> ids, CoreSession session) {
328        DocumentModelList res = null;
329        int size = ids.size();
330        int start = 0;
331        int end = Math.min(VCS_CHUNK_SIZE, size);
332        boolean done = false;
333        while (!done) {
334            DocumentModelList docs = fetchFromVcsChunk(ids.subList(start, end), session);
335            if (res == null) {
336                res = docs;
337            } else {
338                res.addAll(docs);
339            }
340            if (end >= ids.size()) {
341                done = true;
342            } else {
343                start = end;
344                end = Math.min(start + VCS_CHUNK_SIZE, size);
345            }
346        }
347        return res;
348    }
349
350    protected DocumentModelList fetchFromVcsChunk(final List<String> ids, CoreSession session) {
351        int docCount = ids.size();
352        StringBuilder sb = new StringBuilder();
353        sb.append("SELECT * FROM Document WHERE ecm:uuid IN (");
354        for (int i = 0; i < docCount; i++) {
355            sb.append(NXQL.escapeString(ids.get(i)));
356            if (i < docCount - 1) {
357                sb.append(", ");
358            }
359        }
360        sb.append(")");
361        String query = sb.toString();
362        if (log.isDebugEnabled()) {
363            log.debug(String.format("Fetching %d documents from VCS: %s", docCount, query));
364        }
365        return session.query(query);
366    }
367
368    /**
369     * Adapts the given {@link DocumentModelList} as {@link FileSystemItem}s using a cache for the {@link FolderItem}
370     * ancestors.
371     */
372    protected List<FileSystemItem> adaptDocuments(DocumentModelList docs, CoreSession session) {
373        Map<DocumentRef, FolderItem> ancestorCache = new HashMap<>();
374        if (log.isTraceEnabled()) {
375            log.trace(String.format("Caching current FolderItem for doc %s: %s", docPath, getPath()));
376        }
377        ancestorCache.put(new IdRef(docId), this);
378        List<FileSystemItem> descendants = new ArrayList<>(docs.size());
379        for (DocumentModel doc : docs) {
380            FolderItem parent = populateAncestorCache(ancestorCache, doc, session, false);
381            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
382            FileSystemItem descendant = getFileSystemItemAdapterService().getFileSystemItem(doc, parent, false, false,
383                    false);
384            if (descendant != null) {
385                if (descendant.isFolder()) {
386                    if (log.isTraceEnabled()) {
387                        log.trace(String.format("Caching descendant FolderItem for doc %s: %s", doc.getPathAsString(),
388                                descendant.getPath()));
389                    }
390                    ancestorCache.put(doc.getRef(), (FolderItem) descendant);
391                }
392                descendants.add(descendant);
393            }
394        }
395        return descendants;
396    }
397
398    protected FolderItem populateAncestorCache(Map<DocumentRef, FolderItem> cache, DocumentModel doc,
399            CoreSession session, boolean cacheItem) {
400        // TODO: handle collections
401        DocumentRef parentDocRef = session.getParentDocumentRef(doc.getRef());
402        if (parentDocRef == null) {
403            throw new RootlessItemException("Reached repository root");
404        }
405
406        FolderItem parentItem = cache.get(parentDocRef);
407        if (parentItem != null) {
408            if (log.isTraceEnabled()) {
409                log.trace(String.format("Found parent FolderItem in cache for doc %s: %s", doc.getPathAsString(),
410                        parentItem.getPath()));
411            }
412            return getFolderItem(cache, doc, parentItem, cacheItem);
413        }
414
415        if (log.isTraceEnabled()) {
416            log.trace(String.format("No parent FolderItem found in cache for doc %s, computing ancestor cache",
417                    doc.getPathAsString()));
418        }
419        DocumentModel parentDoc = null;
420        try {
421            parentDoc = session.getDocument(parentDocRef);
422        } catch (DocumentSecurityException e) {
423            throw new RootlessItemException(String.format("User %s has no READ access on parent of document %s (%s).",
424                    principal.getName(), doc.getPathAsString(), doc.getId()), e);
425        }
426        parentItem = populateAncestorCache(cache, parentDoc, session, true);
427        return getFolderItem(cache, doc, parentItem, cacheItem);
428    }
429
430    protected FolderItem getFolderItem(Map<DocumentRef, FolderItem> cache, DocumentModel doc, FolderItem parentItem,
431            boolean cacheItem) {
432        if (cacheItem) {
433            // NXP-19442: Avoid useless and costly call to DocumentModel#getLockInfo
434            FileSystemItem fsItem = getFileSystemItemAdapterService().getFileSystemItem(doc, parentItem, true, false,
435                    false);
436            if (fsItem == null) {
437                throw new RootlessItemException(
438                        String.format(
439                                "Reached a document %s that cannot be  adapted as a (possibly virtual) descendant of the top level folder item.",
440                                doc.getPathAsString()));
441            }
442            FolderItem folderItem = (FolderItem) fsItem;
443            if (log.isTraceEnabled()) {
444                log.trace(String.format("Caching FolderItem for doc %s: %s", doc.getPathAsString(),
445                        folderItem.getPath()));
446            }
447            cache.put(doc.getRef(), folderItem);
448            return folderItem;
449        } else {
450            return parentItem;
451        }
452    }
453
454    @Override
455    public boolean getCanCreateChild() {
456        return canCreateChild;
457    }
458
459    @Override
460    public FolderItem createFolder(String name) {
461        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
462            DocumentModel folder = getFileManager().createFolder(session, name, docPath);
463            if (folder == null) {
464                throw new NuxeoException(
465                        String.format(
466                                "Cannot create folder named '%s' as a child of doc %s. Probably because of the allowed sub-types for this doc type, please check them.",
467                                name, docPath));
468            }
469            return (FolderItem) getFileSystemItemAdapterService().getFileSystemItem(folder, this);
470        } catch (NuxeoException e) {
471            e.addInfo(String.format("Error while trying to create folder %s as a child of doc %s", name, docPath));
472            throw e;
473        } catch (IOException e) {
474            throw new NuxeoException(String.format("Error while trying to create folder %s as a child of doc %s", name,
475                    docPath), e);
476        }
477    }
478
479    @Override
480    public FileItem createFile(Blob blob) {
481        String fileName = blob.getFilename();
482        try (CoreSession session = CoreInstance.openCoreSession(repositoryName, principal)) {
483            // TODO: manage conflict (overwrite should not necessarily be true)
484            DocumentModel file = getFileManager().createDocumentFromBlob(session, blob, docPath, true, fileName);
485            if (file == null) {
486                throw new NuxeoException(
487                        String.format(
488                                "Cannot create file '%s' as a child of doc %s. Probably because there are no file importers registered, please check the contributions to the <extension target=\"org.nuxeo.ecm.platform.filemanager.service.FileManagerService\" point=\"plugins\"> extension point.",
489                                fileName, docPath));
490            }
491            return (FileItem) getFileSystemItemAdapterService().getFileSystemItem(file, this);
492        } catch (NuxeoException e) {
493            e.addInfo(String.format("Error while trying to create file %s as a child of doc %s", fileName, docPath));
494            throw e;
495        } catch (IOException e) {
496            throw new NuxeoException(String.format("Error while trying to create file %s as a child of doc %s",
497                    fileName, docPath), e);
498        }
499    }
500
501    /*--------------------- Protected -----------------*/
502    protected void initialize(DocumentModel doc) {
503        this.name = docTitle;
504        this.folder = true;
505        this.canCreateChild = !doc.hasFacet(FacetNames.PUBLISH_SPACE);
506        if (canCreateChild) {
507            if (Framework.getService(ConfigurationService.class).isBooleanPropertyTrue(
508                    PERMISSION_CHECK_OPTIMIZED_PROPERTY)) {
509                // In optimized mode consider that canCreateChild <=> canRename because canRename <=> WriteProperties
510                // and by default WriteProperties <=> Write <=> AddChildren
511                this.canCreateChild = canRename;
512            } else {
513                // In non optimized mode check AddChildren
514                this.canCreateChild = doc.getCoreSession().hasPermission(doc.getRef(), SecurityConstants.ADD_CHILDREN);
515            }
516        }
517        this.canScrollDescendants = true;
518    }
519
520    protected FileManager getFileManager() {
521        return Framework.getLocalService(FileManager.class);
522    }
523
524    /*---------- Needed for JSON deserialization ----------*/
525    protected void setCanCreateChild(boolean canCreateChild) {
526        this.canCreateChild = canCreateChild;
527    }
528
529    protected void setCanScrollDescendants(boolean canScrollDescendants) {
530        this.canScrollDescendants = canScrollDescendants;
531    }
532
533}