001/*
002 * (C) Copyright 2006-2019 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Thierry Delprat
018 *     Florent Guillaume
019 */
020package org.nuxeo.ecm.quota.size;
021
022import static org.nuxeo.ecm.core.api.versioning.VersioningService.VERSIONING_OPTION;
023
024import java.util.Collection;
025import java.util.HashSet;
026import java.util.List;
027import java.util.function.BiConsumer;
028
029import org.apache.logging.log4j.LogManager;
030import org.apache.logging.log4j.Logger;
031import org.nuxeo.ecm.core.api.Blob;
032import org.nuxeo.ecm.core.api.CoreSession;
033import org.nuxeo.ecm.core.api.DocumentModel;
034import org.nuxeo.ecm.core.api.IdRef;
035import org.nuxeo.ecm.core.api.PathRef;
036import org.nuxeo.ecm.core.api.ScrollResult;
037import org.nuxeo.ecm.core.event.Event;
038import org.nuxeo.ecm.core.query.sql.NXQL;
039import org.nuxeo.ecm.core.utils.BlobsExtractor;
040import org.nuxeo.ecm.quota.AbstractQuotaStatsUpdater;
041import org.nuxeo.ecm.quota.QuotaStatsInitialWork;
042import org.nuxeo.runtime.api.Framework;
043import org.nuxeo.runtime.services.config.ConfigurationService;
044import org.nuxeo.runtime.transaction.TransactionHelper;
045
046/**
047 * {@link org.nuxeo.ecm.quota.QuotaStatsUpdater} counting space used by Blobs in document. This implementation does not
048 * track the space used by non-Blob properties.
049 *
050 * @since 8.3
051 */
052public class DocumentsSizeUpdater extends AbstractQuotaStatsUpdater {
053    private static Logger log = LogManager.getLogger(DocumentsSizeUpdater.class);
054
055    public static final String DISABLE_QUOTA_CHECK_LISTENER = "disableQuotaListener";
056
057    public static final String USER_WORKSPACES_ROOT = "UserWorkspacesRoot";
058
059    /** @since 11.1 */
060    public static final String CLEAR_SCROLL_SIZE_PROP = "nuxeo.quota.clear.scroll.size";
061
062    /** @since 11.1 */
063    public static final int DEFAULT_CLEAR_SCROLL_SIZE = 500;
064
065    /** @since 11.1 */
066    public static final String CLEAR_SCROLL_KEEP_ALIVE_PROP = "nuxeo.quota.clear.scroll.keepAliveSeconds";
067
068    /** @since 11.1 */
069    public static final int DEFAULT_CLEAR_SCROLL_KEEP_ALIVE = 60;
070
071    /** @since 11.1 */
072    public static final String INIT_SCROLL_SIZE_PROP = "nuxeo.quota.init.scroll.size";
073
074    /** @since 11.1 */
075    public static final int DEFAULT_INIT_SCROLL_SIZE = 250;
076
077    /** @since 11.1 */
078    public static final String INIT_SCROLL_KEEP_ALIVE_PROP = "nuxeo.quota.init.scroll.keepAliveSeconds";
079
080    /** @since 11.1 */
081    public static final int DEFAULT_INIT_SCROLL_KEEP_ALIVE = 120;
082
083    @Override
084    public void computeInitialStatistics(CoreSession session, QuotaStatsInitialWork currentWorker, String path) {
085        log.debug("Starting initial Quota computation for path: {}", path);
086        String query = "SELECT ecm:uuid FROM Document WHERE ecm:isVersion = 0 AND ecm:isProxy = 0";
087        DocumentModel root;
088        if (path == null) {
089            root = session.getRootDocument();
090        } else {
091            root = session.getDocument(new PathRef(path));
092            query += " AND ecm:path STARTSWITH " + NXQL.escapeString(path);
093        }
094        // get scroll configuration parameters
095        ConfigurationService confService = Framework.getService(ConfigurationService.class);
096        int clearScrollSize = confService.getInteger(CLEAR_SCROLL_SIZE_PROP, DEFAULT_CLEAR_SCROLL_SIZE);
097        int clearScrollKeepAlive = confService.getInteger(CLEAR_SCROLL_KEEP_ALIVE_PROP,
098                DEFAULT_CLEAR_SCROLL_KEEP_ALIVE);
099        int initScrollSize = confService.getInteger(INIT_SCROLL_SIZE_PROP, DEFAULT_INIT_SCROLL_SIZE);
100        int initScrollKeepAlive = confService.getInteger(INIT_SCROLL_KEEP_ALIVE_PROP, DEFAULT_INIT_SCROLL_KEEP_ALIVE);
101
102        // reset on all documents
103        // this will force an update if the quota addon was installed and then removed
104        log.debug("Start scrolling to clear quotas");
105        long clearCount = scrollAndDo(session, query, clearScrollSize, clearScrollKeepAlive,
106                (uuid, idx) -> clearQuotas(session, uuid));
107        log.debug("End scrolling to clear quotas, documentCount={}", clearCount);
108        clearQuotas(session, root.getId());
109        session.save();
110
111        // recompute quota on each doc
112        log.debug("Start scrolling to init quotas");
113        long initCount = scrollAndDo(session, query, initScrollSize, initScrollKeepAlive, (uuid, idx) -> {
114            DocumentModel doc = session.getDocument(new IdRef(uuid));
115            log.trace("process Quota initial computation on uuid={}", doc::getId);
116            log.trace("doc with uuid {} started update", doc::getId);
117            initDocument(session, doc);
118            log.trace("doc with uuid {} update completed", doc::getId);
119            currentWorker.notifyProgress(idx, clearCount);
120        });
121        log.debug("End scrolling to init quotas, documentCount={}", initCount);
122
123        // if recomputing only for descendants of a given path, recompute ancestors from their direct children
124        if (path != null) {
125            DocumentModel doc = root;
126            do {
127                doc = session.getDocument(doc.getParentRef());
128                initDocumentFromChildren(doc);
129            } while (!doc.getPathAsString().equals("/"));
130        }
131    }
132
133    protected long scrollAndDo(CoreSession session, String query, int scrollSize, int scrollKeepAlive,
134            BiConsumer<String, Long> consumer) {
135        long count = 0;
136        ScrollResult<String> scroll = session.scroll(query, scrollSize, scrollKeepAlive);
137        while (scroll.hasResults()) {
138            for (String uuid : scroll.getResults()) {
139                consumer.accept(uuid, ++count);
140            }
141            // commit current scroll
142            session.save();
143            TransactionHelper.commitOrRollbackTransaction();
144            TransactionHelper.startTransaction();
145            // next scroll
146            scroll = session.scroll(scroll.getScrollId());
147        }
148        return count;
149    }
150
151    protected void clearQuotas(CoreSession session, String docID) {
152        DocumentModel doc = session.getDocument(new IdRef(docID));
153        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
154        if (quotaDoc != null) {
155            quotaDoc.clearInfos();
156            quotaDoc.save();
157        }
158    }
159
160    protected void initDocument(CoreSession session, DocumentModel doc) {
161        boolean isDeleted = doc.isTrashed();
162        long size = getBlobsSize(doc);
163        long versionsSize = getVersionsSize(session, doc);
164        updateDocumentAndAncestors(session, doc, size, size + versionsSize, isDeleted ? size : 0, versionsSize);
165    }
166
167    protected void initDocumentFromChildren(DocumentModel doc) {
168        CoreSession session = doc.getCoreSession();
169        boolean isDeleted = doc.isTrashed();
170        long innerSize = getBlobsSize(doc);
171        long versionsSize = getVersionsSize(session, doc);
172        long totalSize = innerSize + versionsSize;
173        long trashSize = isDeleted ? innerSize : 0;
174        for (DocumentModel child : session.getChildren(doc.getRef())) {
175            QuotaAware quotaDoc = child.getAdapter(QuotaAware.class);
176            if (quotaDoc == null) {
177                continue;
178            }
179            totalSize += quotaDoc.getTotalSize();
180            trashSize += quotaDoc.getTrashSize();
181            versionsSize += quotaDoc.getVersionsSize();
182        }
183        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
184        if (quotaDoc == null) {
185            quotaDoc = QuotaAwareDocumentFactory.make(doc);
186        }
187        quotaDoc.setAll(innerSize, totalSize, trashSize, versionsSize);
188        quotaDoc.save();
189    }
190
191    @Override
192    protected void handleQuotaExceeded(QuotaExceededException e, Event event) {
193        String msg = "Current event " + event.getName() + " would break Quota restriction, rolling back";
194        log.info(msg);
195        e.addInfo(msg);
196        event.markRollBack("Quota Exceeded", e);
197    }
198
199    @Override
200    protected void processDocumentCreated(CoreSession session, DocumentModel doc) {
201        if (doc.isVersion()) {
202            // version taken into account by checkout
203            return;
204        }
205        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
206        if (quotaDoc == null) {
207            // always add the quota facet
208            quotaDoc = QuotaAwareDocumentFactory.make(doc);
209            quotaDoc.save();
210        }
211        long size = getBlobsSize(doc);
212        checkQuota(session, doc, size);
213        updateDocumentAndAncestors(session, doc, size, size, 0, 0);
214    }
215
216    @Override
217    protected void processDocumentCheckedIn(CoreSession session, DocumentModel doc) {
218        // nothing to do, we do things at aboutToCheckIn time
219    }
220
221    @Override
222    protected void processDocumentBeforeCheckedIn(CoreSession session, DocumentModel doc) {
223        // on checkin the versions size is incremented (and also the total)
224        long size = getBlobsSize(doc);
225        checkQuota(session, doc, size);
226        // detect if we're currently saving the document or just checking it in
227        boolean allowSave = doc.getContextData().containsKey(VERSIONING_OPTION);
228        updateDocument(doc, 0, size, 0, size, allowSave);
229        updateAncestors(session, doc, size, 0, size);
230    }
231
232    @Override
233    protected void processDocumentCheckedOut(CoreSession session, DocumentModel doc) {
234        // nothing to do, checking out the document doesn't change size
235    }
236
237    @Override
238    protected void processDocumentBeforeCheckedOut(CoreSession session, DocumentModel doc) {
239        // nothing to do, checking out the document doesn't change size
240    }
241
242    @Override
243    protected void processDocumentUpdated(CoreSession session, DocumentModel doc) {
244        // nothing to do, we do things at beforeDocumentModification time
245    }
246
247    @Override
248    protected void processDocumentBeforeUpdate(CoreSession session, DocumentModel doc) {
249        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
250        long oldSize = quotaDoc == null ? 0 : quotaDoc.getInnerSize();
251        long delta = getBlobsSize(doc) - oldSize;
252        checkQuota(session, doc, delta);
253        updateDocument(doc, delta, delta, 0, 0, false); // DO NOT SAVE as this is a "before" event
254        updateAncestors(session, doc, delta, 0, 0);
255    }
256
257    @Override
258    protected void processDocumentCopied(CoreSession session, DocumentModel doc) {
259        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
260        if (quotaDoc == null) {
261            return;
262        }
263        long size = quotaDoc.getTotalSize() - quotaDoc.getVersionsSize() - quotaDoc.getTrashSize();
264        checkQuota(session, doc, size);
265        if (!doc.isFolder() && size > 0) {
266            // when we copy some doc that is not folderish, we don't
267            // copy the versions so we can't rely on the copied quotaDocInfo
268            quotaDoc.resetInfos();
269            quotaDoc.save();
270            updateDocument(doc, size, size, 0, 0);
271        }
272        updateAncestors(session, doc, size, 0, 0);
273    }
274
275    @Override
276    protected void processDocumentMoved(CoreSession session, DocumentModel doc, DocumentModel sourceParent) {
277        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
278        long size = quotaDoc == null ? 0 : quotaDoc.getTotalSize();
279        checkQuota(session, doc, size);
280        long versionsSize = quotaDoc == null ? 0 : quotaDoc.getVersionsSize();
281        // add on new ancestors
282        updateAncestors(session, doc, size, 0, versionsSize);
283        // remove from old ancestors
284        if (sourceParent != null) {
285            updateDocumentAndAncestors(session, sourceParent, 0, -size, 0, -versionsSize);
286        }
287    }
288
289    @Override
290    protected void processDocumentAboutToBeRemoved(CoreSession session, DocumentModel doc) {
291        if (doc.isVersion()) {
292            // for versions we need to decrement the live doc + its parents
293            // We only have to decrement the inner size of this doc
294            // we do not write the right quota on the version, so we need to recompute it instead of
295            // quotaDoc#getInnerSize
296            long size = getBlobsSize(doc);
297            String sourceId = doc.getSourceId();
298            if (size > 0 && sourceId != null) {
299                DocumentModel liveDoc = session.getDocument(new IdRef(sourceId));
300                updateDocumentAndAncestors(session, liveDoc, 0, -size, 0, -size);
301            }
302            return;
303        }
304
305        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
306        long size;
307        long versionsSize;
308        if (quotaDoc == null) {
309            // the document could have been just created and the previous computation
310            // hasn't finished yet, see NXP-13665
311            size = getBlobsSize(doc);
312            versionsSize = 0;
313            log.trace("Document {} doesn't have the facet quotaDoc. Compute impacted size: {}", doc.getId(), size);
314        } else {
315            size = quotaDoc.getTotalSize();
316            versionsSize = quotaDoc.getVersionsSize();
317            log.trace("Found facet quotaDoc on document  {}. Total size: {} and versions size: {}", doc.getId(), size,
318                    versionsSize);
319        }
320        // remove size for all its versions from sizeVersions on parents
321        boolean isDeleted = doc.isTrashed();
322        // when permanently deleting the doc clean the trash if the doc is in the trash
323        // and all archived versions size
324        log.trace("Processing document about to be removed on parents. Total: {}, trash size: {}, versions size: ",
325                size, isDeleted ? size : 0, versionsSize);
326        long deltaTrash = isDeleted ? versionsSize - size : 0;
327        updateAncestors(session, doc, -size, deltaTrash, -versionsSize);
328    }
329
330    @Override
331    protected void processDocumentTrashOp(CoreSession session, DocumentModel doc, boolean isTrashed) {
332        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
333        if (quotaDoc == null) {
334            return;
335        }
336        long size = quotaDoc.getInnerSize();
337        if (log.isTraceEnabled()) {
338            if (quotaDoc.getDoc().isFolder()) {
339                log.trace(quotaDoc.getDoc().getPathAsString() + " is a folder, just inner size (" + size
340                        + ") taken into account for trash size");
341            }
342        }
343        long delta = isTrashed ? size : -size;
344        // constraints check not needed, since the documents stays in the same folder
345        updateDocumentAndAncestors(session, doc, 0, 0, delta, 0);
346    }
347
348    @Override
349    protected void processDocumentBeforeRestore(CoreSession session, DocumentModel doc) {
350        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
351        if (quotaDoc == null) {
352            return;
353        }
354        // remove versions size on parents since they will be recalculated on restore
355        long size = quotaDoc.getTotalSize();
356        long versionsSize = quotaDoc.getVersionsSize();
357        updateAncestors(session, doc, -size, 0, -versionsSize);
358    }
359
360    @Override
361    protected void processDocumentRestored(CoreSession session, DocumentModel doc) {
362        QuotaAware quotaDoc = QuotaAwareDocumentFactory.make(doc);
363        quotaDoc.resetInfos();
364        quotaDoc.save();
365        long size = getBlobsSize(doc);
366        long versionsSize = getVersionsSize(session, doc);
367        updateDocumentAndAncestors(session, doc, size, size + versionsSize, 0, versionsSize);
368    }
369
370    @Override
371    protected boolean needToProcessEventOnDocument(Event event, DocumentModel doc) {
372        if (doc == null) {
373            return false;
374        }
375        if (doc.isProxy()) {
376            return false;
377        }
378        // avoid reentrancy
379        return !Boolean.TRUE.equals(doc.getContextData(DISABLE_QUOTA_CHECK_LISTENER));
380    }
381
382    /** Checks the size delta against the maximum quota specified for this document or an ancestor. */
383    protected void checkQuota(CoreSession session, DocumentModel doc, long delta) {
384        if (delta <= 0) {
385            return;
386        }
387        for (DocumentModel parent : getAncestors(session, doc)) {
388            log.trace("processing {} {}", parent::getId, parent::getPathAsString);
389            QuotaAware quotaDoc = parent.getAdapter(QuotaAware.class);
390            // when enabling quota on user workspaces, the max size set on the
391            // UserWorkspacesRoot is the max size set on every user workspace
392            if (quotaDoc == null || quotaDoc.getMaxQuota() <= 0 || USER_WORKSPACES_ROOT.equals(parent.getType())) {
393                continue;
394            }
395            if (quotaDoc.getTotalSize() + delta > quotaDoc.getMaxQuota()) {
396                log.info("Raising Quota Exception on {} ({})", doc::getId, doc::getPathAsString);
397                throw new QuotaExceededException(parent, doc, quotaDoc.getMaxQuota());
398            }
399        }
400    }
401
402    /** Gets the sum of all blobs sizes for all the document's versions. */
403    protected long getVersionsSize(CoreSession session, DocumentModel doc) {
404        long versionsSize = 0;
405        for (DocumentModel version : session.getVersions(doc.getRef())) {
406            versionsSize += getBlobsSize(version);
407        }
408        return versionsSize;
409    }
410
411    /** Gets the sum of all blobs sizes for this document. */
412    protected long getBlobsSize(DocumentModel doc) {
413        long size = 0;
414        for (Blob blob : getAllBlobs(doc)) {
415            size += blob.getLength();
416        }
417        return size;
418    }
419
420    /** Returns the list of blobs for this document. */
421    protected List<Blob> getAllBlobs(DocumentModel doc) {
422        QuotaSizeService sizeService = Framework.getService(QuotaSizeService.class);
423        Collection<String> excludedPaths = sizeService.getExcludedPathList();
424        BlobsExtractor extractor = new BlobsExtractor();
425        extractor.setExtractorProperties(null, new HashSet<>(excludedPaths), true);
426        return extractor.getBlobs(doc);
427    }
428
429    protected void updateDocument(DocumentModel doc, long deltaInner, long deltaTotal, long deltaTrash,
430            long deltaVersions) {
431        updateDocument(doc, deltaInner, deltaTotal, deltaTrash, deltaVersions, true);
432    }
433
434    protected void updateDocument(DocumentModel doc, long deltaInner, long deltaTotal, long deltaTrash,
435            long deltaVersions, boolean allowSave) {
436        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
437        boolean save = false;
438        if (quotaDoc == null) {
439            log.trace("   add quota on: {} ({})", doc::getId, doc::getPathAsString);
440            quotaDoc = QuotaAwareDocumentFactory.make(doc);
441            save = true;
442        } else {
443            log.trace("   update quota on: {} ({}) ({})", doc::getId, doc::getPathAsString, quotaDoc::getQuotaInfo);
444        }
445        if (deltaInner != 0) {
446            quotaDoc.addInnerSize(deltaInner);
447            save = true;
448        }
449        if (deltaTotal != 0) {
450            quotaDoc.addTotalSize(deltaTotal);
451            save = true;
452        }
453        if (deltaTrash != 0) {
454            quotaDoc.addTrashSize(deltaTrash);
455            save = true;
456        }
457        if (deltaVersions != 0) {
458            quotaDoc.addVersionsSize(deltaVersions);
459            save = true;
460        }
461        if (save && allowSave) {
462            quotaDoc.save();
463        }
464        log.trace("     ==> {} ({}) ({})", doc::getId, doc::getPathAsString, quotaDoc::getQuotaInfo);
465    }
466
467    protected void updateAncestors(CoreSession session, DocumentModel doc, long deltaTotal, long deltaTrash,
468            long deltaVersions) {
469        if (deltaTotal == 0 && deltaTrash == 0 && deltaVersions == 0) {
470            // avoids computing ancestors if there's no update to do
471            return;
472        }
473        List<DocumentModel> ancestors = getAncestors(session, doc);
474        for (DocumentModel ancestor : ancestors) {
475            updateDocument(ancestor, 0, deltaTotal, deltaTrash, deltaVersions);
476        }
477    }
478
479    protected void updateDocumentAndAncestors(CoreSession session, DocumentModel doc, long deltaInner, long deltaTotal,
480            long deltaTrash, long deltaVersions) {
481        updateDocument(doc, deltaInner, deltaTotal, deltaTrash, deltaVersions);
482        updateAncestors(session, doc, deltaTotal, deltaTrash, deltaVersions);
483    }
484
485}