001/*
002 * (C) Copyright 2006-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Thierry Delprat
018 *     Florent Guillaume
019 */
020package org.nuxeo.ecm.quota.size;
021
022import java.io.Serializable;
023import java.util.Collection;
024import java.util.HashSet;
025import java.util.List;
026import java.util.Map;
027
028import org.apache.commons.logging.Log;
029import org.apache.commons.logging.LogFactory;
030import org.nuxeo.ecm.core.api.Blob;
031import org.nuxeo.ecm.core.api.CoreSession;
032import org.nuxeo.ecm.core.api.DocumentModel;
033import org.nuxeo.ecm.core.api.IdRef;
034import org.nuxeo.ecm.core.api.IterableQueryResult;
035import org.nuxeo.ecm.core.api.PathRef;
036import org.nuxeo.ecm.core.event.Event;
037import org.nuxeo.ecm.core.query.sql.NXQL;
038import org.nuxeo.ecm.core.utils.BlobsExtractor;
039import org.nuxeo.ecm.quota.AbstractQuotaStatsUpdater;
040import org.nuxeo.ecm.quota.QuotaStatsInitialWork;
041import org.nuxeo.runtime.api.Framework;
042
043/**
044 * {@link org.nuxeo.ecm.quota.QuotaStatsUpdater} counting space used by Blobs in document. This implementation does not
045 * track the space used by non-Blob properties.
046 *
047 * @since 8.3
048 */
049public class DocumentsSizeUpdater extends AbstractQuotaStatsUpdater {
050
051    private static Log log = LogFactory.getLog(DocumentsSizeUpdater.class);
052
053    public static final String DISABLE_QUOTA_CHECK_LISTENER = "disableQuotaListener";
054
055    public static final String USER_WORKSPACES_ROOT = "UserWorkspacesRoot";
056
057    @Override
058    public void computeInitialStatistics(CoreSession session, QuotaStatsInitialWork currentWorker, String path) {
059        log.debug("Starting initial Quota computation for path: " + path);
060        String query = "SELECT ecm:uuid FROM Document WHERE ecm:isVersion = 0 AND ecm:isProxy = 0";
061        DocumentModel root;
062        if (path == null) {
063            root = session.getRootDocument();
064        } else {
065            root = session.getDocument(new PathRef(path));
066            query += " AND ecm:path STARTSWITH " + NXQL.escapeString(path);
067        }
068        // reset on all documents
069        // this will force an update if the quota addon was installed and then removed
070        long count;
071        IterableQueryResult res = session.queryAndFetch(query, "NXQL");
072        try {
073            count = res.size();
074            log.debug("Start iteration on " + count + " items");
075            for (Map<String, Serializable> r : res) {
076                String uuid = (String) r.get("ecm:uuid");
077                clearQuotas(session, uuid);
078            }
079        } finally {
080            res.close();
081        }
082        clearQuotas(session, root.getId());
083        session.save();
084
085        // recompute quota on each doc
086        res = session.queryAndFetch(query, "NXQL");
087        try {
088            long idx = 0;
089            for (Map<String, Serializable> r : res) {
090                String uuid = (String) r.get("ecm:uuid");
091                DocumentModel doc = session.getDocument(new IdRef(uuid));
092                if (log.isTraceEnabled()) {
093                    log.trace("process Quota initial computation on uuid " + doc.getId());
094                    log.trace("doc with uuid " + doc.getId() + " started update");
095                }
096                initDocument(session, doc);
097                if (log.isTraceEnabled()) {
098                    log.trace("doc with uuid " + doc.getId() + " update completed");
099                }
100                currentWorker.notifyProgress(++idx, count);
101            }
102        } finally {
103            res.close();
104        }
105
106        // if recomputing only for descendants of a given path, recompute ancestors from their direct children
107        if (path != null) {
108            DocumentModel doc = root;
109            do {
110                doc = session.getDocument(doc.getParentRef());
111                initDocumentFromChildren(doc);
112            } while (!doc.getPathAsString().equals("/"));
113        }
114    }
115
116    protected void clearQuotas(CoreSession session, String docID) {
117        DocumentModel doc = session.getDocument(new IdRef(docID));
118        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
119        if (quotaDoc != null) {
120            quotaDoc.clearInfos();
121            quotaDoc.save();
122        }
123    }
124
125    protected void initDocument(CoreSession session, DocumentModel doc) {
126        boolean isDeleted = doc.isTrashed();
127        long size = getBlobsSize(doc);
128        long versionsSize = getVersionsSize(session, doc);
129        updateDocumentAndAncestors(session, doc, size, size + versionsSize, isDeleted ? size : 0, versionsSize);
130    }
131
132    protected void initDocumentFromChildren(DocumentModel doc) {
133        CoreSession session = doc.getCoreSession();
134        boolean isDeleted = doc.isTrashed();
135        long innerSize = getBlobsSize(doc);
136        long versionsSize = getVersionsSize(session, doc);
137        long totalSize = innerSize + versionsSize;
138        long trashSize = isDeleted ? innerSize : 0;
139        for (DocumentModel child : session.getChildren(doc.getRef())) {
140            QuotaAware quotaDoc = child.getAdapter(QuotaAware.class);
141            if (quotaDoc == null) {
142                continue;
143            }
144            totalSize += quotaDoc.getTotalSize();
145            trashSize += quotaDoc.getTrashSize();
146            versionsSize += quotaDoc.getVersionsSize();
147        }
148        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
149        if (quotaDoc == null) {
150            quotaDoc = QuotaAwareDocumentFactory.make(doc);
151        }
152        quotaDoc.setAll(innerSize, totalSize, trashSize, versionsSize);
153        quotaDoc.save();
154    }
155
156    @Override
157    protected void handleQuotaExceeded(QuotaExceededException e, Event event) {
158        String msg = "Current event " + event.getName() + " would break Quota restriction, rolling back";
159        log.info(msg);
160        e.addInfo(msg);
161        event.markRollBack("Quota Exceeded", e);
162    }
163
164    @Override
165    protected void processDocumentCreated(CoreSession session, DocumentModel doc) {
166        if (doc.isVersion()) {
167            // version taken into account by checkout
168            return;
169        }
170        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
171        if (quotaDoc == null) {
172            // always add the quota facet
173            quotaDoc = QuotaAwareDocumentFactory.make(doc);
174            quotaDoc.save();
175        }
176        long size = getBlobsSize(doc);
177        checkQuota(session, doc, size);
178        updateDocumentAndAncestors(session, doc, size, size, 0, 0);
179    }
180
181    @Override
182    protected void processDocumentCheckedIn(CoreSession session, DocumentModel doc) {
183        // on checkin the versions size is incremented (and also the total)
184        long size = getBlobsSize(doc);
185        // no constraints check as total size is not impacted
186        updateDocumentAndAncestors(session, doc, 0, size, 0, size);
187    }
188
189    @Override
190    protected void processDocumentCheckedOut(CoreSession session, DocumentModel doc) {
191        // on checkout we account in the total for the last version size
192        long size = getBlobsSize(doc);
193        checkQuota(session, doc, size);
194        // all quota computation handled on checkin
195    }
196
197    @Override
198    protected void processDocumentUpdated(CoreSession session, DocumentModel doc) {
199        // nothing to do, we do things at beforeDocumentModification time
200    }
201
202    @Override
203    protected void processDocumentBeforeUpdate(CoreSession session, DocumentModel doc) {
204        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
205        long oldSize = quotaDoc == null ? 0 : quotaDoc.getInnerSize();
206        long delta = getBlobsSize(doc) - oldSize;
207        checkQuota(session, doc, delta);
208        updateDocument(doc, delta, delta, 0, 0, false); // DO NOT SAVE as this is a "before" event
209        updateAncestors(session, doc, delta, 0, 0);
210    }
211
212    @Override
213    protected void processDocumentCopied(CoreSession session, DocumentModel doc) {
214        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
215        if (quotaDoc == null) {
216            return;
217        }
218        long size = quotaDoc.getTotalSize() - quotaDoc.getVersionsSize() - quotaDoc.getTrashSize();
219        checkQuota(session, doc, size);
220        if (!doc.isFolder() && size > 0) {
221            // when we copy some doc that is not folderish, we don't
222            // copy the versions so we can't rely on the copied quotaDocInfo
223            quotaDoc.resetInfos();
224            quotaDoc.save();
225            updateDocument(doc, size, size, 0, 0);
226        }
227        updateAncestors(session, doc, size, 0, 0);
228    }
229
230    @Override
231    protected void processDocumentMoved(CoreSession session, DocumentModel doc, DocumentModel sourceParent) {
232        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
233        long size = quotaDoc == null ? 0 : quotaDoc.getTotalSize();
234        checkQuota(session, doc, size);
235        long versionsSize = quotaDoc == null ? 0 : quotaDoc.getVersionsSize();
236        // add on new ancestors
237        updateAncestors(session, doc, size, 0, versionsSize);
238        // remove from old ancestors
239        if (sourceParent != null) {
240            updateDocumentAndAncestors(session, sourceParent, 0, -size, 0, -versionsSize);
241        }
242    }
243
244    @Override
245    protected void processDocumentAboutToBeRemoved(CoreSession session, DocumentModel doc) {
246        if (doc.isVersion()) {
247            // for versions we need to decrement the live doc + its parents
248            // We only have to decrement the inner size of this doc
249            // we do not write the right quota on the version, so we need to recompute it instead of
250            // quotaDoc#getInnerSize
251            long size = getBlobsSize(doc);
252            String sourceId = doc.getSourceId();
253            if (size > 0 && sourceId != null) {
254                DocumentModel liveDoc = session.getDocument(new IdRef(sourceId));
255                updateDocumentAndAncestors(session, liveDoc, 0, -size, 0, -size);
256            }
257            return;
258        }
259
260        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
261        long size;
262        long versionsSize;
263        if (quotaDoc == null) {
264            // the document could have been just created and the previous computation
265            // hasn't finished yet, see NXP-13665
266            size = getBlobsSize(doc);
267            versionsSize = 0;
268            if (log.isTraceEnabled()) {
269                log.trace(
270                        "Document " + doc.getId() + " doesn't have the facet quotaDoc. Compute impacted size:" + size);
271            }
272        } else {
273            size = quotaDoc.getTotalSize();
274            versionsSize = quotaDoc.getVersionsSize();
275            if (log.isTraceEnabled()) {
276                log.trace("Found facet quotaDoc on document  " + doc.getId() + ". Total size: " + size
277                        + " and versions size: " + versionsSize);
278            }
279        }
280        // remove size for all its versions from sizeVersions on parents
281        boolean isDeleted = doc.isTrashed();
282        // when permanently deleting the doc clean the trash if the doc is in the trash
283        // and all archived versions size
284        if (log.isTraceEnabled()) {
285            log.trace("Processing document about to be removed on parents. Total: " + size + " , trash size: "
286                    + (isDeleted ? size : 0) + " , versions size: " + versionsSize);
287        }
288        long deltaTrash = isDeleted ? versionsSize - size : 0;
289        updateAncestors(session, doc, -size, deltaTrash, -versionsSize);
290    }
291
292    @Override
293    protected void processDocumentTrashOp(CoreSession session, DocumentModel doc, boolean isTrashed) {
294        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
295        if (quotaDoc == null) {
296            return;
297        }
298        long size = quotaDoc.getInnerSize();
299        if (log.isTraceEnabled()) {
300            if (quotaDoc.getDoc().isFolder()) {
301                log.trace(quotaDoc.getDoc().getPathAsString() + " is a folder, just inner size (" + size
302                        + ") taken into account for trash size");
303            }
304        }
305        long delta = isTrashed ? size : -size;
306        // constraints check not needed, since the documents stays in the same folder
307        updateDocumentAndAncestors(session, doc, 0, 0, delta, 0);
308    }
309
310    @Override
311    protected void processDocumentBeforeRestore(CoreSession session, DocumentModel doc) {
312        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
313        if (quotaDoc == null) {
314            return;
315        }
316        // remove versions size on parents since they will be recalculated on restore
317        long size = quotaDoc.getTotalSize();
318        long versionsSize = quotaDoc.getVersionsSize();
319        updateAncestors(session, doc, -size, 0, -versionsSize);
320    }
321
322    @Override
323    protected void processDocumentRestored(CoreSession session, DocumentModel doc) {
324        QuotaAware quotaDoc = QuotaAwareDocumentFactory.make(doc);
325        quotaDoc.resetInfos();
326        quotaDoc.save();
327        long size = getBlobsSize(doc);
328        long versionsSize = getVersionsSize(session, doc);
329        updateDocumentAndAncestors(session, doc, size, size + versionsSize, 0, versionsSize);
330    }
331
332    @Override
333    protected boolean needToProcessEventOnDocument(Event event, DocumentModel doc) {
334        if (doc == null) {
335            return false;
336        }
337        if (doc.isProxy()) {
338            return false;
339        }
340        // avoid reentrancy
341        return !Boolean.TRUE.equals(doc.getContextData(DISABLE_QUOTA_CHECK_LISTENER));
342    }
343
344    /** Checks the size delta against the maximum quota specified for this document or an ancestor. */
345    protected void checkQuota(CoreSession session, DocumentModel doc, long delta) {
346        if (delta <= 0) {
347            return;
348        }
349        for (DocumentModel parent : getAncestors(session, doc)) {
350            if (log.isTraceEnabled()) {
351                log.trace("processing " + parent.getId() + " " + parent.getPathAsString());
352            }
353            QuotaAware quotaDoc = parent.getAdapter(QuotaAware.class);
354            // when enabling quota on user workspaces, the max size set on the
355            // UserWorkspacesRoot is the max size set on every user workspace
356            if (quotaDoc == null || quotaDoc.getMaxQuota() <= 0 || USER_WORKSPACES_ROOT.equals(parent.getType())) {
357                continue;
358            }
359            if (quotaDoc.getTotalSize() + delta > quotaDoc.getMaxQuota()) {
360                log.info("Raising Quota Exception on " + doc.getId() + " (" + doc.getPathAsString() + ")");
361                throw new QuotaExceededException(parent, doc, quotaDoc.getMaxQuota());
362            }
363        }
364    }
365
366    /** Gets the sum of all blobs sizes for all the document's versions. */
367    protected long getVersionsSize(CoreSession session, DocumentModel doc) {
368        long versionsSize = 0;
369        for (DocumentModel version : session.getVersions(doc.getRef())) {
370            versionsSize += getBlobsSize(version);
371        }
372        return versionsSize;
373    }
374
375    /** Gets the sum of all blobs sizes for this document. */
376    protected long getBlobsSize(DocumentModel doc) {
377        long size = 0;
378        for (Blob blob : getAllBlobs(doc)) {
379            size += blob.getLength();
380        }
381        return size;
382    }
383
384    /** Returns the list of blobs for this document. */
385    protected List<Blob> getAllBlobs(DocumentModel doc) {
386        QuotaSizeService sizeService = Framework.getService(QuotaSizeService.class);
387        Collection<String> excludedPaths = sizeService.getExcludedPathList();
388        BlobsExtractor extractor = new BlobsExtractor();
389        extractor.setExtractorProperties(null, new HashSet<>(excludedPaths), true);
390        return extractor.getBlobs(doc);
391    }
392
393    protected void updateDocument(DocumentModel doc, long deltaInner, long deltaTotal, long deltaTrash,
394            long deltaVersions) {
395        updateDocument(doc, deltaInner, deltaTotal, deltaTrash, deltaVersions, true);
396    }
397
398    protected void updateDocument(DocumentModel doc, long deltaInner, long deltaTotal, long deltaTrash,
399            long deltaVersions, boolean allowSave) {
400        QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class);
401        boolean save = false;
402        if (quotaDoc == null) {
403            if (log.isTraceEnabled()) {
404                log.trace("   add quota on: " + doc.getId() + " (" + doc.getPathAsString() + ")");
405            }
406            quotaDoc = QuotaAwareDocumentFactory.make(doc);
407            save = true;
408        } else {
409            if (log.isTraceEnabled()) {
410                log.trace("   update quota on: " + doc.getId() + " (" + doc.getPathAsString() + ") ("
411                        + quotaDoc.getQuotaInfo() + ")");
412            }
413        }
414        if (deltaInner != 0) {
415            quotaDoc.addInnerSize(deltaInner);
416            save = true;
417        }
418        if (deltaTotal != 0) {
419            quotaDoc.addTotalSize(deltaTotal);
420            save = true;
421        }
422        if (deltaTrash != 0) {
423            quotaDoc.addTrashSize(deltaTrash);
424            save = true;
425        }
426        if (deltaVersions != 0) {
427            quotaDoc.addVersionsSize(deltaVersions);
428            save = true;
429        }
430        if (save && allowSave) {
431            quotaDoc.save();
432        }
433        if (log.isTraceEnabled()) {
434            log.trace("     ==> " + doc.getId() + " (" + doc.getPathAsString() + ") (" + quotaDoc.getQuotaInfo() + ")");
435        }
436    }
437
438    protected void updateAncestors(CoreSession session, DocumentModel doc, long deltaTotal, long deltaTrash,
439            long deltaVersions) {
440        if (deltaTotal == 0 && deltaTrash == 0 && deltaVersions == 0) {
441            // avoids computing ancestors if there's no update to do
442            return;
443        }
444        List<DocumentModel> ancestors = getAncestors(session, doc);
445        for (DocumentModel ancestor : ancestors) {
446            updateDocument(ancestor, 0, deltaTotal, deltaTrash, deltaVersions);
447        }
448    }
449
450    protected void updateDocumentAndAncestors(CoreSession session, DocumentModel doc, long deltaInner, long deltaTotal,
451            long deltaTrash, long deltaVersions) {
452        updateDocument(doc, deltaInner, deltaTotal, deltaTrash, deltaVersions);
453        updateAncestors(session, doc, deltaTotal, deltaTrash, deltaVersions);
454    }
455
456}