001/* 002 * (C) Copyright 2006-2019 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Thierry Delprat 018 * Florent Guillaume 019 */ 020package org.nuxeo.ecm.quota.size; 021 022import static org.nuxeo.ecm.core.api.versioning.VersioningService.VERSIONING_OPTION; 023 024import java.util.Collection; 025import java.util.HashSet; 026import java.util.List; 027import java.util.function.BiConsumer; 028 029import org.apache.logging.log4j.LogManager; 030import org.apache.logging.log4j.Logger; 031import org.nuxeo.ecm.core.api.Blob; 032import org.nuxeo.ecm.core.api.CoreSession; 033import org.nuxeo.ecm.core.api.DocumentModel; 034import org.nuxeo.ecm.core.api.IdRef; 035import org.nuxeo.ecm.core.api.PathRef; 036import org.nuxeo.ecm.core.api.ScrollResult; 037import org.nuxeo.ecm.core.event.Event; 038import org.nuxeo.ecm.core.query.sql.NXQL; 039import org.nuxeo.ecm.core.utils.BlobsExtractor; 040import org.nuxeo.ecm.quota.AbstractQuotaStatsUpdater; 041import org.nuxeo.ecm.quota.QuotaStatsInitialWork; 042import org.nuxeo.runtime.api.Framework; 043import org.nuxeo.runtime.services.config.ConfigurationService; 044import org.nuxeo.runtime.transaction.TransactionHelper; 045 046/** 047 * {@link org.nuxeo.ecm.quota.QuotaStatsUpdater} counting space used by Blobs in document. This implementation does not 048 * track the space used by non-Blob properties. 049 * 050 * @since 8.3 051 */ 052public class DocumentsSizeUpdater extends AbstractQuotaStatsUpdater { 053 private static Logger log = LogManager.getLogger(DocumentsSizeUpdater.class); 054 055 public static final String DISABLE_QUOTA_CHECK_LISTENER = "disableQuotaListener"; 056 057 public static final String USER_WORKSPACES_ROOT = "UserWorkspacesRoot"; 058 059 /** @since 11.1 */ 060 public static final String CLEAR_SCROLL_SIZE_PROP = "nuxeo.quota.clear.scroll.size"; 061 062 /** @since 11.1 */ 063 public static final int DEFAULT_CLEAR_SCROLL_SIZE = 500; 064 065 /** @since 11.1 */ 066 public static final String CLEAR_SCROLL_KEEP_ALIVE_PROP = "nuxeo.quota.clear.scroll.keepAliveSeconds"; 067 068 /** @since 11.1 */ 069 public static final int DEFAULT_CLEAR_SCROLL_KEEP_ALIVE = 60; 070 071 /** @since 11.1 */ 072 public static final String INIT_SCROLL_SIZE_PROP = "nuxeo.quota.init.scroll.size"; 073 074 /** @since 11.1 */ 075 public static final int DEFAULT_INIT_SCROLL_SIZE = 250; 076 077 /** @since 11.1 */ 078 public static final String INIT_SCROLL_KEEP_ALIVE_PROP = "nuxeo.quota.init.scroll.keepAliveSeconds"; 079 080 /** @since 11.1 */ 081 public static final int DEFAULT_INIT_SCROLL_KEEP_ALIVE = 120; 082 083 @Override 084 public void computeInitialStatistics(CoreSession session, QuotaStatsInitialWork currentWorker, String path) { 085 log.debug("Starting initial Quota computation for path: {}", path); 086 String query = "SELECT ecm:uuid FROM Document WHERE ecm:isVersion = 0 AND ecm:isProxy = 0"; 087 DocumentModel root; 088 if (path == null) { 089 root = session.getRootDocument(); 090 } else { 091 root = session.getDocument(new PathRef(path)); 092 query += " AND ecm:path STARTSWITH " + NXQL.escapeString(path); 093 } 094 // get scroll configuration parameters 095 ConfigurationService confService = Framework.getService(ConfigurationService.class); 096 int clearScrollSize = confService.getInteger(CLEAR_SCROLL_SIZE_PROP, DEFAULT_CLEAR_SCROLL_SIZE); 097 int clearScrollKeepAlive = confService.getInteger(CLEAR_SCROLL_KEEP_ALIVE_PROP, 098 DEFAULT_CLEAR_SCROLL_KEEP_ALIVE); 099 int initScrollSize = confService.getInteger(INIT_SCROLL_SIZE_PROP, DEFAULT_INIT_SCROLL_SIZE); 100 int initScrollKeepAlive = confService.getInteger(INIT_SCROLL_KEEP_ALIVE_PROP, DEFAULT_INIT_SCROLL_KEEP_ALIVE); 101 102 // reset on all documents 103 // this will force an update if the quota addon was installed and then removed 104 log.debug("Start scrolling to clear quotas"); 105 long clearCount = scrollAndDo(session, query, clearScrollSize, clearScrollKeepAlive, 106 (uuid, idx) -> clearQuotas(session, uuid)); 107 log.debug("End scrolling to clear quotas, documentCount={}", clearCount); 108 clearQuotas(session, root.getId()); 109 session.save(); 110 111 // recompute quota on each doc 112 log.debug("Start scrolling to init quotas"); 113 long initCount = scrollAndDo(session, query, initScrollSize, initScrollKeepAlive, (uuid, idx) -> { 114 DocumentModel doc = session.getDocument(new IdRef(uuid)); 115 log.trace("process Quota initial computation on uuid={}", doc::getId); 116 log.trace("doc with uuid {} started update", doc::getId); 117 initDocument(session, doc); 118 log.trace("doc with uuid {} update completed", doc::getId); 119 currentWorker.notifyProgress(idx, clearCount); 120 }); 121 log.debug("End scrolling to init quotas, documentCount={}", initCount); 122 123 // if recomputing only for descendants of a given path, recompute ancestors from their direct children 124 if (path != null) { 125 DocumentModel doc = root; 126 do { 127 doc = session.getDocument(doc.getParentRef()); 128 initDocumentFromChildren(doc); 129 } while (!doc.getPathAsString().equals("/")); 130 } 131 } 132 133 protected long scrollAndDo(CoreSession session, String query, int scrollSize, int scrollKeepAlive, 134 BiConsumer<String, Long> consumer) { 135 long count = 0; 136 ScrollResult<String> scroll = session.scroll(query, scrollSize, scrollKeepAlive); 137 while (scroll.hasResults()) { 138 for (String uuid : scroll.getResults()) { 139 consumer.accept(uuid, ++count); 140 } 141 // commit current scroll 142 session.save(); 143 TransactionHelper.commitOrRollbackTransaction(); 144 TransactionHelper.startTransaction(); 145 // next scroll 146 scroll = session.scroll(scroll.getScrollId()); 147 } 148 return count; 149 } 150 151 protected void clearQuotas(CoreSession session, String docID) { 152 DocumentModel doc = session.getDocument(new IdRef(docID)); 153 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 154 if (quotaDoc != null) { 155 quotaDoc.clearInfos(); 156 quotaDoc.save(); 157 } 158 } 159 160 protected void initDocument(CoreSession session, DocumentModel doc) { 161 boolean isDeleted = doc.isTrashed(); 162 long size = getBlobsSize(doc); 163 long versionsSize = getVersionsSize(session, doc); 164 updateDocumentAndAncestors(session, doc, size, size + versionsSize, isDeleted ? size : 0, versionsSize); 165 } 166 167 protected void initDocumentFromChildren(DocumentModel doc) { 168 CoreSession session = doc.getCoreSession(); 169 boolean isDeleted = doc.isTrashed(); 170 long innerSize = getBlobsSize(doc); 171 long versionsSize = getVersionsSize(session, doc); 172 long totalSize = innerSize + versionsSize; 173 long trashSize = isDeleted ? innerSize : 0; 174 for (DocumentModel child : session.getChildren(doc.getRef())) { 175 QuotaAware quotaDoc = child.getAdapter(QuotaAware.class); 176 if (quotaDoc == null) { 177 continue; 178 } 179 totalSize += quotaDoc.getTotalSize(); 180 trashSize += quotaDoc.getTrashSize(); 181 versionsSize += quotaDoc.getVersionsSize(); 182 } 183 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 184 if (quotaDoc == null) { 185 quotaDoc = QuotaAwareDocumentFactory.make(doc); 186 } 187 quotaDoc.setAll(innerSize, totalSize, trashSize, versionsSize); 188 quotaDoc.save(); 189 } 190 191 @Override 192 protected void handleQuotaExceeded(QuotaExceededException e, Event event) { 193 String msg = "Current event " + event.getName() + " would break Quota restriction, rolling back"; 194 log.info(msg); 195 e.addInfo(msg); 196 event.markRollBack("Quota Exceeded", e); 197 } 198 199 @Override 200 protected void processDocumentCreated(CoreSession session, DocumentModel doc) { 201 if (doc.isVersion()) { 202 // version taken into account by checkout 203 return; 204 } 205 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 206 if (quotaDoc == null) { 207 // always add the quota facet 208 quotaDoc = QuotaAwareDocumentFactory.make(doc); 209 quotaDoc.save(); 210 } 211 long size = getBlobsSize(doc); 212 checkQuota(session, doc, size); 213 updateDocumentAndAncestors(session, doc, size, size, 0, 0); 214 } 215 216 @Override 217 protected void processDocumentCheckedIn(CoreSession session, DocumentModel doc) { 218 // nothing to do, we do things at aboutToCheckIn time 219 } 220 221 @Override 222 protected void processDocumentBeforeCheckedIn(CoreSession session, DocumentModel doc) { 223 // on checkin the versions size is incremented (and also the total) 224 long size = getBlobsSize(doc); 225 checkQuota(session, doc, size); 226 // detect if we're currently saving the document or just checking it in 227 boolean allowSave = doc.getContextData().containsKey(VERSIONING_OPTION); 228 updateDocument(doc, 0, size, 0, size, allowSave); 229 updateAncestors(session, doc, size, 0, size); 230 } 231 232 @Override 233 protected void processDocumentCheckedOut(CoreSession session, DocumentModel doc) { 234 // nothing to do, checking out the document doesn't change size 235 } 236 237 @Override 238 protected void processDocumentBeforeCheckedOut(CoreSession session, DocumentModel doc) { 239 // nothing to do, checking out the document doesn't change size 240 } 241 242 @Override 243 protected void processDocumentUpdated(CoreSession session, DocumentModel doc) { 244 // nothing to do, we do things at beforeDocumentModification time 245 } 246 247 @Override 248 protected void processDocumentBeforeUpdate(CoreSession session, DocumentModel doc) { 249 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 250 long oldSize = quotaDoc == null ? 0 : quotaDoc.getInnerSize(); 251 long delta = getBlobsSize(doc) - oldSize; 252 checkQuota(session, doc, delta); 253 updateDocument(doc, delta, delta, 0, 0, false); // DO NOT SAVE as this is a "before" event 254 updateAncestors(session, doc, delta, 0, 0); 255 } 256 257 @Override 258 protected void processDocumentCopied(CoreSession session, DocumentModel doc) { 259 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 260 if (quotaDoc == null) { 261 return; 262 } 263 long size = quotaDoc.getTotalSize() - quotaDoc.getVersionsSize() - quotaDoc.getTrashSize(); 264 checkQuota(session, doc, size); 265 if (!doc.isFolder() && size > 0) { 266 // when we copy some doc that is not folderish, we don't 267 // copy the versions so we can't rely on the copied quotaDocInfo 268 quotaDoc.resetInfos(); 269 quotaDoc.save(); 270 updateDocument(doc, size, size, 0, 0); 271 } 272 updateAncestors(session, doc, size, 0, 0); 273 } 274 275 @Override 276 protected void processDocumentMoved(CoreSession session, DocumentModel doc, DocumentModel sourceParent) { 277 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 278 long size = quotaDoc == null ? 0 : quotaDoc.getTotalSize(); 279 checkQuota(session, doc, size); 280 long versionsSize = quotaDoc == null ? 0 : quotaDoc.getVersionsSize(); 281 // add on new ancestors 282 updateAncestors(session, doc, size, 0, versionsSize); 283 // remove from old ancestors 284 if (sourceParent != null) { 285 updateDocumentAndAncestors(session, sourceParent, 0, -size, 0, -versionsSize); 286 } 287 } 288 289 @Override 290 protected void processDocumentAboutToBeRemoved(CoreSession session, DocumentModel doc) { 291 if (doc.isVersion()) { 292 // for versions we need to decrement the live doc + its parents 293 // We only have to decrement the inner size of this doc 294 // we do not write the right quota on the version, so we need to recompute it instead of 295 // quotaDoc#getInnerSize 296 long size = getBlobsSize(doc); 297 String sourceId = doc.getSourceId(); 298 if (size > 0 && sourceId != null) { 299 DocumentModel liveDoc = session.getDocument(new IdRef(sourceId)); 300 updateDocumentAndAncestors(session, liveDoc, 0, -size, 0, -size); 301 } 302 return; 303 } 304 305 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 306 long size; 307 long versionsSize; 308 if (quotaDoc == null) { 309 // the document could have been just created and the previous computation 310 // hasn't finished yet, see NXP-13665 311 size = getBlobsSize(doc); 312 versionsSize = 0; 313 log.trace("Document {} doesn't have the facet quotaDoc. Compute impacted size: {}", doc.getId(), size); 314 } else { 315 size = quotaDoc.getTotalSize(); 316 versionsSize = quotaDoc.getVersionsSize(); 317 log.trace("Found facet quotaDoc on document {}. Total size: {} and versions size: {}", doc.getId(), size, 318 versionsSize); 319 } 320 // remove size for all its versions from sizeVersions on parents 321 boolean isDeleted = doc.isTrashed(); 322 // when permanently deleting the doc clean the trash if the doc is in the trash 323 // and all archived versions size 324 log.trace("Processing document about to be removed on parents. Total: {}, trash size: {}, versions size: ", 325 size, isDeleted ? size : 0, versionsSize); 326 long deltaTrash = isDeleted ? versionsSize - size : 0; 327 updateAncestors(session, doc, -size, deltaTrash, -versionsSize); 328 } 329 330 @Override 331 protected void processDocumentTrashOp(CoreSession session, DocumentModel doc, boolean isTrashed) { 332 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 333 if (quotaDoc == null) { 334 return; 335 } 336 long size = quotaDoc.getInnerSize(); 337 if (log.isTraceEnabled()) { 338 if (quotaDoc.getDoc().isFolder()) { 339 log.trace(quotaDoc.getDoc().getPathAsString() + " is a folder, just inner size (" + size 340 + ") taken into account for trash size"); 341 } 342 } 343 long delta = isTrashed ? size : -size; 344 // constraints check not needed, since the documents stays in the same folder 345 updateDocumentAndAncestors(session, doc, 0, 0, delta, 0); 346 } 347 348 @Override 349 protected void processDocumentBeforeRestore(CoreSession session, DocumentModel doc) { 350 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 351 if (quotaDoc == null) { 352 return; 353 } 354 // remove versions size on parents since they will be recalculated on restore 355 long size = quotaDoc.getTotalSize(); 356 long versionsSize = quotaDoc.getVersionsSize(); 357 updateAncestors(session, doc, -size, 0, -versionsSize); 358 } 359 360 @Override 361 protected void processDocumentRestored(CoreSession session, DocumentModel doc) { 362 QuotaAware quotaDoc = QuotaAwareDocumentFactory.make(doc); 363 quotaDoc.resetInfos(); 364 quotaDoc.save(); 365 long size = getBlobsSize(doc); 366 long versionsSize = getVersionsSize(session, doc); 367 updateDocumentAndAncestors(session, doc, size, size + versionsSize, 0, versionsSize); 368 } 369 370 @Override 371 protected boolean needToProcessEventOnDocument(Event event, DocumentModel doc) { 372 if (doc == null) { 373 return false; 374 } 375 if (doc.isProxy()) { 376 return false; 377 } 378 // avoid reentrancy 379 return !Boolean.TRUE.equals(doc.getContextData(DISABLE_QUOTA_CHECK_LISTENER)); 380 } 381 382 /** Checks the size delta against the maximum quota specified for this document or an ancestor. */ 383 protected void checkQuota(CoreSession session, DocumentModel doc, long delta) { 384 if (delta <= 0) { 385 return; 386 } 387 for (DocumentModel parent : getAncestors(session, doc)) { 388 log.trace("processing {} {}", parent::getId, parent::getPathAsString); 389 QuotaAware quotaDoc = parent.getAdapter(QuotaAware.class); 390 // when enabling quota on user workspaces, the max size set on the 391 // UserWorkspacesRoot is the max size set on every user workspace 392 if (quotaDoc == null || quotaDoc.getMaxQuota() <= 0 || USER_WORKSPACES_ROOT.equals(parent.getType())) { 393 continue; 394 } 395 if (quotaDoc.getTotalSize() + delta > quotaDoc.getMaxQuota()) { 396 log.info("Raising Quota Exception on {} ({})", doc::getId, doc::getPathAsString); 397 throw new QuotaExceededException(parent, doc, quotaDoc.getMaxQuota()); 398 } 399 } 400 } 401 402 /** Gets the sum of all blobs sizes for all the document's versions. */ 403 protected long getVersionsSize(CoreSession session, DocumentModel doc) { 404 long versionsSize = 0; 405 for (DocumentModel version : session.getVersions(doc.getRef())) { 406 versionsSize += getBlobsSize(version); 407 } 408 return versionsSize; 409 } 410 411 /** Gets the sum of all blobs sizes for this document. */ 412 protected long getBlobsSize(DocumentModel doc) { 413 long size = 0; 414 for (Blob blob : getAllBlobs(doc)) { 415 size += blob.getLength(); 416 } 417 return size; 418 } 419 420 /** Returns the list of blobs for this document. */ 421 protected List<Blob> getAllBlobs(DocumentModel doc) { 422 QuotaSizeService sizeService = Framework.getService(QuotaSizeService.class); 423 Collection<String> excludedPaths = sizeService.getExcludedPathList(); 424 BlobsExtractor extractor = new BlobsExtractor(); 425 extractor.setExtractorProperties(null, new HashSet<>(excludedPaths), true); 426 return extractor.getBlobs(doc); 427 } 428 429 protected void updateDocument(DocumentModel doc, long deltaInner, long deltaTotal, long deltaTrash, 430 long deltaVersions) { 431 updateDocument(doc, deltaInner, deltaTotal, deltaTrash, deltaVersions, true); 432 } 433 434 protected void updateDocument(DocumentModel doc, long deltaInner, long deltaTotal, long deltaTrash, 435 long deltaVersions, boolean allowSave) { 436 QuotaAware quotaDoc = doc.getAdapter(QuotaAware.class); 437 boolean save = false; 438 if (quotaDoc == null) { 439 log.trace(" add quota on: {} ({})", doc::getId, doc::getPathAsString); 440 quotaDoc = QuotaAwareDocumentFactory.make(doc); 441 save = true; 442 } else { 443 log.trace(" update quota on: {} ({}) ({})", doc::getId, doc::getPathAsString, quotaDoc::getQuotaInfo); 444 } 445 if (deltaInner != 0) { 446 quotaDoc.addInnerSize(deltaInner); 447 save = true; 448 } 449 if (deltaTotal != 0) { 450 quotaDoc.addTotalSize(deltaTotal); 451 save = true; 452 } 453 if (deltaTrash != 0) { 454 quotaDoc.addTrashSize(deltaTrash); 455 save = true; 456 } 457 if (deltaVersions != 0) { 458 quotaDoc.addVersionsSize(deltaVersions); 459 save = true; 460 } 461 if (save && allowSave) { 462 quotaDoc.save(); 463 } 464 log.trace(" ==> {} ({}) ({})", doc::getId, doc::getPathAsString, quotaDoc::getQuotaInfo); 465 } 466 467 protected void updateAncestors(CoreSession session, DocumentModel doc, long deltaTotal, long deltaTrash, 468 long deltaVersions) { 469 if (deltaTotal == 0 && deltaTrash == 0 && deltaVersions == 0) { 470 // avoids computing ancestors if there's no update to do 471 return; 472 } 473 List<DocumentModel> ancestors = getAncestors(session, doc); 474 for (DocumentModel ancestor : ancestors) { 475 updateDocument(ancestor, 0, deltaTotal, deltaTrash, deltaVersions); 476 } 477 } 478 479 protected void updateDocumentAndAncestors(CoreSession session, DocumentModel doc, long deltaInner, long deltaTotal, 480 long deltaTrash, long deltaVersions) { 481 updateDocument(doc, deltaInner, deltaTotal, deltaTrash, deltaVersions); 482 updateAncestors(session, doc, deltaTotal, deltaTrash, deltaVersions); 483 } 484 485}