001/* 002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.storage.dbs; 020 021import static java.lang.Boolean.FALSE; 022import static java.lang.Boolean.TRUE; 023import static org.nuxeo.ecm.core.api.security.SecurityConstants.BROWSE; 024import static org.nuxeo.ecm.core.api.security.SecurityConstants.EVERYONE; 025import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL; 026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_CHANGE_TOKEN; 027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_SYS_CHANGE_TOKEN; 028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_GRANT; 029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_PERMISSION; 030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_STATUS; 031import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_USER; 032import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL; 033import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP; 034import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ANCESTOR_IDS; 035import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_CHANGE_TOKEN; 036import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_JOBID; 037import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ID; 038import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_PROXY; 039import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_VERSION; 040import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_MIXIN_TYPES; 041import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_NAME; 042import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PARENT_ID; 043import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_POS; 044import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PREFIX; 045import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PRIMARY_TYPE; 046import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_IDS; 047import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_TARGET_ID; 048import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_VERSION_SERIES_ID; 049import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_READ_ACL; 050import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_SYS_CHANGE_TOKEN; 051import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_SERIES_ID; 052 053import java.io.Serializable; 054import java.security.Principal; 055import java.util.ArrayList; 056import java.util.Arrays; 057import java.util.Collection; 058import java.util.Collections; 059import java.util.HashMap; 060import java.util.HashSet; 061import java.util.LinkedHashSet; 062import java.util.LinkedList; 063import java.util.List; 064import java.util.Map; 065import java.util.Map.Entry; 066import java.util.Set; 067import java.util.stream.Stream; 068 069import org.apache.commons.lang3.StringUtils; 070import org.apache.commons.logging.Log; 071import org.apache.commons.logging.LogFactory; 072import org.nuxeo.ecm.core.BatchFinderWork; 073import org.nuxeo.ecm.core.BatchProcessorWork; 074import org.nuxeo.ecm.core.api.ConcurrentUpdateException; 075import org.nuxeo.ecm.core.api.PartialList; 076import org.nuxeo.ecm.core.api.SystemPrincipal; 077import org.nuxeo.ecm.core.api.model.DeltaLong; 078import org.nuxeo.ecm.core.api.repository.RepositoryManager; 079import org.nuxeo.ecm.core.query.QueryFilter; 080import org.nuxeo.ecm.core.query.sql.NXQL; 081import org.nuxeo.ecm.core.schema.SchemaManager; 082import org.nuxeo.ecm.core.schema.types.Schema; 083import org.nuxeo.ecm.core.security.SecurityService; 084import org.nuxeo.ecm.core.storage.BaseDocument; 085import org.nuxeo.ecm.core.storage.DefaultFulltextParser; 086import org.nuxeo.ecm.core.storage.FulltextConfiguration; 087import org.nuxeo.ecm.core.storage.FulltextParser; 088import org.nuxeo.ecm.core.storage.FulltextUpdaterWork; 089import org.nuxeo.ecm.core.storage.FulltextUpdaterWork.IndexAndText; 090import org.nuxeo.ecm.core.storage.State; 091import org.nuxeo.ecm.core.storage.State.ListDiff; 092import org.nuxeo.ecm.core.storage.State.StateDiff; 093import org.nuxeo.ecm.core.storage.StateHelper; 094import org.nuxeo.ecm.core.work.api.Work; 095import org.nuxeo.ecm.core.work.api.WorkManager; 096import org.nuxeo.ecm.core.work.api.WorkManager.Scheduling; 097import org.nuxeo.runtime.api.Framework; 098 099/** 100 * Transactional state for a session. 101 * <p> 102 * Until {@code save()} is called, data lives in the transient map. 103 * <p> 104 * Upon save, data is written to the repository, even though it has not yet been committed (this means that other 105 * sessions can read uncommitted data). It's also kept in an undo log in order for rollback to be possible. 106 * <p> 107 * On commit, the undo log is forgotten. On rollback, the undo log is replayed. 108 * 109 * @since 5.9.4 110 */ 111public class DBSTransactionState { 112 113 private static final Log log = LogFactory.getLog(DBSTransactionState.class); 114 115 private static final String KEY_UNDOLOG_CREATE = "__UNDOLOG_CREATE__\0\0"; 116 117 /** Keys used when computing Read ACLs. */ 118 protected static final Set<String> READ_ACL_RECURSION_KEYS = new HashSet<>( 119 Arrays.asList(KEY_READ_ACL, KEY_ACP, KEY_IS_VERSION, KEY_VERSION_SERIES_ID, KEY_PARENT_ID)); 120 121 public static final String READ_ACL_ASYNC_ENABLED_PROPERTY = "nuxeo.core.readacl.async.enabled"; 122 123 public static final String READ_ACL_ASYNC_ENABLED_DEFAULT = "true"; 124 125 public static final String READ_ACL_ASYNC_THRESHOLD_PROPERTY = "nuxeo.core.readacl.async.threshold"; 126 127 public static final String READ_ACL_ASYNC_THRESHOLD_DEFAULT = "500"; 128 129 protected final DBSRepository repository; 130 131 protected final DBSSession session; 132 133 /** Retrieved and created document state. */ 134 protected Map<String, DBSDocumentState> transientStates = new HashMap<>(); 135 136 /** Ids of documents created but not yet saved. */ 137 protected Set<String> transientCreated = new LinkedHashSet<>(); 138 139 /** 140 * Document ids modified as "user changes", which means that a change token should be checked. 141 * 142 * @since 9.2 143 */ 144 protected final Set<Serializable> userChangeIds = new HashSet<>(); 145 146 /** 147 * Undo log. 148 * <p> 149 * A map of document ids to null or State. The value is null when the document has to be deleted when applying the 150 * undo log. Otherwise the value is a State. If the State contains the key {@link #KEY_UNDOLOG_CREATE} then the 151 * state must be re-created completely when applying the undo log, otherwise just applied as an update. 152 * <p> 153 * Null when there is no active transaction. 154 */ 155 protected Map<String, State> undoLog; 156 157 protected final Set<String> browsePermissions; 158 159 public DBSTransactionState(DBSRepository repository, DBSSession session) { 160 this.repository = repository; 161 this.session = session; 162 SecurityService securityService = Framework.getService(SecurityService.class); 163 browsePermissions = new HashSet<>(Arrays.asList(securityService.getPermissionsToCheck(BROWSE))); 164 } 165 166 /** 167 * New transient state for something just read from the repository. 168 */ 169 protected DBSDocumentState newTransientState(State state) { 170 if (state == null) { 171 return null; 172 } 173 String id = (String) state.get(KEY_ID); 174 if (transientStates.containsKey(id)) { 175 throw new IllegalStateException("Already transient: " + id); 176 } 177 DBSDocumentState docState = new DBSDocumentState(state); // copy 178 transientStates.put(id, docState); 179 return docState; 180 } 181 182 /** 183 * Returns a state and marks it as transient, because it's about to be modified or returned to user code (where it 184 * may be modified). 185 */ 186 public DBSDocumentState getStateForUpdate(String id) { 187 // check transient state 188 DBSDocumentState docState = transientStates.get(id); 189 if (docState != null) { 190 return docState; 191 } 192 // fetch from repository 193 State state = repository.readState(id); 194 return newTransientState(state); 195 } 196 197 /** 198 * Returns a state which won't be modified. 199 */ 200 // TODO in some cases it's good to have this kept in memory instead of 201 // rereading from database every time 202 // XXX getStateForReadOneShot 203 public State getStateForRead(String id) { 204 // check transient state 205 DBSDocumentState docState = transientStates.get(id); 206 if (docState != null) { 207 return docState.getState(); 208 } 209 // fetch from repository 210 return repository.readState(id); 211 } 212 213 /** 214 * Returns states and marks them transient, because they're about to be returned to user code (where they may be 215 * modified). 216 */ 217 public List<DBSDocumentState> getStatesForUpdate(List<String> ids) { 218 // check which ones we have to fetch from repository 219 List<String> idsToFetch = new LinkedList<>(); 220 for (String id : ids) { 221 // check transient state 222 DBSDocumentState docState = transientStates.get(id); 223 if (docState != null) { 224 continue; 225 } 226 // will have to fetch it 227 idsToFetch.add(id); 228 } 229 if (!idsToFetch.isEmpty()) { 230 List<State> states = repository.readStates(idsToFetch); 231 for (State state : states) { 232 newTransientState(state); 233 } 234 } 235 // everything now fetched in transient 236 List<DBSDocumentState> docStates = new ArrayList<>(ids.size()); 237 for (String id : ids) { 238 DBSDocumentState docState = transientStates.get(id); 239 if (docState == null) { 240 if (log.isTraceEnabled()) { 241 log.trace("Cannot fetch document with id: " + id, new Throwable("debug stack trace")); 242 } 243 continue; 244 } 245 docStates.add(docState); 246 } 247 return docStates; 248 } 249 250 // XXX TODO for update or for read? 251 public DBSDocumentState getChildState(String parentId, String name) { 252 // check transient state 253 for (DBSDocumentState docState : transientStates.values()) { 254 if (!parentId.equals(docState.getParentId())) { 255 continue; 256 } 257 if (!name.equals(docState.getName())) { 258 continue; 259 } 260 return docState; 261 } 262 // fetch from repository 263 State state = repository.readChildState(parentId, name, Collections.emptySet()); 264 if (state == null) { 265 return null; 266 } 267 String id = (String) state.get(KEY_ID); 268 if (transientStates.containsKey(id)) { 269 // found transient, even though we already checked 270 // that means that in-memory it's not a child, but in-database it's a child (was moved) 271 // -> ignore the database state 272 return null; 273 } 274 return newTransientState(state); 275 } 276 277 public boolean hasChild(String parentId, String name) { 278 // check transient state 279 for (DBSDocumentState docState : transientStates.values()) { 280 if (!parentId.equals(docState.getParentId())) { 281 continue; 282 } 283 if (!name.equals(docState.getName())) { 284 continue; 285 } 286 return true; 287 } 288 // check repository 289 return repository.hasChild(parentId, name, Collections.emptySet()); 290 } 291 292 public List<DBSDocumentState> getChildrenStates(String parentId) { 293 List<DBSDocumentState> docStates = new LinkedList<>(); 294 Set<String> seen = new HashSet<>(); 295 // check transient state 296 for (DBSDocumentState docState : transientStates.values()) { 297 if (!parentId.equals(docState.getParentId())) { 298 continue; 299 } 300 docStates.add(docState); 301 seen.add(docState.getId()); 302 } 303 // fetch from repository 304 List<State> states = repository.queryKeyValue(KEY_PARENT_ID, parentId, seen); 305 for (State state : states) { 306 String id = (String) state.get(KEY_ID); 307 if (transientStates.containsKey(id)) { 308 // found transient, even though we passed an exclusion list for known children 309 // that means that in-memory it's not a child, but in-database it's a child (was moved) 310 // -> ignore the database state 311 continue; 312 } 313 docStates.add(newTransientState(state)); 314 } 315 return docStates; 316 } 317 318 public List<String> getChildrenIds(String parentId) { 319 List<String> children = new ArrayList<>(); 320 Set<String> seen = new HashSet<>(); 321 // check transient state 322 for (DBSDocumentState docState : transientStates.values()) { 323 String id = docState.getId(); 324 if (!parentId.equals(docState.getParentId())) { 325 continue; 326 } 327 seen.add(id); 328 children.add(id); 329 } 330 // fetch from repository 331 List<State> states = repository.queryKeyValue(KEY_PARENT_ID, parentId, seen); 332 for (State state : states) { 333 String id = (String) state.get(KEY_ID); 334 if (transientStates.containsKey(id)) { 335 // found transient, even though we passed an exclusion list for known children 336 // that means that in-memory it's not a child, but in-database it's a child (was moved) 337 // -> ignore the database state 338 continue; 339 } 340 children.add(id); 341 } 342 return new ArrayList<>(children); 343 } 344 345 public boolean hasChildren(String parentId) { 346 // check transient state 347 for (DBSDocumentState docState : transientStates.values()) { 348 if (!parentId.equals(docState.getParentId())) { 349 continue; 350 } 351 return true; 352 } 353 // check repository 354 return repository.queryKeyValuePresence(KEY_PARENT_ID, parentId, Collections.emptySet()); 355 } 356 357 public DBSDocumentState createChild(String id, String parentId, String name, Long pos, String typeName) { 358 // id may be not-null for import 359 if (id == null) { 360 id = repository.generateNewId(); 361 } 362 transientCreated.add(id); 363 DBSDocumentState docState = new DBSDocumentState(); 364 transientStates.put(id, docState); 365 docState.put(KEY_ID, id); 366 docState.put(KEY_PARENT_ID, parentId); 367 docState.put(KEY_ANCESTOR_IDS, getAncestorIds(parentId)); 368 docState.put(KEY_NAME, name); 369 docState.put(KEY_POS, pos); 370 docState.put(KEY_PRIMARY_TYPE, typeName); 371 if (session.changeTokenEnabled) { 372 docState.put(KEY_SYS_CHANGE_TOKEN, INITIAL_SYS_CHANGE_TOKEN); 373 } 374 // update read acls for new doc 375 updateDocumentReadAcls(id); 376 return docState; 377 } 378 379 /** Gets ancestors including id itself. */ 380 protected Object[] getAncestorIds(String id) { 381 if (id == null) { 382 return null; 383 } 384 State state = getStateForRead(id); 385 if (state == null) { 386 throw new RuntimeException("No such id: " + id); 387 } 388 Object[] ancestors = (Object[]) state.get(KEY_ANCESTOR_IDS); 389 if (ancestors == null) { 390 return new Object[] { id }; 391 } else { 392 Object[] newAncestors = new Object[ancestors.length + 1]; 393 System.arraycopy(ancestors, 0, newAncestors, 0, ancestors.length); 394 newAncestors[ancestors.length] = id; 395 return newAncestors; 396 } 397 } 398 399 /** 400 * Copies the document into a newly-created object. 401 * <p> 402 * The copy is automatically saved. 403 */ 404 public DBSDocumentState copy(String id) { 405 DBSDocumentState copyState = new DBSDocumentState(getStateForRead(id)); 406 String copyId = repository.generateNewId(); 407 copyState.put(KEY_ID, copyId); 408 copyState.put(KEY_PROXY_IDS, null); // no proxies to this new doc 409 // other fields updated by the caller 410 transientStates.put(copyId, copyState); 411 transientCreated.add(copyId); 412 return copyState; 413 } 414 415 /** 416 * Updates ancestors recursively after a move. 417 * <p> 418 * Recursing from given doc, replace the first ndel ancestors with those passed. 419 * <p> 420 * Doesn't check transient (assumes save is done). The modifications are automatically saved. 421 */ 422 public void updateAncestors(String id, int ndel, Object[] ancestorIds) { 423 int nadd = ancestorIds.length; 424 Set<String> ids = new HashSet<>(); 425 ids.add(id); 426 try (Stream<State> states = getDescendants(id, Collections.emptySet(), 0)) { 427 states.forEach(state -> ids.add((String) state.get(KEY_ID))); 428 } 429 // we collect all ids first to avoid reentrancy to the repository 430 for (String cid : ids) { 431 // XXX TODO oneShot update, don't pollute transient space 432 DBSDocumentState docState = getStateForUpdate(cid); 433 Object[] ancestors = (Object[]) docState.get(KEY_ANCESTOR_IDS); 434 Object[] newAncestors; 435 if (ancestors == null) { 436 newAncestors = ancestorIds.clone(); 437 } else { 438 newAncestors = new Object[ancestors.length - ndel + nadd]; 439 System.arraycopy(ancestorIds, 0, newAncestors, 0, nadd); 440 System.arraycopy(ancestors, ndel, newAncestors, nadd, ancestors.length - ndel); 441 } 442 docState.put(KEY_ANCESTOR_IDS, newAncestors); 443 } 444 } 445 446 protected int getReadAclsAsyncThreshold() { 447 boolean enabled = Boolean.parseBoolean( 448 Framework.getProperty(READ_ACL_ASYNC_ENABLED_PROPERTY, READ_ACL_ASYNC_ENABLED_DEFAULT)); 449 if (enabled) { 450 return Integer.parseInt( 451 Framework.getProperty(READ_ACL_ASYNC_THRESHOLD_PROPERTY, READ_ACL_ASYNC_THRESHOLD_DEFAULT)); 452 } else { 453 return 0; 454 } 455 } 456 457 /** 458 * Updates the Read ACLs recursively on a document. 459 */ 460 public void updateTreeReadAcls(String id) { 461 // versions too XXX TODO 462 463 save(); // flush everything to the database 464 465 // update the doc itself 466 updateDocumentReadAcls(id); 467 468 // check if we have a small enough number of descendants that we can process them synchronously 469 int limit = getReadAclsAsyncThreshold(); 470 Set<String> ids = new HashSet<>(); 471 try (Stream<State> states = getDescendants(id, Collections.emptySet(), limit)) { 472 states.forEach(state -> ids.add((String) state.get(KEY_ID))); 473 } 474 if (limit == 0 || ids.size() < limit) { 475 // update all descendants synchronously 476 ids.forEach(this::updateDocumentReadAcls); 477 } else { 478 // update the direct children synchronously, the rest asynchronously 479 480 // update the direct children (with a limit in case it's too big) 481 String nxql = String.format("SELECT ecm:uuid FROM Document WHERE ecm:parentId = '%s'", id); 482 Principal principal = new SystemPrincipal(null); 483 QueryFilter queryFilter = new QueryFilter(principal, null, null, null, Collections.emptyList(), limit, 0); 484 PartialList<Map<String, Serializable>> pl = session.queryProjection(nxql, NXQL.NXQL, queryFilter, false, 0, 485 new Object[0]); 486 for (Map<String, Serializable> map : pl) { 487 String childId = (String) map.get(NXQL.ECM_UUID); 488 updateDocumentReadAcls(childId); 489 } 490 491 // asynchronous work to do the whole tree 492 nxql = String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", id); 493 Work work = new FindReadAclsWork(repository.getName(), nxql, null); 494 Framework.getService(WorkManager.class).schedule(work); 495 } 496 } 497 498 /** 499 * Work to find the ids of documents for which Read ACLs must be recomputed, and launch the needed update works. 500 * 501 * @since 9.10 502 */ 503 public static class FindReadAclsWork extends BatchFinderWork { 504 505 private static final long serialVersionUID = 1L; 506 507 public FindReadAclsWork(String repositoryName, String nxql, String originatingUsername) { 508 super(repositoryName, nxql, originatingUsername); 509 } 510 511 @Override 512 public String getTitle() { 513 return "Find descendants for Read ACLs"; 514 } 515 516 @Override 517 public String getCategory() { 518 return "security"; 519 } 520 521 @Override 522 public int getBatchSize() { 523 return 500; 524 } 525 526 @Override 527 public Work getBatchProcessorWork(List<String> docIds) { 528 return new UpdateReadAclsWork(repositoryName, docIds, getOriginatingUsername()); 529 } 530 } 531 532 /** 533 * Work to update the Read ACLs on a list of documents, without recursion. 534 * 535 * @since 9.10 536 */ 537 public static class UpdateReadAclsWork extends BatchProcessorWork { 538 539 private static final long serialVersionUID = 1L; 540 541 public UpdateReadAclsWork(String repositoryName, List<String> docIds, String originatingUsername) { 542 super(repositoryName, docIds, originatingUsername); 543 } 544 545 @Override 546 public String getTitle() { 547 return "Update Read ACLs"; 548 } 549 550 @Override 551 public String getCategory() { 552 return "security"; 553 } 554 555 @Override 556 public int getBatchSize() { 557 return 50; 558 } 559 560 @Override 561 public void processBatch(List<String> docIds) { 562 session.updateReadACLs(docIds); 563 } 564 } 565 566 /** 567 * Updates the Read ACLs on a document (not recursively), bypassing transient space and caches for the document 568 * itself (not the ancestors, needed for ACL inheritance and for which caching is useful). 569 */ 570 public void updateReadACLs(Collection<String> docIds) { 571 docIds.forEach(id -> updateDocumentReadAclsNoCache(id)); 572 } 573 574 /** 575 * Updates the Read ACLs on a document (not recursively) 576 */ 577 protected void updateDocumentReadAcls(String id) { 578 DBSDocumentState docState = getStateForUpdate(id); 579 docState.put(KEY_READ_ACL, getReadACL(docState.getState())); 580 } 581 582 /** 583 * Updates the Read ACLs on a document, without polluting caches. 584 * <p> 585 * When fetching parents recursively to compute inheritance, the regular transient space and repository caching are 586 * used. 587 */ 588 protected void updateDocumentReadAclsNoCache(String id) { 589 // no transient for state read, and we don't want to trash caches 590 // fetch from repository only the properties needed for Read ACL computation and recursion 591 State state = repository.readPartialState(id, READ_ACL_RECURSION_KEYS); 592 State oldState = new State(1); 593 oldState.put(KEY_READ_ACL, state.get(KEY_READ_ACL)); 594 // compute new value 595 State newState = new State(1); 596 newState.put(KEY_READ_ACL, getReadACL(state)); 597 StateDiff diff = StateHelper.diff(oldState, newState); 598 if (!diff.isEmpty()) { 599 // no transient for state write, we write directly and just invalidate caches 600 repository.updateState(id, diff, null); 601 } 602 } 603 604 /** 605 * Gets the Read ACL (flat list of users having browse permission, including inheritance) on a document. 606 */ 607 protected String[] getReadACL(State state) { 608 Set<String> racls = new HashSet<>(); 609 LOOP: do { 610 @SuppressWarnings("unchecked") 611 List<Serializable> aclList = (List<Serializable>) state.get(KEY_ACP); 612 if (aclList != null) { 613 for (Serializable aclSer : aclList) { 614 State aclMap = (State) aclSer; 615 @SuppressWarnings("unchecked") 616 List<Serializable> aceList = (List<Serializable>) aclMap.get(KEY_ACL); 617 for (Serializable aceSer : aceList) { 618 State aceMap = (State) aceSer; 619 String username = (String) aceMap.get(KEY_ACE_USER); 620 String permission = (String) aceMap.get(KEY_ACE_PERMISSION); 621 Boolean granted = (Boolean) aceMap.get(KEY_ACE_GRANT); 622 Long status = (Long) aceMap.get(KEY_ACE_STATUS); 623 if (TRUE.equals(granted) && browsePermissions.contains(permission) 624 && (status == null || status == 1)) { 625 racls.add(username); 626 } 627 if (FALSE.equals(granted)) { 628 if (!EVERYONE.equals(username)) { 629 // TODO log 630 racls.add(UNSUPPORTED_ACL); 631 } 632 break LOOP; 633 } 634 } 635 } 636 } 637 // get the parent; for a version the parent is the live document 638 String parentKey = TRUE.equals(state.get(KEY_IS_VERSION)) ? KEY_VERSION_SERIES_ID : KEY_PARENT_ID; 639 String parentId = (String) state.get(parentKey); 640 state = parentId == null ? null : getStateForRead(parentId); 641 } while (state != null); 642 643 // sort to have canonical order 644 List<String> racl = new ArrayList<>(racls); 645 Collections.sort(racl); 646 return racl.toArray(new String[racl.size()]); 647 } 648 649 protected Stream<State> getDescendants(String id, Set<String> keys, int limit) { 650 return repository.getDescendants(id, keys, limit); 651 } 652 653 public List<DBSDocumentState> getKeyValuedStates(String key, Object value) { 654 List<DBSDocumentState> docStates = new LinkedList<>(); 655 Set<String> seen = new HashSet<>(); 656 // check transient state 657 for (DBSDocumentState docState : transientStates.values()) { 658 if (!value.equals(docState.get(key))) { 659 continue; 660 } 661 docStates.add(docState); 662 seen.add(docState.getId()); 663 } 664 // fetch from repository 665 List<State> states = repository.queryKeyValue(key, value, seen); 666 for (State state : states) { 667 docStates.add(newTransientState(state)); 668 } 669 return docStates; 670 } 671 672 public List<DBSDocumentState> getKeyValuedStates(String key1, Object value1, String key2, Object value2) { 673 List<DBSDocumentState> docStates = new LinkedList<>(); 674 Set<String> seen = new HashSet<>(); 675 // check transient state 676 for (DBSDocumentState docState : transientStates.values()) { 677 seen.add(docState.getId()); 678 if (!(value1.equals(docState.get(key1)) && value2.equals(docState.get(key2)))) { 679 continue; 680 } 681 docStates.add(docState); 682 } 683 // fetch from repository 684 List<State> states = repository.queryKeyValue(key1, value1, key2, value2, seen); 685 for (State state : states) { 686 docStates.add(newTransientState(state)); 687 } 688 return docStates; 689 } 690 691 /** 692 * Removes a list of documents. 693 * <p> 694 * Called after a {@link #save} has been done. 695 */ 696 public void removeStates(Set<String> ids) { 697 if (undoLog != null) { 698 for (String id : ids) { 699 if (undoLog.containsKey(id)) { 700 // there's already a create or an update in the undo log 701 State oldUndo = undoLog.get(id); 702 if (oldUndo == null) { 703 // create + delete -> forget 704 undoLog.remove(id); 705 } else { 706 // update + delete -> original old state to re-create 707 oldUndo.put(KEY_UNDOLOG_CREATE, TRUE); 708 } 709 } else { 710 // just delete -> store old state to re-create 711 State oldState = StateHelper.deepCopy(getStateForRead(id)); 712 oldState.put(KEY_UNDOLOG_CREATE, TRUE); 713 undoLog.put(id, oldState); 714 } 715 } 716 } 717 for (String id : ids) { 718 transientStates.remove(id); 719 } 720 repository.deleteStates(ids); 721 } 722 723 public void markUserChange(String id) { 724 userChangeIds.add(id); 725 } 726 727 /** 728 * Writes transient state to database. 729 * <p> 730 * An undo log is kept in order to rollback the transaction later if needed. 731 */ 732 public void save() { 733 updateProxies(); 734 List<Work> works; 735 if (!repository.isFulltextDisabled()) { 736 // TODO getting fulltext already does a getStateChange 737 works = getFulltextWorks(); 738 } else { 739 works = Collections.emptyList(); 740 } 741 List<State> statesToCreate = new ArrayList<>(); 742 for (String id : transientCreated) { // ordered 743 DBSDocumentState docState = transientStates.get(id); 744 docState.setNotDirty(); 745 if (undoLog != null) { 746 undoLog.put(id, null); // marker to denote create 747 } 748 State state = docState.getState(); 749 state.put(KEY_CHANGE_TOKEN, INITIAL_CHANGE_TOKEN); 750 statesToCreate.add(state); 751 } 752 if (!statesToCreate.isEmpty()) { 753 repository.createStates(statesToCreate); 754 } 755 for (DBSDocumentState docState : transientStates.values()) { 756 String id = docState.getId(); 757 if (transientCreated.contains(id)) { 758 continue; // already done 759 } 760 StateDiff diff = docState.getStateChange(); 761 if (diff != null) { 762 if (undoLog != null) { 763 if (!undoLog.containsKey(id)) { 764 undoLog.put(id, StateHelper.deepCopy(docState.getOriginalState())); 765 } 766 // else there's already a create or an update in the undo log so original info is enough 767 } 768 ChangeTokenUpdater changeTokenUpdater; 769 if (session.changeTokenEnabled) { 770 // increment system change token 771 Long base = (Long) docState.get(KEY_SYS_CHANGE_TOKEN); 772 docState.put(KEY_SYS_CHANGE_TOKEN, DeltaLong.valueOf(base, 1)); 773 diff.put(KEY_SYS_CHANGE_TOKEN, DeltaLong.valueOf(base, 1)); 774 // update change token if applicable (user change) 775 if (userChangeIds.contains(id)) { 776 changeTokenUpdater = new ChangeTokenUpdater(docState); 777 } else { 778 changeTokenUpdater = null; 779 } 780 } else { 781 changeTokenUpdater = null; 782 } 783 repository.updateState(id, diff, changeTokenUpdater); 784 } 785 docState.setNotDirty(); 786 } 787 transientCreated.clear(); 788 userChangeIds.clear(); 789 scheduleWork(works); 790 } 791 792 /** 793 * Logic to get the conditions to use to match and update a change token. 794 * <p> 795 * This may be called several times for a single DBS document update, because the low-level storage may need several 796 * database updates for a single high-level update in some cases. 797 * 798 * @since 9.1 799 */ 800 public static class ChangeTokenUpdater { 801 802 protected final DBSDocumentState docState; 803 804 protected Long oldToken; 805 806 public ChangeTokenUpdater(DBSDocumentState docState) { 807 this.docState = docState; 808 oldToken = (Long) docState.getOriginalState().get(KEY_CHANGE_TOKEN); 809 } 810 811 /** 812 * Gets the conditions to use to match a change token. 813 */ 814 public Map<String, Serializable> getConditions() { 815 return Collections.singletonMap(KEY_CHANGE_TOKEN, oldToken); 816 } 817 818 /** 819 * Gets the updates to make to write the updated change token. 820 */ 821 public Map<String, Serializable> getUpdates() { 822 Long newToken; 823 if (oldToken == null) { 824 // document without change token, just created 825 newToken = INITIAL_CHANGE_TOKEN; 826 } else { 827 newToken = BaseDocument.updateChangeToken(oldToken); 828 } 829 // also store the new token in the state (without marking dirty), for the next update 830 docState.getState().put(KEY_CHANGE_TOKEN, newToken); 831 oldToken = newToken; 832 return Collections.singletonMap(KEY_CHANGE_TOKEN, newToken); 833 } 834 } 835 836 protected void applyUndoLog() { 837 Set<String> deletes = new HashSet<>(); 838 for (Entry<String, State> es : undoLog.entrySet()) { 839 String id = es.getKey(); 840 State state = es.getValue(); 841 if (state == null) { 842 deletes.add(id); 843 } else { 844 boolean recreate = state.remove(KEY_UNDOLOG_CREATE) != null; 845 if (recreate) { 846 repository.createState(state); 847 } else { 848 // undo update 849 State currentState = repository.readState(id); 850 if (currentState != null) { 851 StateDiff diff = StateHelper.diff(currentState, state); 852 if (!diff.isEmpty()) { 853 repository.updateState(id, diff, null); 854 } 855 } 856 // else we expected to read a current state but it was concurrently deleted... 857 // in that case leave it deleted 858 } 859 } 860 } 861 if (!deletes.isEmpty()) { 862 repository.deleteStates(deletes); 863 } 864 } 865 866 /** 867 * Checks if the changed documents are proxy targets, and updates the proxies if that's the case. 868 */ 869 protected void updateProxies() { 870 for (String id : transientCreated) { // ordered 871 DBSDocumentState docState = transientStates.get(id); 872 updateProxies(docState); 873 } 874 // copy as we may modify proxies 875 for (String id : transientStates.keySet().toArray(new String[0])) { 876 DBSDocumentState docState = transientStates.get(id); 877 if (transientCreated.contains(id)) { 878 continue; // already done 879 } 880 if (docState.isDirty()) { 881 updateProxies(docState); 882 } 883 } 884 } 885 886 protected void updateProxies(DBSDocumentState target) { 887 Object[] proxyIds = (Object[]) target.get(KEY_PROXY_IDS); 888 if (proxyIds != null) { 889 for (Object proxyId : proxyIds) { 890 try { 891 updateProxy(target, (String) proxyId); 892 } catch (ConcurrentUpdateException e) { 893 e.addInfo("On doc " + target.getId()); 894 log.error(e, e); 895 // do not throw, this avoids crashing the session 896 } 897 } 898 } 899 } 900 901 /** 902 * Updates the state of a proxy based on its target. 903 */ 904 protected void updateProxy(DBSDocumentState target, String proxyId) { 905 DBSDocumentState proxy = getStateForUpdate(proxyId); 906 if (proxy == null) { 907 throw new ConcurrentUpdateException("Proxy " + proxyId + " concurrently deleted"); 908 } 909 SchemaManager schemaManager = Framework.getService(SchemaManager.class); 910 // clear all proxy data 911 for (String key : proxy.getState().keyArray()) { 912 if (!isProxySpecific(key, schemaManager)) { 913 proxy.put(key, null); 914 } 915 } 916 // copy from target 917 for (Entry<String, Serializable> en : target.getState().entrySet()) { 918 String key = en.getKey(); 919 if (!isProxySpecific(key, schemaManager)) { 920 proxy.put(key, StateHelper.deepCopy(en.getValue())); 921 } 922 } 923 } 924 925 /** 926 * Things that we don't touch on a proxy when updating it. 927 */ 928 protected boolean isProxySpecific(String key, SchemaManager schemaManager) { 929 switch (key) { 930 // these are placeful stuff 931 case KEY_ID: 932 case KEY_PARENT_ID: 933 case KEY_ANCESTOR_IDS: 934 case KEY_NAME: 935 case KEY_POS: 936 case KEY_ACP: 937 case KEY_READ_ACL: 938 // these are proxy-specific 939 case KEY_IS_PROXY: 940 case KEY_PROXY_TARGET_ID: 941 case KEY_PROXY_VERSION_SERIES_ID: 942 case KEY_IS_VERSION: 943 case KEY_PROXY_IDS: 944 return true; 945 } 946 int p = key.indexOf(':'); 947 if (p == -1) { 948 // no prefix, assume not proxy-specific 949 return false; 950 } 951 String prefix = key.substring(0, p); 952 Schema schema = schemaManager.getSchemaFromPrefix(prefix); 953 if (schema == null) { 954 schema = schemaManager.getSchema(prefix); 955 if (schema == null) { 956 // unknown prefix, assume not proxy-specific 957 return false; 958 } 959 } 960 return schemaManager.isProxySchema(schema.getName(), null); // type unused 961 } 962 963 /** 964 * Called when created in a transaction. 965 * 966 * @since 7.4 967 */ 968 public void begin() { 969 undoLog = new HashMap<>(); 970 repository.begin(); 971 } 972 973 /** 974 * Saves and flushes to database. 975 */ 976 public void commit() { 977 save(); 978 commitSave(); 979 repository.commit(); 980 } 981 982 /** 983 * Commits the saved state to the database. 984 */ 985 protected void commitSave() { 986 // clear transient, this means that after this references to states will be stale 987 // TODO mark states as invalid 988 clearTransient(); 989 // the transaction ended, the proxied DBSSession will disappear and cannot be reused anyway 990 undoLog = null; 991 } 992 993 /** 994 * Rolls back the save state by applying the undo log. 995 */ 996 public void rollback() { 997 clearTransient(); 998 applyUndoLog(); 999 // the transaction ended, the proxied DBSSession will disappear and cannot be reused anyway 1000 undoLog = null; 1001 repository.rollback(); 1002 } 1003 1004 protected void clearTransient() { 1005 transientStates.clear(); 1006 transientCreated.clear(); 1007 } 1008 1009 /** 1010 * Gets the fulltext updates to do. Called at save() time. 1011 * 1012 * @return a list of {@link Work} instances to schedule post-commit. 1013 */ 1014 protected List<Work> getFulltextWorks() { 1015 Set<String> docsWithDirtyStrings = new HashSet<>(); 1016 Set<String> docsWithDirtyBinaries = new HashSet<>(); 1017 findDirtyDocuments(docsWithDirtyStrings, docsWithDirtyBinaries); 1018 if (docsWithDirtyStrings.isEmpty() && docsWithDirtyBinaries.isEmpty()) { 1019 return Collections.emptyList(); 1020 } 1021 List<Work> works = new LinkedList<>(); 1022 getFulltextSimpleWorks(works, docsWithDirtyStrings); 1023 getFulltextBinariesWorks(works, docsWithDirtyBinaries); 1024 return works; 1025 } 1026 1027 /** 1028 * Finds the documents having dirty text or dirty binaries that have to be reindexed as fulltext. 1029 * 1030 * @param docsWithDirtyStrings set of ids, updated by this method 1031 * @param docWithDirtyBinaries set of ids, updated by this method 1032 */ 1033 protected void findDirtyDocuments(Set<String> docsWithDirtyStrings, Set<String> docWithDirtyBinaries) { 1034 for (DBSDocumentState docState : transientStates.values()) { 1035 State originalState = docState.getOriginalState(); 1036 State state = docState.getState(); 1037 if (originalState == state) { 1038 continue; 1039 } 1040 StateDiff diff = StateHelper.diff(originalState, state); 1041 if (diff.isEmpty()) { 1042 continue; 1043 } 1044 StateDiff rdiff = StateHelper.diff(state, originalState); 1045 // we do diffs in both directions to capture removal of complex list elements, 1046 // for instance for {foo: [{bar: baz}] -> {foo: []} 1047 // diff paths = foo and rdiff paths = foo/*/bar 1048 Set<String> paths = new HashSet<>(); 1049 DirtyPathsFinder dirtyPathsFinder = new DirtyPathsFinder(paths); 1050 dirtyPathsFinder.findDirtyPaths(diff); 1051 dirtyPathsFinder.findDirtyPaths(rdiff); 1052 FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration(); 1053 boolean dirtyStrings = false; 1054 boolean dirtyBinaries = false; 1055 for (String path : paths) { 1056 Set<String> indexesSimple = fulltextConfiguration.indexesByPropPathSimple.get(path); 1057 if (indexesSimple != null && !indexesSimple.isEmpty()) { 1058 dirtyStrings = true; 1059 if (dirtyBinaries) { 1060 break; 1061 } 1062 } 1063 Set<String> indexesBinary = fulltextConfiguration.indexesByPropPathBinary.get(path); 1064 if (indexesBinary != null && !indexesBinary.isEmpty()) { 1065 dirtyBinaries = true; 1066 if (dirtyStrings) { 1067 break; 1068 } 1069 } 1070 } 1071 if (dirtyStrings) { 1072 docsWithDirtyStrings.add(docState.getId()); 1073 } 1074 if (dirtyBinaries) { 1075 docWithDirtyBinaries.add(docState.getId()); 1076 } 1077 } 1078 } 1079 1080 /** 1081 * Iterates on a state diff to find the paths corresponding to dirty values. 1082 * 1083 * @since 7.10-HF04, 8.1 1084 */ 1085 protected static class DirtyPathsFinder { 1086 1087 protected Set<String> paths; 1088 1089 public DirtyPathsFinder(Set<String> paths) { 1090 this.paths = paths; 1091 } 1092 1093 public void findDirtyPaths(StateDiff value) { 1094 findDirtyPaths(value, null); 1095 } 1096 1097 protected void findDirtyPaths(Object value, String path) { 1098 if (value instanceof Object[]) { 1099 findDirtyPaths((Object[]) value, path); 1100 } else if (value instanceof List) { 1101 findDirtyPaths((List<?>) value, path); 1102 } else if (value instanceof ListDiff) { 1103 findDirtyPaths((ListDiff) value, path); 1104 } else if (value instanceof State) { 1105 findDirtyPaths((State) value, path); 1106 } else { 1107 paths.add(path); 1108 } 1109 } 1110 1111 protected void findDirtyPaths(Object[] value, String path) { 1112 String newPath = path + "/*"; 1113 for (Object v : value) { 1114 findDirtyPaths(v, newPath); 1115 } 1116 } 1117 1118 protected void findDirtyPaths(List<?> value, String path) { 1119 String newPath = path + "/*"; 1120 for (Object v : value) { 1121 findDirtyPaths(v, newPath); 1122 } 1123 } 1124 1125 protected void findDirtyPaths(ListDiff value, String path) { 1126 String newPath = path + "/*"; 1127 if (value.diff != null) { 1128 findDirtyPaths(value.diff, newPath); 1129 } 1130 if (value.rpush != null) { 1131 findDirtyPaths(value.rpush, newPath); 1132 } 1133 } 1134 1135 protected void findDirtyPaths(State value, String path) { 1136 for (Entry<String, Serializable> es : value.entrySet()) { 1137 String key = es.getKey(); 1138 Serializable v = es.getValue(); 1139 String newPath = path == null ? key : path + "/" + key; 1140 findDirtyPaths(v, newPath); 1141 } 1142 } 1143 } 1144 1145 protected void getFulltextSimpleWorks(List<Work> works, Set<String> docsWithDirtyStrings) { 1146 // TODO XXX make configurable, see also FulltextExtractorWork 1147 FulltextParser fulltextParser = new DefaultFulltextParser(); 1148 FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration(); 1149 if (fulltextConfiguration.fulltextSearchDisabled) { 1150 return; 1151 } 1152 // update simpletext on documents with dirty strings 1153 for (String id : docsWithDirtyStrings) { 1154 if (id == null) { 1155 // cannot happen, but has been observed :( 1156 log.error("Got null doc id in fulltext update, cannot happen"); 1157 continue; 1158 } 1159 DBSDocumentState docState = getStateForUpdate(id); 1160 if (docState == null) { 1161 // cannot happen 1162 continue; 1163 } 1164 String documentType = docState.getPrimaryType(); 1165 // Object[] mixinTypes = (Object[]) docState.get(KEY_MIXIN_TYPES); 1166 1167 if (!fulltextConfiguration.isFulltextIndexable(documentType)) { 1168 continue; 1169 } 1170 docState.put(KEY_FULLTEXT_JOBID, docState.getId()); 1171 FulltextFinder fulltextFinder = new FulltextFinder(fulltextParser, docState, session); 1172 List<IndexAndText> indexesAndText = new LinkedList<>(); 1173 for (String indexName : fulltextConfiguration.indexNames) { 1174 // TODO paths from config 1175 String text = fulltextFinder.findFulltext(indexName); 1176 indexesAndText.add(new IndexAndText(indexName, text)); 1177 } 1178 if (!indexesAndText.isEmpty()) { 1179 Work work = new FulltextUpdaterWork(repository.getName(), id, true, false, indexesAndText); 1180 works.add(work); 1181 } 1182 } 1183 } 1184 1185 protected void getFulltextBinariesWorks(List<Work> works, Set<String> docWithDirtyBinaries) { 1186 if (docWithDirtyBinaries.isEmpty()) { 1187 return; 1188 } 1189 1190 FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration(); 1191 1192 // mark indexing in progress, so that future copies (including versions) 1193 // will be indexed as well 1194 for (String id : docWithDirtyBinaries) { 1195 DBSDocumentState docState = getStateForUpdate(id); 1196 if (docState == null) { 1197 // cannot happen 1198 continue; 1199 } 1200 if (!fulltextConfiguration.isFulltextIndexable(docState.getPrimaryType())) { 1201 continue; 1202 } 1203 docState.put(KEY_FULLTEXT_JOBID, docState.getId()); 1204 } 1205 1206 // FulltextExtractorWork does fulltext extraction using converters 1207 // and then schedules a FulltextUpdaterWork to write the results 1208 // single-threaded 1209 for (String id : docWithDirtyBinaries) { 1210 // don't exclude proxies 1211 Work work = new DBSFulltextExtractorWork(repository.getName(), id); 1212 works.add(work); 1213 } 1214 } 1215 1216 protected static class FulltextFinder { 1217 1218 protected final FulltextParser fulltextParser; 1219 1220 protected final DBSDocumentState document; 1221 1222 protected final DBSSession session; 1223 1224 protected final String documentType; 1225 1226 protected final Object[] mixinTypes; 1227 1228 /** 1229 * Prepares parsing for one document. 1230 */ 1231 public FulltextFinder(FulltextParser fulltextParser, DBSDocumentState document, DBSSession session) { 1232 this.fulltextParser = fulltextParser; 1233 this.document = document; 1234 this.session = session; 1235 if (document == null) { 1236 documentType = null; 1237 mixinTypes = null; 1238 } else { // null in tests 1239 documentType = document.getPrimaryType(); 1240 mixinTypes = (Object[]) document.get(KEY_MIXIN_TYPES); 1241 } 1242 } 1243 1244 /** 1245 * Parses the document for one index. 1246 */ 1247 public String findFulltext(String indexName) { 1248 // TODO indexName 1249 // TODO paths 1250 List<String> strings = new ArrayList<>(); 1251 findFulltext(indexName, document.getState(), strings); 1252 return StringUtils.join(strings, ' '); 1253 } 1254 1255 protected void findFulltext(String indexName, State state, List<String> strings) { 1256 for (Entry<String, Serializable> en : state.entrySet()) { 1257 String key = en.getKey(); 1258 if (key.startsWith(KEY_PREFIX)) { 1259 switch (key) { 1260 // allow indexing of this: 1261 case DBSDocument.KEY_NAME: 1262 break; 1263 default: 1264 continue; 1265 } 1266 } 1267 Serializable value = en.getValue(); 1268 if (value instanceof State) { 1269 State s = (State) value; 1270 findFulltext(indexName, s, strings); 1271 } else if (value instanceof List) { 1272 @SuppressWarnings("unchecked") 1273 List<State> v = (List<State>) value; 1274 for (State s : v) { 1275 findFulltext(indexName, s, strings); 1276 } 1277 } else if (value instanceof Object[]) { 1278 Object[] ar = (Object[]) value; 1279 for (Object v : ar) { 1280 if (v instanceof String) { 1281 fulltextParser.parse((String) v, null, strings); 1282 } else { 1283 // arrays are homogeneous, no need to continue 1284 break; 1285 } 1286 } 1287 } else { 1288 if (value instanceof String) { 1289 fulltextParser.parse((String) value, null, strings); 1290 } 1291 } 1292 } 1293 } 1294 } 1295 1296 protected void scheduleWork(List<Work> works) { 1297 // do async fulltext indexing only if high-level sessions are available 1298 RepositoryManager repositoryManager = Framework.getService(RepositoryManager.class); 1299 if (repositoryManager != null && !works.isEmpty()) { 1300 WorkManager workManager = Framework.getService(WorkManager.class); 1301 for (Work work : works) { 1302 // schedule work post-commit 1303 // in non-tx mode, this may execute it nearly immediately 1304 workManager.schedule(work, Scheduling.IF_NOT_SCHEDULED, true); 1305 } 1306 } 1307 } 1308 1309}