001/*
002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.dbs;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023import static org.nuxeo.ecm.core.api.security.SecurityConstants.BROWSE;
024import static org.nuxeo.ecm.core.api.security.SecurityConstants.EVERYONE;
025import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_CHANGE_TOKEN;
027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_SYS_CHANGE_TOKEN;
028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_GRANT;
029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_PERMISSION;
030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_STATUS;
031import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_USER;
032import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
033import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
034import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ANCESTOR_IDS;
035import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_CHANGE_TOKEN;
036import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_JOBID;
037import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ID;
038import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_PROXY;
039import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_VERSION;
040import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_NAME;
041import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PARENT_ID;
042import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_POS;
043import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PRIMARY_TYPE;
044import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_IDS;
045import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_TARGET_ID;
046import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_VERSION_SERIES_ID;
047import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_READ_ACL;
048import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_SYS_CHANGE_TOKEN;
049import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_SERIES_ID;
050
051import java.io.Serializable;
052import java.util.ArrayList;
053import java.util.Arrays;
054import java.util.Collection;
055import java.util.Collections;
056import java.util.HashMap;
057import java.util.HashSet;
058import java.util.LinkedHashSet;
059import java.util.LinkedList;
060import java.util.List;
061import java.util.Map;
062import java.util.Map.Entry;
063import java.util.Set;
064import java.util.stream.Stream;
065
066import org.apache.commons.logging.Log;
067import org.apache.commons.logging.LogFactory;
068import org.nuxeo.ecm.core.BatchFinderWork;
069import org.nuxeo.ecm.core.BatchProcessorWork;
070import org.nuxeo.ecm.core.api.ConcurrentUpdateException;
071import org.nuxeo.ecm.core.api.NuxeoPrincipal;
072import org.nuxeo.ecm.core.api.PartialList;
073import org.nuxeo.ecm.core.api.SystemPrincipal;
074import org.nuxeo.ecm.core.api.model.DeltaLong;
075import org.nuxeo.ecm.core.api.repository.FulltextConfiguration;
076import org.nuxeo.ecm.core.api.repository.RepositoryManager;
077import org.nuxeo.ecm.core.query.QueryFilter;
078import org.nuxeo.ecm.core.query.sql.NXQL;
079import org.nuxeo.ecm.core.schema.SchemaManager;
080import org.nuxeo.ecm.core.schema.types.Schema;
081import org.nuxeo.ecm.core.security.SecurityService;
082import org.nuxeo.ecm.core.storage.BaseDocument;
083import org.nuxeo.ecm.core.storage.FulltextExtractorWork;
084import org.nuxeo.ecm.core.storage.State;
085import org.nuxeo.ecm.core.storage.State.ListDiff;
086import org.nuxeo.ecm.core.storage.State.StateDiff;
087import org.nuxeo.ecm.core.storage.StateHelper;
088import org.nuxeo.ecm.core.work.api.Work;
089import org.nuxeo.ecm.core.work.api.WorkManager;
090import org.nuxeo.ecm.core.work.api.WorkManager.Scheduling;
091import org.nuxeo.runtime.api.Framework;
092
093/**
094 * Transactional state for a session.
095 * <p>
096 * Until {@code save()} is called, data lives in the transient map.
097 * <p>
098 * Upon save, data is written to the repository, even though it has not yet been committed (this means that other
099 * sessions can read uncommitted data). It's also kept in an undo log in order for rollback to be possible.
100 * <p>
101 * On commit, the undo log is forgotten. On rollback, the undo log is replayed.
102 *
103 * @since 5.9.4
104 */
105public class DBSTransactionState {
106
107    private static final Log log = LogFactory.getLog(DBSTransactionState.class);
108
109    private static final String KEY_UNDOLOG_CREATE = "__UNDOLOG_CREATE__\0\0";
110
111    /** Keys used when computing Read ACLs. */
112    protected static final Set<String> READ_ACL_RECURSION_KEYS = new HashSet<>(
113            Arrays.asList(KEY_READ_ACL, KEY_ACP, KEY_IS_VERSION, KEY_VERSION_SERIES_ID, KEY_PARENT_ID));
114
115    public static final String READ_ACL_ASYNC_ENABLED_PROPERTY = "nuxeo.core.readacl.async.enabled";
116
117    public static final String READ_ACL_ASYNC_ENABLED_DEFAULT = "true";
118
119    public static final String READ_ACL_ASYNC_THRESHOLD_PROPERTY = "nuxeo.core.readacl.async.threshold";
120
121    public static final String READ_ACL_ASYNC_THRESHOLD_DEFAULT = "500";
122
123    protected final DBSRepository repository;
124
125    protected final DBSSession session;
126
127    /** Retrieved and created document state. */
128    protected Map<String, DBSDocumentState> transientStates = new HashMap<>();
129
130    /** Ids of documents created but not yet saved. */
131    protected Set<String> transientCreated = new LinkedHashSet<>();
132
133    /**
134     * Document ids modified as "user changes", which means that a change token should be checked.
135     *
136     * @since 9.2
137     */
138    protected final Set<Serializable> userChangeIds = new HashSet<>();
139
140    /**
141     * Undo log.
142     * <p>
143     * A map of document ids to null or State. The value is null when the document has to be deleted when applying the
144     * undo log. Otherwise the value is a State. If the State contains the key {@link #KEY_UNDOLOG_CREATE} then the
145     * state must be re-created completely when applying the undo log, otherwise just applied as an update.
146     * <p>
147     * Null when there is no active transaction.
148     */
149    protected Map<String, State> undoLog;
150
151    protected final Set<String> browsePermissions;
152
153    public DBSTransactionState(DBSRepository repository, DBSSession session) {
154        this.repository = repository;
155        this.session = session;
156        SecurityService securityService = Framework.getService(SecurityService.class);
157        browsePermissions = new HashSet<>(Arrays.asList(securityService.getPermissionsToCheck(BROWSE)));
158    }
159
160    /**
161     * New transient state for something just read from the repository.
162     */
163    protected DBSDocumentState newTransientState(State state) {
164        if (state == null) {
165            return null;
166        }
167        String id = (String) state.get(KEY_ID);
168        if (transientStates.containsKey(id)) {
169            throw new IllegalStateException("Already transient: " + id);
170        }
171        DBSDocumentState docState = new DBSDocumentState(state); // copy
172        transientStates.put(id, docState);
173        return docState;
174    }
175
176    /**
177     * Returns a state and marks it as transient, because it's about to be modified or returned to user code (where it
178     * may be modified).
179     */
180    public DBSDocumentState getStateForUpdate(String id) {
181        // check transient state
182        DBSDocumentState docState = transientStates.get(id);
183        if (docState != null) {
184            return docState;
185        }
186        // fetch from repository
187        State state = repository.readState(id);
188        return newTransientState(state);
189    }
190
191    /**
192     * Returns a state which won't be modified.
193     */
194    // TODO in some cases it's good to have this kept in memory instead of
195    // rereading from database every time
196    // XXX getStateForReadOneShot
197    public State getStateForRead(String id) {
198        // check transient state
199        DBSDocumentState docState = transientStates.get(id);
200        if (docState != null) {
201            return docState.getState();
202        }
203        // fetch from repository
204        return repository.readState(id);
205    }
206
207    /**
208     * Returns states and marks them transient, because they're about to be returned to user code (where they may be
209     * modified).
210     */
211    public List<DBSDocumentState> getStatesForUpdate(Collection<String> ids) {
212        // check which ones we have to fetch from repository
213        List<String> idsToFetch = new LinkedList<>();
214        for (String id : ids) {
215            // check transient state
216            DBSDocumentState docState = transientStates.get(id);
217            if (docState != null) {
218                continue;
219            }
220            // will have to fetch it
221            idsToFetch.add(id);
222        }
223        if (!idsToFetch.isEmpty()) {
224            List<State> states = repository.readStates(idsToFetch);
225            for (State state : states) {
226                newTransientState(state);
227            }
228        }
229        // everything now fetched in transient
230        List<DBSDocumentState> docStates = new ArrayList<>(ids.size());
231        for (String id : ids) {
232            DBSDocumentState docState = transientStates.get(id);
233            if (docState == null) {
234                if (log.isTraceEnabled()) {
235                    log.trace("Cannot fetch document with id: " + id, new Throwable("debug stack trace"));
236                }
237                continue;
238            }
239            docStates.add(docState);
240        }
241        return docStates;
242    }
243
244    // XXX TODO for update or for read?
245    public DBSDocumentState getChildState(String parentId, String name) {
246        // check transient state
247        for (DBSDocumentState docState : transientStates.values()) {
248            if (!parentId.equals(docState.getParentId())) {
249                continue;
250            }
251            if (!name.equals(docState.getName())) {
252                continue;
253            }
254            return docState;
255        }
256        // fetch from repository
257        State state = repository.readChildState(parentId, name, Collections.emptySet());
258        if (state == null) {
259            return null;
260        }
261        String id = (String) state.get(KEY_ID);
262        if (transientStates.containsKey(id)) {
263            // found transient, even though we already checked
264            // that means that in-memory it's not a child, but in-database it's a child (was moved)
265            // -> ignore the database state
266            return null;
267        }
268        return newTransientState(state);
269    }
270
271    public boolean hasChild(String parentId, String name) {
272        // check transient state
273        for (DBSDocumentState docState : transientStates.values()) {
274            if (!parentId.equals(docState.getParentId())) {
275                continue;
276            }
277            if (!name.equals(docState.getName())) {
278                continue;
279            }
280            return true;
281        }
282        // check repository
283        return repository.hasChild(parentId, name, Collections.emptySet());
284    }
285
286    public List<DBSDocumentState> getChildrenStates(String parentId) {
287        List<DBSDocumentState> docStates = new LinkedList<>();
288        Set<String> seen = new HashSet<>();
289        // check transient state
290        for (DBSDocumentState docState : transientStates.values()) {
291            if (!parentId.equals(docState.getParentId())) {
292                continue;
293            }
294            docStates.add(docState);
295            seen.add(docState.getId());
296        }
297        // fetch from repository
298        List<State> states = repository.queryKeyValue(KEY_PARENT_ID, parentId, seen);
299        for (State state : states) {
300            String id = (String) state.get(KEY_ID);
301            if (transientStates.containsKey(id)) {
302                // found transient, even though we passed an exclusion list for known children
303                // that means that in-memory it's not a child, but in-database it's a child (was moved)
304                // -> ignore the database state
305                continue;
306            }
307            docStates.add(newTransientState(state));
308        }
309        return docStates;
310    }
311
312    public List<String> getChildrenIds(String parentId) {
313        List<String> children = new ArrayList<>();
314        Set<String> seen = new HashSet<>();
315        // check transient state
316        for (DBSDocumentState docState : transientStates.values()) {
317            String id = docState.getId();
318            if (!parentId.equals(docState.getParentId())) {
319                continue;
320            }
321            seen.add(id);
322            children.add(id);
323        }
324        // fetch from repository
325        List<State> states = repository.queryKeyValue(KEY_PARENT_ID, parentId, seen);
326        for (State state : states) {
327            String id = (String) state.get(KEY_ID);
328            if (transientStates.containsKey(id)) {
329                // found transient, even though we passed an exclusion list for known children
330                // that means that in-memory it's not a child, but in-database it's a child (was moved)
331                // -> ignore the database state
332                continue;
333            }
334            children.add(id);
335        }
336        return new ArrayList<>(children);
337    }
338
339    public boolean hasChildren(String parentId) {
340        // check transient state
341        for (DBSDocumentState docState : transientStates.values()) {
342            if (!parentId.equals(docState.getParentId())) {
343                continue;
344            }
345            return true;
346        }
347        // check repository
348        return repository.queryKeyValuePresence(KEY_PARENT_ID, parentId, Collections.emptySet());
349    }
350
351    public DBSDocumentState createChild(String id, String parentId, String name, Long pos, String typeName) {
352        // id may be not-null for import
353        if (id == null) {
354            id = repository.generateNewId();
355        }
356        transientCreated.add(id);
357        DBSDocumentState docState = new DBSDocumentState();
358        transientStates.put(id, docState);
359        docState.put(KEY_ID, id);
360        docState.put(KEY_PARENT_ID, parentId);
361        docState.put(KEY_ANCESTOR_IDS, getAncestorIds(parentId));
362        docState.put(KEY_NAME, name);
363        docState.put(KEY_POS, pos);
364        docState.put(KEY_PRIMARY_TYPE, typeName);
365        if (session.changeTokenEnabled) {
366            docState.put(KEY_SYS_CHANGE_TOKEN, INITIAL_SYS_CHANGE_TOKEN);
367        }
368        // update read acls for new doc
369        updateDocumentReadAcls(id);
370        return docState;
371    }
372
373    /** Gets ancestors including id itself. */
374    protected Object[] getAncestorIds(String id) {
375        if (id == null) {
376            return null;
377        }
378        State state = getStateForRead(id);
379        if (state == null) {
380            throw new RuntimeException("No such id: " + id);
381        }
382        Object[] ancestors = (Object[]) state.get(KEY_ANCESTOR_IDS);
383        if (ancestors == null) {
384            return new Object[] { id };
385        } else {
386            Object[] newAncestors = new Object[ancestors.length + 1];
387            System.arraycopy(ancestors, 0, newAncestors, 0, ancestors.length);
388            newAncestors[ancestors.length] = id;
389            return newAncestors;
390        }
391    }
392
393    /**
394     * Copies the document into a newly-created object.
395     * <p>
396     * The copy is automatically saved.
397     */
398    public DBSDocumentState copy(String id) {
399        DBSDocumentState copyState = new DBSDocumentState(getStateForRead(id));
400        String copyId = repository.generateNewId();
401        copyState.put(KEY_ID, copyId);
402        copyState.put(KEY_PROXY_IDS, null); // no proxies to this new doc
403        // other fields updated by the caller
404        transientStates.put(copyId, copyState);
405        transientCreated.add(copyId);
406        return copyState;
407    }
408
409    /**
410     * Updates ancestors recursively after a move.
411     * <p>
412     * Recursing from given doc, replace the first ndel ancestors with those passed.
413     * <p>
414     * Doesn't check transient (assumes save is done). The modifications are automatically saved.
415     */
416    public void updateAncestors(String id, int ndel, Object[] ancestorIds) {
417        int nadd = ancestorIds.length;
418        Set<String> ids = new HashSet<>();
419        ids.add(id);
420        try (Stream<State> states = getDescendants(id, Collections.emptySet(), 0)) {
421            states.forEach(state -> ids.add((String) state.get(KEY_ID)));
422        }
423        // we collect all ids first to avoid reentrancy to the repository
424        for (String cid : ids) {
425            // XXX TODO oneShot update, don't pollute transient space
426            DBSDocumentState docState = getStateForUpdate(cid);
427            Object[] ancestors = (Object[]) docState.get(KEY_ANCESTOR_IDS);
428            Object[] newAncestors;
429            if (ancestors == null) {
430                newAncestors = ancestorIds.clone();
431            } else {
432                newAncestors = new Object[ancestors.length - ndel + nadd];
433                System.arraycopy(ancestorIds, 0, newAncestors, 0, nadd);
434                System.arraycopy(ancestors, ndel, newAncestors, nadd, ancestors.length - ndel);
435            }
436            docState.put(KEY_ANCESTOR_IDS, newAncestors);
437        }
438    }
439
440    protected int getReadAclsAsyncThreshold() {
441        boolean enabled = Boolean.parseBoolean(
442                Framework.getProperty(READ_ACL_ASYNC_ENABLED_PROPERTY, READ_ACL_ASYNC_ENABLED_DEFAULT));
443        if (enabled) {
444            return Integer.parseInt(
445                    Framework.getProperty(READ_ACL_ASYNC_THRESHOLD_PROPERTY, READ_ACL_ASYNC_THRESHOLD_DEFAULT));
446        } else {
447            return 0;
448        }
449    }
450
451    /**
452     * Updates the Read ACLs recursively on a document.
453     */
454    public void updateTreeReadAcls(String id) {
455        // versions too XXX TODO
456
457        save(); // flush everything to the database
458
459        // update the doc itself
460        updateDocumentReadAcls(id);
461
462        // check if we have a small enough number of descendants that we can process them synchronously
463        int limit = getReadAclsAsyncThreshold();
464        Set<String> ids = new HashSet<>();
465        try (Stream<State> states = getDescendants(id, Collections.emptySet(), limit)) {
466            states.forEach(state -> ids.add((String) state.get(KEY_ID)));
467        }
468        if (limit == 0 || ids.size() < limit) {
469            // update all descendants synchronously
470            ids.forEach(this::updateDocumentReadAcls);
471        } else {
472            // update the direct children synchronously, the rest asynchronously
473
474            // update the direct children (with a limit in case it's too big)
475            String nxql = String.format("SELECT ecm:uuid FROM Document WHERE ecm:parentId = '%s'", id);
476            NuxeoPrincipal principal = new SystemPrincipal(null);
477            QueryFilter queryFilter = new QueryFilter(principal, null, null, null, Collections.emptyList(), limit, 0);
478            PartialList<Map<String, Serializable>> pl = session.queryProjection(nxql, NXQL.NXQL, queryFilter, false, 0,
479                    new Object[0]);
480            for (Map<String, Serializable> map : pl) {
481                String childId = (String) map.get(NXQL.ECM_UUID);
482                updateDocumentReadAcls(childId);
483            }
484
485            // asynchronous work to do the whole tree
486            nxql = String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", id);
487            Work work = new FindReadAclsWork(repository.getName(), nxql, null);
488            Framework.getService(WorkManager.class).schedule(work);
489        }
490    }
491
492    /**
493     * Work to find the ids of documents for which Read ACLs must be recomputed, and launch the needed update works.
494     *
495     * @since 9.10
496     */
497    public static class FindReadAclsWork extends BatchFinderWork {
498
499        private static final long serialVersionUID = 1L;
500
501        public FindReadAclsWork(String repositoryName, String nxql, String originatingUsername) {
502            super(repositoryName, nxql, originatingUsername);
503        }
504
505        @Override
506        public String getTitle() {
507            return "Find descendants for Read ACLs";
508        }
509
510        @Override
511        public String getCategory() {
512            return "security";
513        }
514
515        @Override
516        public int getBatchSize() {
517            return 500;
518        }
519
520        @Override
521        public Work getBatchProcessorWork(List<String> docIds) {
522            return new UpdateReadAclsWork(repositoryName, docIds, getOriginatingUsername());
523        }
524    }
525
526    /**
527     * Work to update the Read ACLs on a list of documents, without recursion.
528     *
529     * @since 9.10
530     */
531    public static class UpdateReadAclsWork extends BatchProcessorWork {
532
533        private static final long serialVersionUID = 1L;
534
535        public UpdateReadAclsWork(String repositoryName, List<String> docIds, String originatingUsername) {
536            super(repositoryName, docIds, originatingUsername);
537        }
538
539        @Override
540        public String getTitle() {
541            return "Update Read ACLs";
542        }
543
544        @Override
545        public String getCategory() {
546            return "security";
547        }
548
549        @Override
550        public int getBatchSize() {
551            return 50;
552        }
553
554        @Override
555        public void processBatch(List<String> docIds) {
556            session.updateReadACLs(docIds);
557        }
558    }
559
560    /**
561     * Updates the Read ACLs on a document (not recursively), bypassing transient space and caches for the document
562     * itself (not the ancestors, needed for ACL inheritance and for which caching is useful).
563     */
564    public void updateReadACLs(Collection<String> docIds) {
565        docIds.forEach(id -> updateDocumentReadAclsNoCache(id));
566    }
567
568    /**
569     * Updates the Read ACLs on a document (not recursively)
570     */
571    protected void updateDocumentReadAcls(String id) {
572        DBSDocumentState docState = getStateForUpdate(id);
573        docState.put(KEY_READ_ACL, getReadACL(docState.getState()));
574    }
575
576    /**
577     * Updates the Read ACLs on a document, without polluting caches.
578     * <p>
579     * When fetching parents recursively to compute inheritance, the regular transient space and repository caching are
580     * used.
581     */
582    protected void updateDocumentReadAclsNoCache(String id) {
583        // no transient for state read, and we don't want to trash caches
584        // fetch from repository only the properties needed for Read ACL computation and recursion
585        State state = repository.readPartialState(id, READ_ACL_RECURSION_KEYS);
586        State oldState = new State(1);
587        oldState.put(KEY_READ_ACL, state.get(KEY_READ_ACL));
588        // compute new value
589        State newState = new State(1);
590        newState.put(KEY_READ_ACL, getReadACL(state));
591        StateDiff diff = StateHelper.diff(oldState, newState);
592        if (!diff.isEmpty()) {
593            // no transient for state write, we write directly and just invalidate caches
594            repository.updateState(id, diff, null);
595        }
596    }
597
598    /**
599     * Gets the Read ACL (flat list of users having browse permission, including inheritance) on a document.
600     */
601    protected String[] getReadACL(State state) {
602        Set<String> racls = new HashSet<>();
603        LOOP: do {
604            @SuppressWarnings("unchecked")
605            List<Serializable> aclList = (List<Serializable>) state.get(KEY_ACP);
606            if (aclList != null) {
607                for (Serializable aclSer : aclList) {
608                    State aclMap = (State) aclSer;
609                    @SuppressWarnings("unchecked")
610                    List<Serializable> aceList = (List<Serializable>) aclMap.get(KEY_ACL);
611                    for (Serializable aceSer : aceList) {
612                        State aceMap = (State) aceSer;
613                        String username = (String) aceMap.get(KEY_ACE_USER);
614                        String permission = (String) aceMap.get(KEY_ACE_PERMISSION);
615                        Boolean granted = (Boolean) aceMap.get(KEY_ACE_GRANT);
616                        Long status = (Long) aceMap.get(KEY_ACE_STATUS);
617                        if (TRUE.equals(granted) && browsePermissions.contains(permission)
618                                && (status == null || status == 1)) {
619                            racls.add(username);
620                        }
621                        if (FALSE.equals(granted)) {
622                            if (!EVERYONE.equals(username)) {
623                                // TODO log
624                                racls.add(UNSUPPORTED_ACL);
625                            }
626                            break LOOP;
627                        }
628                    }
629                }
630            }
631            // get the parent; for a version the parent is the live document
632            String parentKey = TRUE.equals(state.get(KEY_IS_VERSION)) ? KEY_VERSION_SERIES_ID : KEY_PARENT_ID;
633            String parentId = (String) state.get(parentKey);
634            state = parentId == null ? null : getStateForRead(parentId);
635        } while (state != null);
636
637        // sort to have canonical order
638        List<String> racl = new ArrayList<>(racls);
639        Collections.sort(racl);
640        return racl.toArray(new String[racl.size()]);
641    }
642
643    protected Stream<State> getDescendants(String id, Set<String> keys, int limit) {
644        return repository.getDescendants(id, keys, limit);
645    }
646
647    public List<DBSDocumentState> getKeyValuedStates(String key, Object value) {
648        List<DBSDocumentState> docStates = new LinkedList<>();
649        Set<String> seen = new HashSet<>();
650        // check transient state
651        for (DBSDocumentState docState : transientStates.values()) {
652            if (!value.equals(docState.get(key))) {
653                continue;
654            }
655            docStates.add(docState);
656            seen.add(docState.getId());
657        }
658        // fetch from repository
659        List<State> states = repository.queryKeyValue(key, value, seen);
660        for (State state : states) {
661            docStates.add(newTransientState(state));
662        }
663        return docStates;
664    }
665
666    public List<DBSDocumentState> getKeyValuedStates(String key1, Object value1, String key2, Object value2) {
667        List<DBSDocumentState> docStates = new LinkedList<>();
668        Set<String> seen = new HashSet<>();
669        // check transient state
670        for (DBSDocumentState docState : transientStates.values()) {
671            seen.add(docState.getId());
672            if (!(value1.equals(docState.get(key1)) && value2.equals(docState.get(key2)))) {
673                continue;
674            }
675            docStates.add(docState);
676        }
677        // fetch from repository
678        List<State> states = repository.queryKeyValue(key1, value1, key2, value2, seen);
679        for (State state : states) {
680            docStates.add(newTransientState(state));
681        }
682        return docStates;
683    }
684
685    /**
686     * Removes a list of documents.
687     * <p>
688     * Called after a {@link #save} has been done.
689     */
690    public void removeStates(Set<String> ids) {
691        if (undoLog != null) {
692            for (String id : ids) {
693                if (undoLog.containsKey(id)) {
694                    // there's already a create or an update in the undo log
695                    State oldUndo = undoLog.get(id);
696                    if (oldUndo == null) {
697                        // create + delete -> forget
698                        undoLog.remove(id);
699                    } else {
700                        // update + delete -> original old state to re-create
701                        oldUndo.put(KEY_UNDOLOG_CREATE, TRUE);
702                    }
703                } else {
704                    // just delete -> store old state to re-create
705                    State oldState = StateHelper.deepCopy(getStateForRead(id));
706                    oldState.put(KEY_UNDOLOG_CREATE, TRUE);
707                    undoLog.put(id, oldState);
708                }
709            }
710        }
711        for (String id : ids) {
712            transientStates.remove(id);
713        }
714        repository.deleteStates(ids);
715    }
716
717    public void markUserChange(String id) {
718        userChangeIds.add(id);
719    }
720
721    /**
722     * Writes transient state to database.
723     * <p>
724     * An undo log is kept in order to rollback the transaction later if needed.
725     */
726    public void save() {
727        updateProxies();
728        List<Work> works;
729        if (!repository.isFulltextDisabled()) {
730            // TODO getting fulltext already does a getStateChange
731            works = getFulltextWorks();
732        } else {
733            works = Collections.emptyList();
734        }
735        List<State> statesToCreate = new ArrayList<>();
736        for (String id : transientCreated) { // ordered
737            DBSDocumentState docState = transientStates.get(id);
738            docState.setNotDirty();
739            if (undoLog != null) {
740                undoLog.put(id, null); // marker to denote create
741            }
742            State state = docState.getState();
743            state.put(KEY_CHANGE_TOKEN, INITIAL_CHANGE_TOKEN);
744            statesToCreate.add(state);
745        }
746        if (!statesToCreate.isEmpty()) {
747            repository.createStates(statesToCreate);
748        }
749        for (DBSDocumentState docState : transientStates.values()) {
750            String id = docState.getId();
751            if (transientCreated.contains(id)) {
752                continue; // already done
753            }
754            StateDiff diff = docState.getStateChange();
755            if (diff != null) {
756                try {
757                    if (undoLog != null) {
758                        if (!undoLog.containsKey(id)) {
759                            undoLog.put(id, StateHelper.deepCopy(docState.getOriginalState()));
760                        }
761                        // else there's already a create or an update in the undo log so original info is enough
762                    }
763                    ChangeTokenUpdater changeTokenUpdater;
764                    if (session.changeTokenEnabled) {
765                        // increment system change token
766                        Long base = (Long) docState.get(KEY_SYS_CHANGE_TOKEN);
767                        docState.put(KEY_SYS_CHANGE_TOKEN, DeltaLong.valueOf(base, 1));
768                        diff.put(KEY_SYS_CHANGE_TOKEN, DeltaLong.valueOf(base, 1));
769                        // update change token if applicable (user change)
770                        if (userChangeIds.contains(id)) {
771                            changeTokenUpdater = new ChangeTokenUpdater(docState);
772                        } else {
773                            changeTokenUpdater = null;
774                        }
775                    } else {
776                        changeTokenUpdater = null;
777                    }
778                    repository.updateState(id, diff, changeTokenUpdater);
779                } finally {
780                    docState.setNotDirty();
781                }
782            }
783        }
784        transientCreated.clear();
785        userChangeIds.clear();
786        scheduleWork(works);
787    }
788
789    /**
790     * Logic to get the conditions to use to match and update a change token.
791     * <p>
792     * This may be called several times for a single DBS document update, because the low-level storage may need several
793     * database updates for a single high-level update in some cases.
794     *
795     * @since 9.1
796     */
797    public static class ChangeTokenUpdater {
798
799        protected final DBSDocumentState docState;
800
801        protected Long oldToken;
802
803        public ChangeTokenUpdater(DBSDocumentState docState) {
804            this.docState = docState;
805            oldToken = (Long) docState.getOriginalState().get(KEY_CHANGE_TOKEN);
806        }
807
808        /**
809         * Gets the conditions to use to match a change token.
810         */
811        public Map<String, Serializable> getConditions() {
812            return Collections.singletonMap(KEY_CHANGE_TOKEN, oldToken);
813        }
814
815        /**
816         * Gets the updates to make to write the updated change token.
817         */
818        public Map<String, Serializable> getUpdates() {
819            Long newToken;
820            if (oldToken == null) {
821                // document without change token, just created
822                newToken = INITIAL_CHANGE_TOKEN;
823            } else {
824                newToken = BaseDocument.updateChangeToken(oldToken);
825            }
826            // also store the new token in the state (without marking dirty), for the next update
827            docState.getState().put(KEY_CHANGE_TOKEN, newToken);
828            oldToken = newToken;
829            return Collections.singletonMap(KEY_CHANGE_TOKEN, newToken);
830        }
831    }
832
833    protected void applyUndoLog() {
834        Set<String> deletes = new HashSet<>();
835        for (Entry<String, State> es : undoLog.entrySet()) {
836            String id = es.getKey();
837            State state = es.getValue();
838            if (state == null) {
839                deletes.add(id);
840            } else {
841                boolean recreate = state.remove(KEY_UNDOLOG_CREATE) != null;
842                if (recreate) {
843                    repository.createState(state);
844                } else {
845                    // undo update
846                    State currentState = repository.readState(id);
847                    if (currentState != null) {
848                        StateDiff diff = StateHelper.diff(currentState, state);
849                        if (!diff.isEmpty()) {
850                            repository.updateState(id, diff, null);
851                        }
852                    }
853                    // else we expected to read a current state but it was concurrently deleted...
854                    // in that case leave it deleted
855                }
856            }
857        }
858        if (!deletes.isEmpty()) {
859            repository.deleteStates(deletes);
860        }
861    }
862
863    /**
864     * Checks if the changed documents are proxy targets, and updates the proxies if that's the case.
865     */
866    protected void updateProxies() {
867        for (String id : transientCreated) { // ordered
868            DBSDocumentState docState = transientStates.get(id);
869            updateProxies(docState);
870        }
871        // copy as we may modify proxies
872        for (String id : transientStates.keySet().toArray(new String[0])) {
873            DBSDocumentState docState = transientStates.get(id);
874            if (transientCreated.contains(id)) {
875                continue; // already done
876            }
877            if (docState.isDirty()) {
878                updateProxies(docState);
879            }
880        }
881    }
882
883    protected void updateProxies(DBSDocumentState target) {
884        Object[] proxyIds = (Object[]) target.get(KEY_PROXY_IDS);
885        if (proxyIds != null) {
886            for (Object proxyId : proxyIds) {
887                try {
888                    updateProxy(target, (String) proxyId);
889                } catch (ConcurrentUpdateException e) {
890                    e.addInfo("On doc " + target.getId());
891                    log.error(e, e);
892                    // do not throw, this avoids crashing the session
893                }
894            }
895        }
896    }
897
898    /**
899     * Updates the state of a proxy based on its target.
900     */
901    protected void updateProxy(DBSDocumentState target, String proxyId) {
902        DBSDocumentState proxy = getStateForUpdate(proxyId);
903        if (proxy == null) {
904            throw new ConcurrentUpdateException("Proxy " + proxyId + " concurrently deleted");
905        }
906        SchemaManager schemaManager = Framework.getService(SchemaManager.class);
907        // clear all proxy data
908        for (String key : proxy.getState().keyArray()) {
909            if (!isProxySpecific(key, schemaManager)) {
910                proxy.put(key, null);
911            }
912        }
913        // copy from target
914        for (Entry<String, Serializable> en : target.getState().entrySet()) {
915            String key = en.getKey();
916            if (!isProxySpecific(key, schemaManager)) {
917                proxy.put(key, StateHelper.deepCopy(en.getValue()));
918            }
919        }
920    }
921
922    /**
923     * Things that we don't touch on a proxy when updating it.
924     */
925    protected boolean isProxySpecific(String key, SchemaManager schemaManager) {
926        switch (key) {
927        // these are placeful stuff
928        case KEY_ID:
929        case KEY_PARENT_ID:
930        case KEY_ANCESTOR_IDS:
931        case KEY_NAME:
932        case KEY_POS:
933        case KEY_ACP:
934        case KEY_READ_ACL:
935            // these are proxy-specific
936        case KEY_IS_PROXY:
937        case KEY_PROXY_TARGET_ID:
938        case KEY_PROXY_VERSION_SERIES_ID:
939        case KEY_IS_VERSION:
940        case KEY_PROXY_IDS:
941            return true;
942        }
943        int p = key.indexOf(':');
944        if (p == -1) {
945            // no prefix, assume not proxy-specific
946            return false;
947        }
948        String prefix = key.substring(0, p);
949        Schema schema = schemaManager.getSchemaFromPrefix(prefix);
950        if (schema == null) {
951            schema = schemaManager.getSchema(prefix);
952            if (schema == null) {
953                // unknown prefix, assume not proxy-specific
954                return false;
955            }
956        }
957        return schemaManager.isProxySchema(schema.getName(), null); // type unused
958    }
959
960    /**
961     * Called when created in a transaction.
962     *
963     * @since 7.4
964     */
965    public void begin() {
966        undoLog = new HashMap<>();
967        repository.begin();
968    }
969
970    /**
971     * Saves and flushes to database.
972     */
973    public void commit() {
974        save();
975        commitSave();
976        repository.commit();
977    }
978
979    /**
980     * Commits the saved state to the database.
981     */
982    protected void commitSave() {
983        // clear transient, this means that after this references to states will be stale
984        // TODO mark states as invalid
985        clearTransient();
986        // the transaction ended, the proxied DBSSession will disappear and cannot be reused anyway
987        undoLog = null;
988    }
989
990    /**
991     * Rolls back the save state by applying the undo log.
992     */
993    public void rollback() {
994        clearTransient();
995        applyUndoLog();
996        // the transaction ended, the proxied DBSSession will disappear and cannot be reused anyway
997        undoLog = null;
998        repository.rollback();
999    }
1000
1001    protected void clearTransient() {
1002        transientStates.clear();
1003        transientCreated.clear();
1004    }
1005
1006    /**
1007     * Gets the fulltext updates to do. Called at save() time.
1008     *
1009     * @return a list of {@link Work} instances to schedule post-commit.
1010     */
1011    protected List<Work> getFulltextWorks() {
1012        Set<String> docsWithDirtyStrings = new HashSet<>();
1013        Set<String> docsWithDirtyBinaries = new HashSet<>();
1014        findDirtyDocuments(docsWithDirtyStrings, docsWithDirtyBinaries);
1015        if (repository.getFulltextConfiguration().fulltextSearchDisabled) {
1016            // We only need to update dirty simple strings if fulltext search is not disabled
1017            // because in that case Elasticsearch will do its own extraction/indexing.
1018            // We need to detect dirty binary strings in all cases, because Elasticsearch
1019            // will need them even if the repository itself doesn't use them for search.
1020            docsWithDirtyStrings = Collections.emptySet();
1021        }
1022        Set<String> dirtyIds = new HashSet<>();
1023        dirtyIds.addAll(docsWithDirtyStrings);
1024        dirtyIds.addAll(docsWithDirtyBinaries);
1025        if (dirtyIds.isEmpty()) {
1026            return Collections.emptyList();
1027        }
1028        markIndexingInProgress(dirtyIds);
1029        List<Work> works = new ArrayList<>(dirtyIds.size());
1030        for (String id : dirtyIds) {
1031            boolean updateSimpleText = docsWithDirtyStrings.contains(id);
1032            boolean updateBinaryText = docsWithDirtyBinaries.contains(id);
1033            Work work = new FulltextExtractorWork(repository.getName(), id, updateSimpleText, updateBinaryText, true);
1034            works.add(work);
1035        }
1036        return works;
1037    }
1038
1039    protected void markIndexingInProgress(Set<String> ids) {
1040        FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration();
1041        for (DBSDocumentState docState : getStatesForUpdate(ids)) {
1042            if (!fulltextConfiguration.isFulltextIndexable(docState.getPrimaryType())) {
1043                continue;
1044            }
1045            docState.put(KEY_FULLTEXT_JOBID, docState.getId());
1046        }
1047    }
1048
1049    /**
1050     * Finds the documents having dirty text or dirty binaries that have to be reindexed as fulltext.
1051     *
1052     * @param docsWithDirtyStrings set of ids, updated by this method
1053     * @param docsWithDirtyBinaries set of ids, updated by this method
1054     */
1055    protected void findDirtyDocuments(Set<String> docsWithDirtyStrings, Set<String> docsWithDirtyBinaries) {
1056        for (DBSDocumentState docState : transientStates.values()) {
1057            State originalState = docState.getOriginalState();
1058            State state = docState.getState();
1059            if (originalState == state) {
1060                continue;
1061            }
1062            StateDiff diff = StateHelper.diff(originalState, state);
1063            if (diff.isEmpty()) {
1064                continue;
1065            }
1066            StateDiff rdiff = StateHelper.diff(state, originalState);
1067            // we do diffs in both directions to capture removal of complex list elements,
1068            // for instance for {foo: [{bar: baz}] -> {foo: []}
1069            // diff paths = foo and rdiff paths = foo/*/bar
1070            Set<String> paths = new HashSet<>();
1071            DirtyPathsFinder dirtyPathsFinder = new DirtyPathsFinder(paths);
1072            dirtyPathsFinder.findDirtyPaths(diff);
1073            dirtyPathsFinder.findDirtyPaths(rdiff);
1074            FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration();
1075            boolean dirtyStrings = false;
1076            boolean dirtyBinaries = false;
1077            for (String path : paths) {
1078                Set<String> indexesSimple = fulltextConfiguration.indexesByPropPathSimple.get(path);
1079                if (indexesSimple != null && !indexesSimple.isEmpty()) {
1080                    dirtyStrings = true;
1081                    if (dirtyBinaries) {
1082                        break;
1083                    }
1084                }
1085                Set<String> indexesBinary = fulltextConfiguration.indexesByPropPathBinary.get(path);
1086                if (indexesBinary != null && !indexesBinary.isEmpty()) {
1087                    dirtyBinaries = true;
1088                    if (dirtyStrings) {
1089                        break;
1090                    }
1091                }
1092            }
1093            if (dirtyStrings) {
1094                docsWithDirtyStrings.add(docState.getId());
1095            }
1096            if (dirtyBinaries) {
1097                docsWithDirtyBinaries.add(docState.getId());
1098            }
1099        }
1100    }
1101
1102    /**
1103     * Iterates on a state diff to find the paths corresponding to dirty values.
1104     *
1105     * @since 7.10-HF04, 8.1
1106     */
1107    protected static class DirtyPathsFinder {
1108
1109        protected Set<String> paths;
1110
1111        public DirtyPathsFinder(Set<String> paths) {
1112            this.paths = paths;
1113        }
1114
1115        public void findDirtyPaths(StateDiff value) {
1116            findDirtyPaths(value, null);
1117        }
1118
1119        protected void findDirtyPaths(Object value, String path) {
1120            if (value instanceof Object[]) {
1121                findDirtyPaths((Object[]) value, path);
1122            } else if (value instanceof List) {
1123                findDirtyPaths((List<?>) value, path);
1124            } else if (value instanceof ListDiff) {
1125                findDirtyPaths((ListDiff) value, path);
1126            } else if (value instanceof State) {
1127                findDirtyPaths((State) value, path);
1128            } else {
1129                paths.add(path);
1130            }
1131        }
1132
1133        protected void findDirtyPaths(Object[] value, String path) {
1134            String newPath = path + "/*";
1135            for (Object v : value) {
1136                findDirtyPaths(v, newPath);
1137            }
1138        }
1139
1140        protected void findDirtyPaths(List<?> value, String path) {
1141            String newPath = path + "/*";
1142            for (Object v : value) {
1143                findDirtyPaths(v, newPath);
1144            }
1145        }
1146
1147        protected void findDirtyPaths(ListDiff value, String path) {
1148            String newPath = path + "/*";
1149            if (value.diff != null) {
1150                findDirtyPaths(value.diff, newPath);
1151            }
1152            if (value.rpush != null) {
1153                findDirtyPaths(value.rpush, newPath);
1154            }
1155        }
1156
1157        protected void findDirtyPaths(State value, String path) {
1158            for (Entry<String, Serializable> es : value.entrySet()) {
1159                String key = es.getKey();
1160                Serializable v = es.getValue();
1161                String newPath = path == null ? key : path + "/" + key;
1162                findDirtyPaths(v, newPath);
1163            }
1164        }
1165    }
1166
1167    protected void scheduleWork(List<Work> works) {
1168        // do async fulltext indexing only if high-level sessions are available
1169        RepositoryManager repositoryManager = Framework.getService(RepositoryManager.class);
1170        if (repositoryManager != null && !works.isEmpty()) {
1171            WorkManager workManager = Framework.getService(WorkManager.class);
1172            for (Work work : works) {
1173                // schedule work post-commit
1174                // in non-tx mode, this may execute it nearly immediately
1175                workManager.schedule(work, Scheduling.IF_NOT_SCHEDULED, true);
1176            }
1177        }
1178    }
1179
1180}