001/*
002 * (C) Copyright 2014-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.dbs;
020
021import static java.lang.Boolean.FALSE;
022import static java.lang.Boolean.TRUE;
023import static org.nuxeo.ecm.core.api.security.SecurityConstants.BROWSE;
024import static org.nuxeo.ecm.core.api.security.SecurityConstants.EVERYONE;
025import static org.nuxeo.ecm.core.api.security.SecurityConstants.UNSUPPORTED_ACL;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_CHANGE_TOKEN;
027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_SYS_CHANGE_TOKEN;
028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_GRANT;
029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_PERMISSION;
030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_STATUS;
031import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_USER;
032import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
033import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
034import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ANCESTOR_IDS;
035import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_CHANGE_TOKEN;
036import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_JOBID;
037import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ID;
038import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_PROXY;
039import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_VERSION;
040import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_MIXIN_TYPES;
041import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_NAME;
042import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PARENT_ID;
043import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_POS;
044import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PREFIX;
045import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PRIMARY_TYPE;
046import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_IDS;
047import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_TARGET_ID;
048import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_VERSION_SERIES_ID;
049import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_READ_ACL;
050import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_SYS_CHANGE_TOKEN;
051import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_SERIES_ID;
052
053import java.io.Serializable;
054import java.security.Principal;
055import java.util.ArrayList;
056import java.util.Arrays;
057import java.util.Collection;
058import java.util.Collections;
059import java.util.HashMap;
060import java.util.HashSet;
061import java.util.LinkedHashSet;
062import java.util.LinkedList;
063import java.util.List;
064import java.util.Map;
065import java.util.Map.Entry;
066import java.util.stream.Stream;
067import java.util.Set;
068
069import org.apache.commons.lang.StringUtils;
070import org.apache.commons.logging.Log;
071import org.apache.commons.logging.LogFactory;
072import org.nuxeo.ecm.core.BatchFinderWork;
073import org.nuxeo.ecm.core.BatchProcessorWork;
074import org.nuxeo.ecm.core.api.ConcurrentUpdateException;
075import org.nuxeo.ecm.core.api.model.DeltaLong;
076import org.nuxeo.ecm.core.api.PartialList;
077import org.nuxeo.ecm.core.api.SystemPrincipal;
078import org.nuxeo.ecm.core.api.repository.RepositoryManager;
079import org.nuxeo.ecm.core.query.QueryFilter;
080import org.nuxeo.ecm.core.query.sql.NXQL;
081import org.nuxeo.ecm.core.schema.SchemaManager;
082import org.nuxeo.ecm.core.schema.types.Schema;
083import org.nuxeo.ecm.core.security.SecurityService;
084import org.nuxeo.ecm.core.storage.BaseDocument;
085import org.nuxeo.ecm.core.storage.DefaultFulltextParser;
086import org.nuxeo.ecm.core.storage.FulltextConfiguration;
087import org.nuxeo.ecm.core.storage.FulltextParser;
088import org.nuxeo.ecm.core.storage.FulltextUpdaterWork;
089import org.nuxeo.ecm.core.storage.FulltextUpdaterWork.IndexAndText;
090import org.nuxeo.ecm.core.storage.State;
091import org.nuxeo.ecm.core.storage.State.ListDiff;
092import org.nuxeo.ecm.core.storage.State.StateDiff;
093import org.nuxeo.ecm.core.storage.StateHelper;
094import org.nuxeo.ecm.core.work.api.Work;
095import org.nuxeo.ecm.core.work.api.WorkManager;
096import org.nuxeo.ecm.core.work.api.WorkManager.Scheduling;
097import org.nuxeo.runtime.api.Framework;
098
099/**
100 * Transactional state for a session.
101 * <p>
102 * Until {@code save()} is called, data lives in the transient map.
103 * <p>
104 * Upon save, data is written to the repository, even though it has not yet been committed (this means that other
105 * sessions can read uncommitted data). It's also kept in an undo log in order for rollback to be possible.
106 * <p>
107 * On commit, the undo log is forgotten. On rollback, the undo log is replayed.
108 *
109 * @since 5.9.4
110 */
111public class DBSTransactionState {
112
113    private static final Log log = LogFactory.getLog(DBSTransactionState.class);
114
115    private static final String KEY_UNDOLOG_CREATE = "__UNDOLOG_CREATE__\0\0";
116
117    /** Keys used when computing Read ACLs. */
118    protected static final Set<String> READ_ACL_RECURSION_KEYS = new HashSet<>(
119            Arrays.asList(KEY_READ_ACL, KEY_ACP, KEY_IS_VERSION, KEY_VERSION_SERIES_ID, KEY_PARENT_ID));
120
121    public static final String READ_ACL_ASYNC_ENABLED_PROPERTY = "nuxeo.core.readacl.async.enabled";
122
123    public static final String READ_ACL_ASYNC_ENABLED_DEFAULT = "true";
124
125    public static final String READ_ACL_ASYNC_THRESHOLD_PROPERTY = "nuxeo.core.readacl.async.threshold";
126
127    public static final String READ_ACL_ASYNC_THRESHOLD_DEFAULT = "500";
128
129    protected final DBSRepository repository;
130
131    protected final DBSSession session;
132
133    /** Retrieved and created document state. */
134    protected Map<String, DBSDocumentState> transientStates = new HashMap<>();
135
136    /** Ids of documents created but not yet saved. */
137    protected Set<String> transientCreated = new LinkedHashSet<>();
138
139    /**
140     * Document ids modified as "user changes", which means that a change token should be checked.
141     *
142     * @since 9.2
143     */
144    protected final Set<Serializable> userChangeIds = new HashSet<>();
145
146    /**
147     * Undo log.
148     * <p>
149     * A map of document ids to null or State. The value is null when the document has to be deleted when applying the
150     * undo log. Otherwise the value is a State. If the State contains the key {@link #KEY_UNDOLOG_CREATE} then the
151     * state must be re-created completely when applying the undo log, otherwise just applied as an update.
152     * <p>
153     * Null when there is no active transaction.
154     */
155    protected Map<String, State> undoLog;
156
157    protected final Set<String> browsePermissions;
158
159    public DBSTransactionState(DBSRepository repository, DBSSession session) {
160        this.repository = repository;
161        this.session = session;
162        SecurityService securityService = Framework.getService(SecurityService.class);
163        browsePermissions = new HashSet<>(Arrays.asList(securityService.getPermissionsToCheck(BROWSE)));
164    }
165
166    /**
167     * New transient state for something just read from the repository.
168     */
169    protected DBSDocumentState newTransientState(State state) {
170        if (state == null) {
171            return null;
172        }
173        String id = (String) state.get(KEY_ID);
174        if (transientStates.containsKey(id)) {
175            throw new IllegalStateException("Already transient: " + id);
176        }
177        DBSDocumentState docState = new DBSDocumentState(state); // copy
178        transientStates.put(id, docState);
179        return docState;
180    }
181
182    /**
183     * Returns a state and marks it as transient, because it's about to be modified or returned to user code (where it
184     * may be modified).
185     */
186    public DBSDocumentState getStateForUpdate(String id) {
187        // check transient state
188        DBSDocumentState docState = transientStates.get(id);
189        if (docState != null) {
190            return docState;
191        }
192        // fetch from repository
193        State state = repository.readState(id);
194        return newTransientState(state);
195    }
196
197    /**
198     * Returns a state which won't be modified.
199     */
200    // TODO in some cases it's good to have this kept in memory instead of
201    // rereading from database every time
202    // XXX getStateForReadOneShot
203    public State getStateForRead(String id) {
204        // check transient state
205        DBSDocumentState docState = transientStates.get(id);
206        if (docState != null) {
207            return docState.getState();
208        }
209        // fetch from repository
210        return repository.readState(id);
211    }
212
213    /**
214     * Returns states and marks them transient, because they're about to be returned to user code (where they may be
215     * modified).
216     */
217    public List<DBSDocumentState> getStatesForUpdate(List<String> ids) {
218        // check which ones we have to fetch from repository
219        List<String> idsToFetch = new LinkedList<>();
220        for (String id : ids) {
221            // check transient state
222            DBSDocumentState docState = transientStates.get(id);
223            if (docState != null) {
224                continue;
225            }
226            // will have to fetch it
227            idsToFetch.add(id);
228        }
229        if (!idsToFetch.isEmpty()) {
230            List<State> states = repository.readStates(idsToFetch);
231            for (State state : states) {
232                newTransientState(state);
233            }
234        }
235        // everything now fetched in transient
236        List<DBSDocumentState> docStates = new ArrayList<>(ids.size());
237        for (String id : ids) {
238            DBSDocumentState docState = transientStates.get(id);
239            if (docState == null) {
240                if (log.isTraceEnabled()) {
241                    log.trace("Cannot fetch document with id: " + id, new Throwable("debug stack trace"));
242                }
243                continue;
244            }
245            docStates.add(docState);
246        }
247        return docStates;
248    }
249
250    // XXX TODO for update or for read?
251    public DBSDocumentState getChildState(String parentId, String name) {
252        // check transient state
253        for (DBSDocumentState docState : transientStates.values()) {
254            if (!parentId.equals(docState.getParentId())) {
255                continue;
256            }
257            if (!name.equals(docState.getName())) {
258                continue;
259            }
260            return docState;
261        }
262        // fetch from repository
263        State state = repository.readChildState(parentId, name, Collections.emptySet());
264        if (state == null) {
265            return null;
266        }
267        String id = (String) state.get(KEY_ID);
268        if (transientStates.containsKey(id)) {
269            // found transient, even though we already checked
270            // that means that in-memory it's not a child, but in-database it's a child (was moved)
271            // -> ignore the database state
272            return null;
273        }
274        return newTransientState(state);
275    }
276
277    public boolean hasChild(String parentId, String name) {
278        // check transient state
279        for (DBSDocumentState docState : transientStates.values()) {
280            if (!parentId.equals(docState.getParentId())) {
281                continue;
282            }
283            if (!name.equals(docState.getName())) {
284                continue;
285            }
286            return true;
287        }
288        // check repository
289        return repository.hasChild(parentId, name, Collections.emptySet());
290    }
291
292    public List<DBSDocumentState> getChildrenStates(String parentId) {
293        List<DBSDocumentState> docStates = new LinkedList<>();
294        Set<String> seen = new HashSet<>();
295        // check transient state
296        for (DBSDocumentState docState : transientStates.values()) {
297            if (!parentId.equals(docState.getParentId())) {
298                continue;
299            }
300            docStates.add(docState);
301            seen.add(docState.getId());
302        }
303        // fetch from repository
304        List<State> states = repository.queryKeyValue(KEY_PARENT_ID, parentId, seen);
305        for (State state : states) {
306            String id = (String) state.get(KEY_ID);
307            if (transientStates.containsKey(id)) {
308                // found transient, even though we passed an exclusion list for known children
309                // that means that in-memory it's not a child, but in-database it's a child (was moved)
310                // -> ignore the database state
311                continue;
312            }
313            docStates.add(newTransientState(state));
314        }
315        return docStates;
316    }
317
318    public List<String> getChildrenIds(String parentId) {
319        List<String> children = new ArrayList<>();
320        Set<String> seen = new HashSet<>();
321        // check transient state
322        for (DBSDocumentState docState : transientStates.values()) {
323            String id = docState.getId();
324            if (!parentId.equals(docState.getParentId())) {
325                continue;
326            }
327            seen.add(id);
328            children.add(id);
329        }
330        // fetch from repository
331        List<State> states = repository.queryKeyValue(KEY_PARENT_ID, parentId, seen);
332        for (State state : states) {
333            String id = (String) state.get(KEY_ID);
334            if (transientStates.containsKey(id)) {
335                // found transient, even though we passed an exclusion list for known children
336                // that means that in-memory it's not a child, but in-database it's a child (was moved)
337                // -> ignore the database state
338                continue;
339            }
340            children.add(id);
341        }
342        return new ArrayList<>(children);
343    }
344
345    public boolean hasChildren(String parentId) {
346        // check transient state
347        for (DBSDocumentState docState : transientStates.values()) {
348            if (!parentId.equals(docState.getParentId())) {
349                continue;
350            }
351            return true;
352        }
353        // check repository
354        return repository.queryKeyValuePresence(KEY_PARENT_ID, parentId, Collections.emptySet());
355    }
356
357    public DBSDocumentState createChild(String id, String parentId, String name, Long pos, String typeName) {
358        // id may be not-null for import
359        if (id == null) {
360            id = repository.generateNewId();
361        }
362        transientCreated.add(id);
363        DBSDocumentState docState = new DBSDocumentState();
364        transientStates.put(id, docState);
365        docState.put(KEY_ID, id);
366        docState.put(KEY_PARENT_ID, parentId);
367        docState.put(KEY_ANCESTOR_IDS, getAncestorIds(parentId));
368        docState.put(KEY_NAME, name);
369        docState.put(KEY_POS, pos);
370        docState.put(KEY_PRIMARY_TYPE, typeName);
371        if (session.changeTokenEnabled) {
372            docState.put(KEY_SYS_CHANGE_TOKEN, INITIAL_SYS_CHANGE_TOKEN);
373        }
374        // update read acls for new doc
375        updateDocumentReadAcls(id);
376        return docState;
377    }
378
379    /** Gets ancestors including id itself. */
380    protected Object[] getAncestorIds(String id) {
381        if (id == null) {
382            return null;
383        }
384        State state = getStateForRead(id);
385        if (state == null) {
386            throw new RuntimeException("No such id: " + id);
387        }
388        Object[] ancestors = (Object[]) state.get(KEY_ANCESTOR_IDS);
389        if (ancestors == null) {
390            return new Object[] { id };
391        } else {
392            Object[] newAncestors = new Object[ancestors.length + 1];
393            System.arraycopy(ancestors, 0, newAncestors, 0, ancestors.length);
394            newAncestors[ancestors.length] = id;
395            return newAncestors;
396        }
397    }
398
399    /**
400     * Copies the document into a newly-created object.
401     * <p>
402     * The copy is automatically saved.
403     */
404    public DBSDocumentState copy(String id) {
405        DBSDocumentState copyState = new DBSDocumentState(getStateForRead(id));
406        String copyId = repository.generateNewId();
407        copyState.put(KEY_ID, copyId);
408        copyState.put(KEY_PROXY_IDS, null); // no proxies to this new doc
409        // other fields updated by the caller
410        transientStates.put(copyId, copyState);
411        transientCreated.add(copyId);
412        return copyState;
413    }
414
415    /**
416     * Updates ancestors recursively after a move.
417     * <p>
418     * Recursing from given doc, replace the first ndel ancestors with those passed.
419     * <p>
420     * Doesn't check transient (assumes save is done). The modifications are automatically saved.
421     */
422    public void updateAncestors(String id, int ndel, Object[] ancestorIds) {
423        int nadd = ancestorIds.length;
424        Set<String> ids = new HashSet<>();
425        ids.add(id);
426        try (Stream<State> states = getDescendants(id, Collections.emptySet(), 0)) {
427            states.forEach(state -> ids.add((String) state.get(KEY_ID)));
428        }
429        // we collect all ids first to avoid reentrancy to the repository
430        for (String cid : ids) {
431            // XXX TODO oneShot update, don't pollute transient space
432            DBSDocumentState docState = getStateForUpdate(cid);
433            Object[] ancestors = (Object[]) docState.get(KEY_ANCESTOR_IDS);
434            Object[] newAncestors;
435            if (ancestors == null) {
436                newAncestors = ancestorIds.clone();
437            } else {
438                newAncestors = new Object[ancestors.length - ndel + nadd];
439                System.arraycopy(ancestorIds, 0, newAncestors, 0, nadd);
440                System.arraycopy(ancestors, ndel, newAncestors, nadd, ancestors.length - ndel);
441            }
442            docState.put(KEY_ANCESTOR_IDS, newAncestors);
443        }
444    }
445
446    protected int getReadAclsAsyncThreshold() {
447        boolean enabled = Boolean.parseBoolean(
448                Framework.getProperty(READ_ACL_ASYNC_ENABLED_PROPERTY, READ_ACL_ASYNC_ENABLED_DEFAULT));
449        if (enabled) {
450            return Integer.parseInt(
451                    Framework.getProperty(READ_ACL_ASYNC_THRESHOLD_PROPERTY, READ_ACL_ASYNC_THRESHOLD_DEFAULT));
452        } else {
453            return 0;
454        }
455    }
456
457    /**
458     * Updates the Read ACLs recursively on a document.
459     */
460    public void updateTreeReadAcls(String id) {
461        // versions too XXX TODO
462
463        save(); // flush everything to the database
464
465        // update the doc itself
466        updateDocumentReadAcls(id);
467
468        // check if we have a small enough number of descendants that we can process them synchronously
469        int limit = getReadAclsAsyncThreshold();
470        Set<String> ids = new HashSet<>();
471        try (Stream<State> states = getDescendants(id, Collections.emptySet(), limit)) {
472            states.forEach(state -> ids.add((String) state.get(KEY_ID)));
473        }
474        if (limit == 0 || ids.size() < limit) {
475            // update all descendants synchronously
476            ids.forEach(this::updateDocumentReadAcls);
477        } else {
478            // update the direct children synchronously, the rest asynchronously
479
480            // update the direct children (with a limit in case it's too big)
481            String nxql = String.format("SELECT ecm:uuid FROM Document WHERE ecm:parentId = '%s'", id);
482            Principal principal = new SystemPrincipal(null);
483            QueryFilter queryFilter = new QueryFilter(principal, null, null, null, Collections.emptyList(), limit, 0);
484            PartialList<Map<String, Serializable>> pl = session.queryProjection(nxql, NXQL.NXQL, queryFilter, false, 0,
485                    new Object[0]);
486            for (Map<String, Serializable> map : pl) {
487                String childId = (String) map.get(NXQL.ECM_UUID);
488                updateDocumentReadAcls(childId);
489            }
490
491            // asynchronous work to do the whole tree
492            nxql = String.format("SELECT ecm:uuid FROM Document WHERE ecm:ancestorId = '%s'", id);
493            Work work = new FindReadAclsWork(repository.getName(), nxql, null);
494            Framework.getService(WorkManager.class).schedule(work);
495        }
496    }
497
498    /**
499     * Work to find the ids of documents for which Read ACLs must be recomputed, and launch the needed update works.
500     *
501     * @since 9.10
502     */
503    public static class FindReadAclsWork extends BatchFinderWork {
504
505        private static final long serialVersionUID = 1L;
506
507        public FindReadAclsWork(String repositoryName, String nxql, String originatingUsername) {
508            super(repositoryName, nxql, originatingUsername);
509        }
510
511        @Override
512        public String getTitle() {
513            return "Find descendants for Read ACLs";
514        }
515
516        @Override
517        public String getCategory() {
518            return "security";
519        }
520
521        @Override
522        public int getBatchSize() {
523            return 500;
524        }
525
526        @Override
527        public Work getBatchProcessorWork(List<String> docIds) {
528            return new UpdateReadAclsWork(repositoryName, docIds, getOriginatingUsername());
529        }
530    }
531
532    /**
533     * Work to update the Read ACLs on a list of documents, without recursion.
534     *
535     * @since 9.10
536     */
537    public static class UpdateReadAclsWork extends BatchProcessorWork {
538
539        private static final long serialVersionUID = 1L;
540
541        public UpdateReadAclsWork(String repositoryName, List<String> docIds, String originatingUsername) {
542            super(repositoryName, docIds, originatingUsername);
543        }
544
545        @Override
546        public String getTitle() {
547            return "Update Read ACLs";
548        }
549
550        @Override
551        public String getCategory() {
552            return "security";
553        }
554
555        @Override
556        public int getBatchSize() {
557            return 50;
558        }
559
560        @Override
561        public void processBatch(List<String> docIds) {
562            session.updateReadACLs(docIds);
563        }
564    }
565
566    /**
567     * Updates the Read ACLs on a document (not recursively), bypassing transient space and caches for the document
568     * itself (not the ancestors, needed for ACL inheritance and for which caching is useful).
569     */
570    public void updateReadACLs(Collection<String> docIds) {
571        docIds.forEach(id -> updateDocumentReadAclsNoCache(id));
572    }
573
574    /**
575     * Updates the Read ACLs on a document (not recursively)
576     */
577    protected void updateDocumentReadAcls(String id) {
578        DBSDocumentState docState = getStateForUpdate(id);
579        docState.put(KEY_READ_ACL, getReadACL(docState.getState()));
580    }
581
582    /**
583     * Updates the Read ACLs on a document, without polluting caches.
584     * <p>
585     * When fetching parents recursively to compute inheritance, the regular transient space and repository caching are
586     * used.
587     */
588    protected void updateDocumentReadAclsNoCache(String id) {
589        // no transient for state read, and we don't want to trash caches
590        // fetch from repository only the properties needed for Read ACL computation and recursion
591        State state = repository.readPartialState(id, READ_ACL_RECURSION_KEYS);
592        State oldState = new State(1);
593        oldState.put(KEY_READ_ACL, state.get(KEY_READ_ACL));
594        // compute new value
595        State newState = new State(1);
596        newState.put(KEY_READ_ACL, getReadACL(state));
597        StateDiff diff = StateHelper.diff(oldState, newState);
598        if (!diff.isEmpty()) {
599            // no transient for state write, we write directly and just invalidate caches
600            repository.updateState(id, diff, null);
601        }
602    }
603
604    /**
605     * Gets the Read ACL (flat list of users having browse permission, including inheritance) on a document.
606     */
607    protected String[] getReadACL(State state) {
608        Set<String> racls = new HashSet<>();
609        LOOP: do {
610            @SuppressWarnings("unchecked")
611            List<Serializable> aclList = (List<Serializable>) state.get(KEY_ACP);
612            if (aclList != null) {
613                for (Serializable aclSer : aclList) {
614                    State aclMap = (State) aclSer;
615                    @SuppressWarnings("unchecked")
616                    List<Serializable> aceList = (List<Serializable>) aclMap.get(KEY_ACL);
617                    for (Serializable aceSer : aceList) {
618                        State aceMap = (State) aceSer;
619                        String username = (String) aceMap.get(KEY_ACE_USER);
620                        String permission = (String) aceMap.get(KEY_ACE_PERMISSION);
621                        Boolean granted = (Boolean) aceMap.get(KEY_ACE_GRANT);
622                        Long status = (Long) aceMap.get(KEY_ACE_STATUS);
623                        if (TRUE.equals(granted) && browsePermissions.contains(permission)
624                                && (status == null || status == 1)) {
625                            racls.add(username);
626                        }
627                        if (FALSE.equals(granted)) {
628                            if (!EVERYONE.equals(username)) {
629                                // TODO log
630                                racls.add(UNSUPPORTED_ACL);
631                            }
632                            break LOOP;
633                        }
634                    }
635                }
636            }
637            // get the parent; for a version the parent is the live document
638            String parentKey = TRUE.equals(state.get(KEY_IS_VERSION)) ? KEY_VERSION_SERIES_ID : KEY_PARENT_ID;
639            String parentId = (String) state.get(parentKey);
640            state = parentId == null ? null : getStateForRead(parentId);
641        } while (state != null);
642
643        // sort to have canonical order
644        List<String> racl = new ArrayList<>(racls);
645        Collections.sort(racl);
646        return racl.toArray(new String[racl.size()]);
647    }
648
649    protected Stream<State> getDescendants(String id, Set<String> keys, int limit) {
650        return repository.getDescendants(id, keys, limit);
651    }
652
653    public List<DBSDocumentState> getKeyValuedStates(String key, Object value) {
654        List<DBSDocumentState> docStates = new LinkedList<>();
655        Set<String> seen = new HashSet<>();
656        // check transient state
657        for (DBSDocumentState docState : transientStates.values()) {
658            if (!value.equals(docState.get(key))) {
659                continue;
660            }
661            docStates.add(docState);
662            seen.add(docState.getId());
663        }
664        // fetch from repository
665        List<State> states = repository.queryKeyValue(key, value, seen);
666        for (State state : states) {
667            docStates.add(newTransientState(state));
668        }
669        return docStates;
670    }
671
672    public List<DBSDocumentState> getKeyValuedStates(String key1, Object value1, String key2, Object value2) {
673        List<DBSDocumentState> docStates = new LinkedList<>();
674        Set<String> seen = new HashSet<>();
675        // check transient state
676        for (DBSDocumentState docState : transientStates.values()) {
677            seen.add(docState.getId());
678            if (!(value1.equals(docState.get(key1)) && value2.equals(docState.get(key2)))) {
679                continue;
680            }
681            docStates.add(docState);
682        }
683        // fetch from repository
684        List<State> states = repository.queryKeyValue(key1, value1, key2, value2, seen);
685        for (State state : states) {
686            docStates.add(newTransientState(state));
687        }
688        return docStates;
689    }
690
691    /**
692     * Removes a list of documents.
693     * <p>
694     * Called after a {@link #save} has been done.
695     */
696    public void removeStates(Set<String> ids) {
697        if (undoLog != null) {
698            for (String id : ids) {
699                if (undoLog.containsKey(id)) {
700                    // there's already a create or an update in the undo log
701                    State oldUndo = undoLog.get(id);
702                    if (oldUndo == null) {
703                        // create + delete -> forget
704                        undoLog.remove(id);
705                    } else {
706                        // update + delete -> original old state to re-create
707                        oldUndo.put(KEY_UNDOLOG_CREATE, TRUE);
708                    }
709                } else {
710                    // just delete -> store old state to re-create
711                    State oldState = StateHelper.deepCopy(getStateForRead(id));
712                    oldState.put(KEY_UNDOLOG_CREATE, TRUE);
713                    undoLog.put(id, oldState);
714                }
715            }
716        }
717        for (String id : ids) {
718            transientStates.remove(id);
719        }
720        repository.deleteStates(ids);
721    }
722
723    public void markUserChange(String id) {
724        userChangeIds.add(id);
725    }
726
727    /**
728     * Writes transient state to database.
729     * <p>
730     * An undo log is kept in order to rollback the transaction later if needed.
731     */
732    public void save() {
733        updateProxies();
734        List<Work> works;
735        if (!repository.isFulltextDisabled()) {
736            // TODO getting fulltext already does a getStateChange
737            works = getFulltextWorks();
738        } else {
739            works = Collections.emptyList();
740        }
741        List<State> statesToCreate = new ArrayList<>();
742        for (String id : transientCreated) { // ordered
743            DBSDocumentState docState = transientStates.get(id);
744            docState.setNotDirty();
745            if (undoLog != null) {
746                undoLog.put(id, null); // marker to denote create
747            }
748            State state = docState.getState();
749            state.put(KEY_CHANGE_TOKEN, INITIAL_CHANGE_TOKEN);
750            statesToCreate.add(state);
751        }
752        if (!statesToCreate.isEmpty()) {
753            repository.createStates(statesToCreate);
754        }
755        for (DBSDocumentState docState : transientStates.values()) {
756            String id = docState.getId();
757            if (transientCreated.contains(id)) {
758                continue; // already done
759            }
760            StateDiff diff = docState.getStateChange();
761            if (diff != null) {
762                if (undoLog != null) {
763                    if (!undoLog.containsKey(id)) {
764                        undoLog.put(id, StateHelper.deepCopy(docState.getOriginalState()));
765                    }
766                    // else there's already a create or an update in the undo log so original info is enough
767                }
768                ChangeTokenUpdater changeTokenUpdater;
769                if (session.changeTokenEnabled) {
770                    // increment system change token
771                    Long base = (Long) docState.get(KEY_SYS_CHANGE_TOKEN);
772                    docState.put(KEY_SYS_CHANGE_TOKEN, DeltaLong.valueOf(base, 1));
773                    diff.put(KEY_SYS_CHANGE_TOKEN, DeltaLong.valueOf(base, 1));
774                    // update change token if applicable (user change)
775                    if (userChangeIds.contains(id)) {
776                        changeTokenUpdater = new ChangeTokenUpdater(docState);
777                    } else {
778                        changeTokenUpdater = null;
779                    }
780                } else {
781                    changeTokenUpdater = null;
782                }
783                repository.updateState(id, diff, changeTokenUpdater);
784            }
785            docState.setNotDirty();
786        }
787        transientCreated.clear();
788        userChangeIds.clear();
789        scheduleWork(works);
790    }
791
792    /**
793     * Logic to get the conditions to use to match and update a change token.
794     * <p>
795     * This may be called several times for a single DBS document update, because the low-level storage may need several
796     * database updates for a single high-level update in some cases.
797     *
798     * @since 9.1
799     */
800    public static class ChangeTokenUpdater {
801
802        protected final DBSDocumentState docState;
803
804        protected Long oldToken;
805
806        public ChangeTokenUpdater(DBSDocumentState docState) {
807            this.docState = docState;
808            oldToken = (Long) docState.getOriginalState().get(KEY_CHANGE_TOKEN);
809        }
810
811        /**
812         * Gets the conditions to use to match a change token.
813         */
814        public Map<String, Serializable> getConditions() {
815            return Collections.singletonMap(KEY_CHANGE_TOKEN, oldToken);
816        }
817
818        /**
819         * Gets the updates to make to write the updated change token.
820         */
821        public Map<String, Serializable> getUpdates() {
822            Long newToken;
823            if (oldToken == null) {
824                // document without change token, just created
825                newToken = INITIAL_CHANGE_TOKEN;
826            } else {
827                newToken = BaseDocument.updateChangeToken(oldToken);
828            }
829            // also store the new token in the state (without marking dirty), for the next update
830            docState.getState().put(KEY_CHANGE_TOKEN, newToken);
831            oldToken = newToken;
832            return Collections.singletonMap(KEY_CHANGE_TOKEN, newToken);
833        }
834    }
835
836    protected void applyUndoLog() {
837        Set<String> deletes = new HashSet<>();
838        for (Entry<String, State> es : undoLog.entrySet()) {
839            String id = es.getKey();
840            State state = es.getValue();
841            if (state == null) {
842                deletes.add(id);
843            } else {
844                boolean recreate = state.remove(KEY_UNDOLOG_CREATE) != null;
845                if (recreate) {
846                    repository.createState(state);
847                } else {
848                    // undo update
849                    State currentState = repository.readState(id);
850                    if (currentState != null) {
851                        StateDiff diff = StateHelper.diff(currentState, state);
852                        if (!diff.isEmpty()) {
853                            repository.updateState(id, diff, null);
854                        }
855                    }
856                    // else we expected to read a current state but it was concurrently deleted...
857                    // in that case leave it deleted
858                }
859            }
860        }
861        if (!deletes.isEmpty()) {
862            repository.deleteStates(deletes);
863        }
864    }
865
866    /**
867     * Checks if the changed documents are proxy targets, and updates the proxies if that's the case.
868     */
869    protected void updateProxies() {
870        for (String id : transientCreated) { // ordered
871            DBSDocumentState docState = transientStates.get(id);
872            updateProxies(docState);
873        }
874        // copy as we may modify proxies
875        for (String id : transientStates.keySet().toArray(new String[0])) {
876            DBSDocumentState docState = transientStates.get(id);
877            if (transientCreated.contains(id)) {
878                continue; // already done
879            }
880            if (docState.isDirty()) {
881                updateProxies(docState);
882            }
883        }
884    }
885
886    protected void updateProxies(DBSDocumentState target) {
887        Object[] proxyIds = (Object[]) target.get(KEY_PROXY_IDS);
888        if (proxyIds != null) {
889            for (Object proxyId : proxyIds) {
890                try {
891                    updateProxy(target, (String) proxyId);
892                } catch (ConcurrentUpdateException e) {
893                    e.addInfo("On doc " + target.getId());
894                    log.error(e, e);
895                    // do not throw, this avoids crashing the session
896                }
897            }
898        }
899    }
900
901    /**
902     * Updates the state of a proxy based on its target.
903     */
904    protected void updateProxy(DBSDocumentState target, String proxyId) {
905        DBSDocumentState proxy = getStateForUpdate(proxyId);
906        if (proxy == null) {
907            throw new ConcurrentUpdateException("Proxy " + proxyId + " concurrently deleted");
908        }
909        SchemaManager schemaManager = Framework.getService(SchemaManager.class);
910        // clear all proxy data
911        for (String key : proxy.getState().keyArray()) {
912            if (!isProxySpecific(key, schemaManager)) {
913                proxy.put(key, null);
914            }
915        }
916        // copy from target
917        for (Entry<String, Serializable> en : target.getState().entrySet()) {
918            String key = en.getKey();
919            if (!isProxySpecific(key, schemaManager)) {
920                proxy.put(key, StateHelper.deepCopy(en.getValue()));
921            }
922        }
923    }
924
925    /**
926     * Things that we don't touch on a proxy when updating it.
927     */
928    protected boolean isProxySpecific(String key, SchemaManager schemaManager) {
929        switch (key) {
930        // these are placeful stuff
931        case KEY_ID:
932        case KEY_PARENT_ID:
933        case KEY_ANCESTOR_IDS:
934        case KEY_NAME:
935        case KEY_POS:
936        case KEY_ACP:
937        case KEY_READ_ACL:
938            // these are proxy-specific
939        case KEY_IS_PROXY:
940        case KEY_PROXY_TARGET_ID:
941        case KEY_PROXY_VERSION_SERIES_ID:
942        case KEY_IS_VERSION:
943        case KEY_PROXY_IDS:
944            return true;
945        }
946        int p = key.indexOf(':');
947        if (p == -1) {
948            // no prefix, assume not proxy-specific
949            return false;
950        }
951        String prefix = key.substring(0, p);
952        Schema schema = schemaManager.getSchemaFromPrefix(prefix);
953        if (schema == null) {
954            schema = schemaManager.getSchema(prefix);
955            if (schema == null) {
956                // unknown prefix, assume not proxy-specific
957                return false;
958            }
959        }
960        return schemaManager.isProxySchema(schema.getName(), null); // type unused
961    }
962
963    /**
964     * Called when created in a transaction.
965     *
966     * @since 7.4
967     */
968    public void begin() {
969        undoLog = new HashMap<>();
970        repository.begin();
971    }
972
973    /**
974     * Saves and flushes to database.
975     */
976    public void commit() {
977        save();
978        commitSave();
979        repository.commit();
980    }
981
982    /**
983     * Commits the saved state to the database.
984     */
985    protected void commitSave() {
986        // clear transient, this means that after this references to states will be stale
987        // TODO mark states as invalid
988        clearTransient();
989        // the transaction ended, the proxied DBSSession will disappear and cannot be reused anyway
990        undoLog = null;
991    }
992
993    /**
994     * Rolls back the save state by applying the undo log.
995     */
996    public void rollback() {
997        clearTransient();
998        applyUndoLog();
999        // the transaction ended, the proxied DBSSession will disappear and cannot be reused anyway
1000        undoLog = null;
1001        repository.rollback();
1002    }
1003
1004    protected void clearTransient() {
1005        transientStates.clear();
1006        transientCreated.clear();
1007    }
1008
1009    /**
1010     * Gets the fulltext updates to do. Called at save() time.
1011     *
1012     * @return a list of {@link Work} instances to schedule post-commit.
1013     */
1014    protected List<Work> getFulltextWorks() {
1015        Set<String> docsWithDirtyStrings = new HashSet<>();
1016        Set<String> docsWithDirtyBinaries = new HashSet<>();
1017        findDirtyDocuments(docsWithDirtyStrings, docsWithDirtyBinaries);
1018        if (docsWithDirtyStrings.isEmpty() && docsWithDirtyBinaries.isEmpty()) {
1019            return Collections.emptyList();
1020        }
1021        List<Work> works = new LinkedList<>();
1022        getFulltextSimpleWorks(works, docsWithDirtyStrings);
1023        getFulltextBinariesWorks(works, docsWithDirtyBinaries);
1024        return works;
1025    }
1026
1027    /**
1028     * Finds the documents having dirty text or dirty binaries that have to be reindexed as fulltext.
1029     *
1030     * @param docsWithDirtyStrings set of ids, updated by this method
1031     * @param docWithDirtyBinaries set of ids, updated by this method
1032     */
1033    protected void findDirtyDocuments(Set<String> docsWithDirtyStrings, Set<String> docWithDirtyBinaries) {
1034        for (DBSDocumentState docState : transientStates.values()) {
1035            State originalState = docState.getOriginalState();
1036            State state = docState.getState();
1037            if (originalState == state) {
1038                continue;
1039            }
1040            StateDiff diff = StateHelper.diff(originalState, state);
1041            if (diff.isEmpty()) {
1042                continue;
1043            }
1044            StateDiff rdiff = StateHelper.diff(state, originalState);
1045            // we do diffs in both directions to capture removal of complex list elements,
1046            // for instance for {foo: [{bar: baz}] -> {foo: []}
1047            // diff paths = foo and rdiff paths = foo/*/bar
1048            Set<String> paths = new HashSet<>();
1049            DirtyPathsFinder dirtyPathsFinder = new DirtyPathsFinder(paths);
1050            dirtyPathsFinder.findDirtyPaths(diff);
1051            dirtyPathsFinder.findDirtyPaths(rdiff);
1052            FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration();
1053            boolean dirtyStrings = false;
1054            boolean dirtyBinaries = false;
1055            for (String path : paths) {
1056                Set<String> indexesSimple = fulltextConfiguration.indexesByPropPathSimple.get(path);
1057                if (indexesSimple != null && !indexesSimple.isEmpty()) {
1058                    dirtyStrings = true;
1059                    if (dirtyBinaries) {
1060                        break;
1061                    }
1062                }
1063                Set<String> indexesBinary = fulltextConfiguration.indexesByPropPathBinary.get(path);
1064                if (indexesBinary != null && !indexesBinary.isEmpty()) {
1065                    dirtyBinaries = true;
1066                    if (dirtyStrings) {
1067                        break;
1068                    }
1069                }
1070            }
1071            if (dirtyStrings) {
1072                docsWithDirtyStrings.add(docState.getId());
1073            }
1074            if (dirtyBinaries) {
1075                docWithDirtyBinaries.add(docState.getId());
1076            }
1077        }
1078    }
1079
1080    /**
1081     * Iterates on a state diff to find the paths corresponding to dirty values.
1082     *
1083     * @since 7.10-HF04, 8.1
1084     */
1085    protected static class DirtyPathsFinder {
1086
1087        protected Set<String> paths;
1088
1089        public DirtyPathsFinder(Set<String> paths) {
1090            this.paths = paths;
1091        }
1092
1093        public void findDirtyPaths(StateDiff value) {
1094            findDirtyPaths(value, null);
1095        }
1096
1097        protected void findDirtyPaths(Object value, String path) {
1098            if (value instanceof Object[]) {
1099                findDirtyPaths((Object[]) value, path);
1100            } else if (value instanceof List) {
1101                findDirtyPaths((List<?>) value, path);
1102            } else if (value instanceof ListDiff) {
1103                findDirtyPaths((ListDiff) value, path);
1104            } else if (value instanceof State) {
1105                findDirtyPaths((State) value, path);
1106            } else {
1107                paths.add(path);
1108            }
1109        }
1110
1111        protected void findDirtyPaths(Object[] value, String path) {
1112            String newPath = path + "/*";
1113            for (Object v : value) {
1114                findDirtyPaths(v, newPath);
1115            }
1116        }
1117
1118        protected void findDirtyPaths(List<?> value, String path) {
1119            String newPath = path + "/*";
1120            for (Object v : value) {
1121                findDirtyPaths(v, newPath);
1122            }
1123        }
1124
1125        protected void findDirtyPaths(ListDiff value, String path) {
1126            String newPath = path + "/*";
1127            if (value.diff != null) {
1128                findDirtyPaths(value.diff, newPath);
1129            }
1130            if (value.rpush != null) {
1131                findDirtyPaths(value.rpush, newPath);
1132            }
1133        }
1134
1135        protected void findDirtyPaths(State value, String path) {
1136            for (Entry<String, Serializable> es : value.entrySet()) {
1137                String key = es.getKey();
1138                Serializable v = es.getValue();
1139                String newPath = path == null ? key : path + "/" + key;
1140                findDirtyPaths(v, newPath);
1141            }
1142        }
1143    }
1144
1145    protected void getFulltextSimpleWorks(List<Work> works, Set<String> docsWithDirtyStrings) {
1146        // TODO XXX make configurable, see also FulltextExtractorWork
1147        FulltextParser fulltextParser = new DefaultFulltextParser();
1148        FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration();
1149        if (fulltextConfiguration.fulltextSearchDisabled) {
1150            return;
1151        }
1152        // update simpletext on documents with dirty strings
1153        for (String id : docsWithDirtyStrings) {
1154            if (id == null) {
1155                // cannot happen, but has been observed :(
1156                log.error("Got null doc id in fulltext update, cannot happen");
1157                continue;
1158            }
1159            DBSDocumentState docState = getStateForUpdate(id);
1160            if (docState == null) {
1161                // cannot happen
1162                continue;
1163            }
1164            String documentType = docState.getPrimaryType();
1165            // Object[] mixinTypes = (Object[]) docState.get(KEY_MIXIN_TYPES);
1166
1167            if (!fulltextConfiguration.isFulltextIndexable(documentType)) {
1168                continue;
1169            }
1170            docState.put(KEY_FULLTEXT_JOBID, docState.getId());
1171            FulltextFinder fulltextFinder = new FulltextFinder(fulltextParser, docState, session);
1172            List<IndexAndText> indexesAndText = new LinkedList<>();
1173            for (String indexName : fulltextConfiguration.indexNames) {
1174                // TODO paths from config
1175                String text = fulltextFinder.findFulltext(indexName);
1176                indexesAndText.add(new IndexAndText(indexName, text));
1177            }
1178            if (!indexesAndText.isEmpty()) {
1179                Work work = new FulltextUpdaterWork(repository.getName(), id, true, false, indexesAndText);
1180                works.add(work);
1181            }
1182        }
1183    }
1184
1185    protected void getFulltextBinariesWorks(List<Work> works, Set<String> docWithDirtyBinaries) {
1186        if (docWithDirtyBinaries.isEmpty()) {
1187            return;
1188        }
1189
1190        FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration();
1191
1192        // mark indexing in progress, so that future copies (including versions)
1193        // will be indexed as well
1194        for (String id : docWithDirtyBinaries) {
1195            DBSDocumentState docState = getStateForUpdate(id);
1196            if (docState == null) {
1197                // cannot happen
1198                continue;
1199            }
1200            if (!fulltextConfiguration.isFulltextIndexable(docState.getPrimaryType())) {
1201                continue;
1202            }
1203            docState.put(KEY_FULLTEXT_JOBID, docState.getId());
1204        }
1205
1206        // FulltextExtractorWork does fulltext extraction using converters
1207        // and then schedules a FulltextUpdaterWork to write the results
1208        // single-threaded
1209        for (String id : docWithDirtyBinaries) {
1210            // don't exclude proxies
1211            Work work = new DBSFulltextExtractorWork(repository.getName(), id);
1212            works.add(work);
1213        }
1214    }
1215
1216    protected static class FulltextFinder {
1217
1218        protected final FulltextParser fulltextParser;
1219
1220        protected final DBSDocumentState document;
1221
1222        protected final DBSSession session;
1223
1224        protected final String documentType;
1225
1226        protected final Object[] mixinTypes;
1227
1228        /**
1229         * Prepares parsing for one document.
1230         */
1231        public FulltextFinder(FulltextParser fulltextParser, DBSDocumentState document, DBSSession session) {
1232            this.fulltextParser = fulltextParser;
1233            this.document = document;
1234            this.session = session;
1235            if (document == null) {
1236                documentType = null;
1237                mixinTypes = null;
1238            } else { // null in tests
1239                documentType = document.getPrimaryType();
1240                mixinTypes = (Object[]) document.get(KEY_MIXIN_TYPES);
1241            }
1242        }
1243
1244        /**
1245         * Parses the document for one index.
1246         */
1247        public String findFulltext(String indexName) {
1248            // TODO indexName
1249            // TODO paths
1250            List<String> strings = new ArrayList<>();
1251            findFulltext(indexName, document.getState(), strings);
1252            return StringUtils.join(strings, ' ');
1253        }
1254
1255        protected void findFulltext(String indexName, State state, List<String> strings) {
1256            for (Entry<String, Serializable> en : state.entrySet()) {
1257                String key = en.getKey();
1258                if (key.startsWith(KEY_PREFIX)) {
1259                    switch (key) {
1260                    // allow indexing of this:
1261                    case DBSDocument.KEY_NAME:
1262                        break;
1263                    default:
1264                        continue;
1265                    }
1266                }
1267                Serializable value = en.getValue();
1268                if (value instanceof State) {
1269                    State s = (State) value;
1270                    findFulltext(indexName, s, strings);
1271                } else if (value instanceof List) {
1272                    @SuppressWarnings("unchecked")
1273                    List<State> v = (List<State>) value;
1274                    for (State s : v) {
1275                        findFulltext(indexName, s, strings);
1276                    }
1277                } else if (value instanceof Object[]) {
1278                    Object[] ar = (Object[]) value;
1279                    for (Object v : ar) {
1280                        if (v instanceof String) {
1281                            fulltextParser.parse((String) v, null, strings);
1282                        } else {
1283                            // arrays are homogeneous, no need to continue
1284                            break;
1285                        }
1286                    }
1287                } else {
1288                    if (value instanceof String) {
1289                        fulltextParser.parse((String) value, null, strings);
1290                    }
1291                }
1292            }
1293        }
1294    }
1295
1296    protected void scheduleWork(List<Work> works) {
1297        // do async fulltext indexing only if high-level sessions are available
1298        RepositoryManager repositoryManager = Framework.getService(RepositoryManager.class);
1299        if (repositoryManager != null && !works.isEmpty()) {
1300            WorkManager workManager = Framework.getService(WorkManager.class);
1301            for (Work work : works) {
1302                // schedule work post-commit
1303                // in non-tx mode, this may execute it nearly immediately
1304                workManager.schedule(work, Scheduling.IF_NOT_SCHEDULED, true);
1305            }
1306        }
1307    }
1308
1309}