001/*
002 * (C) Copyright 2006-2019 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.sql;
020
021import java.io.Serializable;
022import java.util.ArrayList;
023import java.util.Arrays;
024import java.util.Collection;
025import java.util.Collections;
026import java.util.Deque;
027import java.util.GregorianCalendar;
028import java.util.HashMap;
029import java.util.HashSet;
030import java.util.LinkedHashSet;
031import java.util.LinkedList;
032import java.util.List;
033import java.util.Map;
034import java.util.Map.Entry;
035import java.util.Set;
036import java.util.function.Consumer;
037import java.util.stream.Collectors;
038
039import org.apache.commons.collections.map.AbstractReferenceMap;
040import org.apache.commons.collections.map.ReferenceMap;
041import org.apache.commons.logging.Log;
042import org.apache.commons.logging.LogFactory;
043import org.nuxeo.ecm.core.api.ConcurrentUpdateException;
044import org.nuxeo.ecm.core.api.DocumentExistsException;
045import org.nuxeo.ecm.core.api.NuxeoException;
046import org.nuxeo.ecm.core.api.model.DeltaLong;
047import org.nuxeo.ecm.core.model.BaseSession;
048import org.nuxeo.ecm.core.model.BaseSession.VersionAclMode;
049import org.nuxeo.ecm.core.schema.FacetNames;
050import org.nuxeo.ecm.core.storage.BaseDocument;
051import org.nuxeo.ecm.core.storage.sql.Fragment.State;
052import org.nuxeo.ecm.core.storage.sql.RowMapper.CopyResult;
053import org.nuxeo.ecm.core.storage.sql.RowMapper.IdWithTypes;
054import org.nuxeo.ecm.core.storage.sql.RowMapper.NodeInfo;
055import org.nuxeo.ecm.core.storage.sql.RowMapper.RowBatch;
056import org.nuxeo.ecm.core.storage.sql.RowMapper.RowUpdate;
057import org.nuxeo.ecm.core.storage.sql.SimpleFragment.FieldComparator;
058import org.nuxeo.runtime.api.Framework;
059import org.nuxeo.runtime.metrics.MetricsService;
060
061import io.dropwizard.metrics5.Counter;
062import io.dropwizard.metrics5.MetricName;
063import io.dropwizard.metrics5.MetricRegistry;
064import io.dropwizard.metrics5.SharedMetricRegistries;
065
066/**
067 * This class holds persistence context information.
068 * <p>
069 * All non-saved modified data is referenced here. At save time, the data is sent to the database by the {@link Mapper}.
070 * The database will at some time later be committed by the external transaction manager in effect.
071 * <p>
072 * Internally a fragment can be in at most one of the "pristine" or "modified" map. After a save() all the fragments are
073 * pristine, and may be partially invalidated after commit by other local or clustered contexts that committed too.
074 * <p>
075 * Depending on the table, the context may hold {@link SimpleFragment}s, which represent one row,
076 * {@link CollectionFragment}s, which represent several rows.
077 * <p>
078 * This class is not thread-safe, it should be tied to a single session and the session itself should not be used
079 * concurrently.
080 */
081public class PersistenceContext {
082
083    protected static final Log log = LogFactory.getLog(PersistenceContext.class);
084
085    /**
086     * Property for threshold at which we warn that a Selection may be too big, with stack trace.
087     *
088     * @since 7.1
089     */
090    public static final String SEL_WARN_THRESHOLD_PROP = "org.nuxeo.vcs.selection.warn.threshold";
091
092    public static final String SEL_WARN_THRESHOLD_DEFAULT = "15000";
093
094    protected static final FieldComparator POS_COMPARATOR = new FieldComparator(Model.HIER_CHILD_POS_KEY);
095
096    protected static final FieldComparator VER_CREATED_COMPARATOR = new FieldComparator(Model.VERSION_CREATED_KEY);
097
098    protected final Model model;
099
100    // protected because accessed by Fragment.refetch()
101    protected final RowMapper mapper;
102
103    protected final SessionImpl session;
104
105    protected final VersionAclMode versionAclMode;
106
107    protected final boolean disableReadVersionPermission;
108
109    // selection context for complex properties
110    protected final SelectionContext hierComplex;
111
112    // selection context for non-complex properties
113    // public because used by unit tests
114    public final SelectionContext hierNonComplex;
115
116    // selection context for versions by series
117    private final SelectionContext seriesVersions;
118
119    // selection context for proxies by series
120    private final SelectionContext seriesProxies;
121
122    // selection context for proxies by target
123    private final SelectionContext targetProxies;
124
125    private final List<SelectionContext> selections;
126
127    /**
128     * The pristine fragments. All held data is identical to what is present in the database and could be refetched if
129     * needed.
130     * <p>
131     * This contains fragment that are {@link State#PRISTINE} or {@link State#ABSENT}, or in some cases
132     * {@link State#INVALIDATED_MODIFIED} or {@link State#INVALIDATED_DELETED}.
133     * <p>
134     * Pristine fragments must be kept here when referenced by the application, because the application must get the
135     * same fragment object if asking for it twice, even in two successive transactions.
136     * <p>
137     * This is memory-sensitive, a fragment can always be refetched if nobody uses it and the GC collects it. Use a weak
138     * reference for the values, we don't hold them longer than they need to be referenced, as the underlying mapper
139     * also has its own cache.
140     */
141    protected final Map<RowId, Fragment> pristine;
142
143    /**
144     * The fragments changed by the session.
145     * <p>
146     * This contains fragment that are {@link State#CREATED}, {@link State#MODIFIED} or {@link State#DELETED}.
147     */
148    protected final Map<RowId, Fragment> modified;
149
150    /**
151     * Fragment ids generated but not yet saved. We know that any fragment with one of these ids cannot exist in the
152     * database.
153     */
154    private final Set<Serializable> createdIds;
155
156    /**
157     * Document ids modified as "user changes", which means that a change token should be checked.
158     *
159     * @since 9.2
160     */
161    protected final Set<Serializable> userChangeIds = new HashSet<>();
162
163    /**
164     * Cache statistics
165     *
166     * @since 5.7
167     */
168    protected final MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricsService.class.getName());
169
170    protected final Counter cacheCount;
171
172    protected final Counter cacheHitCount;
173
174    /**
175     * Threshold at which we warn that a Selection may be too big, with stack trace.
176     */
177    protected long bigSelWarnThreshold;
178
179    @SuppressWarnings("unchecked")
180    public PersistenceContext(Model model, RowMapper mapper, SessionImpl session) {
181        this.model = model;
182        this.mapper = mapper;
183        this.session = session;
184        versionAclMode = VersionAclMode.getConfiguration();
185        disableReadVersionPermission = BaseSession.isReadVersionPermissionDisabled();
186        hierComplex = new SelectionContext(SelectionType.CHILDREN, Boolean.TRUE, mapper, this);
187        hierNonComplex = new SelectionContext(SelectionType.CHILDREN, Boolean.FALSE, mapper, this);
188        seriesVersions = new SelectionContext(SelectionType.SERIES_VERSIONS, null, mapper, this);
189        selections = new ArrayList<>(Arrays.asList(hierComplex, hierNonComplex, seriesVersions));
190        if (model.proxiesEnabled) {
191            seriesProxies = new SelectionContext(SelectionType.SERIES_PROXIES, null, mapper, this);
192            targetProxies = new SelectionContext(SelectionType.TARGET_PROXIES, null, mapper, this);
193            selections.add(seriesProxies);
194            selections.add(targetProxies);
195        } else {
196            seriesProxies = null;
197            targetProxies = null;
198        }
199
200        // use a weak reference for the values, we don't hold them longer than
201        // they need to be referenced, as the underlying mapper also has its own
202        // cache
203        pristine = new ReferenceMap(AbstractReferenceMap.HARD, AbstractReferenceMap.WEAK);
204        modified = new HashMap<>();
205        // this has to be linked to keep creation order, as foreign keys
206        // are used and need this
207        createdIds = new LinkedHashSet<>();
208        cacheCount = registry.counter(MetricName.build("nuxeo", "repositories", "repository", "cache", "read")
209                                                .tagged("repository", session.getRepositoryName()));
210        cacheHitCount = registry.counter(MetricName.build("nuxeo", "repositories", "repository", "cache", "hit")
211                                                   .tagged("repository", session.getRepositoryName()));
212        try {
213            bigSelWarnThreshold = Long.parseLong(
214                    Framework.getProperty(SEL_WARN_THRESHOLD_PROP, SEL_WARN_THRESHOLD_DEFAULT));
215        } catch (NumberFormatException e) {
216            log.error("Invalid value for " + SEL_WARN_THRESHOLD_PROP + ": "
217                    + Framework.getProperty(SEL_WARN_THRESHOLD_PROP));
218        }
219    }
220
221    protected int clearCaches() {
222        mapper.clearCache();
223        // TODO there should be a synchronization here
224        // but this is a rare operation and we don't call
225        // it if a transaction is in progress
226        int n = clearLocalCaches();
227        modified.clear(); // not empty when rolling back before save
228        createdIds.clear();
229        userChangeIds.clear();
230        return n;
231    }
232
233    protected int clearLocalCaches() {
234        for (SelectionContext sel : selections) {
235            sel.clearCaches();
236        }
237        int n = pristine.size();
238        pristine.clear();
239        return n;
240    }
241
242    protected long getCacheSize() {
243        return getCachePristineSize() + getCacheSelectionSize() + getCacheMapperSize();
244    }
245
246    protected long getCacheMapperSize() {
247        return mapper.getCacheSize();
248    }
249
250    protected long getCachePristineSize() {
251        return pristine.size();
252    }
253
254    protected long getCacheSelectionSize() {
255        int size = 0;
256        for (SelectionContext sel : selections) {
257            size += sel.getSize();
258        }
259        return size;
260    }
261
262    /**
263     * Generates a new id, or used a pre-generated one (import).
264     */
265    protected Serializable generateNewId(Serializable id) {
266        if (id == null) {
267            id = mapper.generateNewId();
268        }
269        createdIds.add(id);
270        return id;
271    }
272
273    protected boolean isIdNew(Serializable id) {
274        return createdIds.contains(id);
275    }
276
277    /**
278     * Marks this document id as belonging to a user change.
279     *
280     * @since 9.2
281     */
282    protected void markUserChange(Serializable id) {
283        if (session.changeTokenEnabled) {
284            userChangeIds.add(id);
285        }
286    }
287
288    /**
289     * Saves all the created, modified and deleted rows into a batch object, for later execution.
290     * <p>
291     * Also updates the passed fragmentsToClearDirty list with dirty modified fragments, for later call of clearDirty
292     * (it's important to call it later and not now because for delta values we need the delta during batch write, and
293     * they are cleared by clearDirty).
294     */
295    protected RowBatch getSaveBatch(List<Fragment> fragmentsToClearDirty) {
296        RowBatch batch = new RowBatch();
297
298        // update change tokens
299        Map<Serializable, Map<String, Serializable>> rowUpdateConditions = new HashMap<>();
300        if (session.changeTokenEnabled) {
301            // find which docs are created
302            for (Serializable id : createdIds) {
303                SimpleFragment hier = getHier(id, false);
304                if (hier == null) {
305                    // was created and deleted before save
306                    continue;
307                }
308                hier.put(Model.MAIN_CHANGE_TOKEN_KEY, Model.INITIAL_CHANGE_TOKEN);
309            }
310            // find which docs are modified
311            Set<Serializable> modifiedDocIds = findModifiedDocuments();
312            for (Serializable id : modifiedDocIds) {
313                SimpleFragment hier = getHier(id, false);
314                // increment system change token
315                Long base = (Long) hier.get(Model.MAIN_SYS_CHANGE_TOKEN_KEY);
316                hier.put(Model.MAIN_SYS_CHANGE_TOKEN_KEY, DeltaLong.valueOf(base, 1));
317                // update change token if applicable (user change)
318                if (userChangeIds.contains(id)) {
319                    Map<String, Serializable> conditions = updateChangeToken(hier);
320                    rowUpdateConditions.put(id, conditions);
321                }
322            }
323            userChangeIds.clear();
324        }
325
326        // created main rows are saved first in the batch (in their order of
327        // creation), because they are used as foreign keys in all other tables
328        for (Serializable id : createdIds) {
329            RowId rowId = new RowId(Model.HIER_TABLE_NAME, id);
330            Fragment fragment = modified.remove(rowId);
331            if (fragment == null) {
332                // was created and deleted before save
333                continue;
334            }
335            batch.creates.add(fragment.row);
336            fragment.clearDirty();
337            fragment.setPristine();
338            pristine.put(rowId, fragment);
339        }
340        createdIds.clear();
341
342        // save the rest
343        for (Entry<RowId, Fragment> en : modified.entrySet()) {
344            RowId rowId = en.getKey();
345            Fragment fragment = en.getValue();
346            switch (fragment.getState()) {
347            case CREATED:
348                batch.creates.add(fragment.row);
349                fragment.clearDirty();
350                fragment.setPristine();
351                // modified map cleared at end of loop
352                pristine.put(rowId, fragment);
353                break;
354            case MODIFIED:
355                RowUpdate rowu = fragment.getRowUpdate();
356                if (rowu != null) {
357                    if (Model.HIER_TABLE_NAME.equals(fragment.row.tableName)) {
358                        Map<String, Serializable> conditions = rowUpdateConditions.get(fragment.getId());
359                        if (conditions != null) {
360                            rowu.setConditions(conditions);
361                        }
362                    }
363                    batch.updates.add(rowu);
364                    fragmentsToClearDirty.add(fragment);
365                }
366                fragment.setPristine();
367                // modified map cleared at end of loop
368                pristine.put(rowId, fragment);
369                break;
370            case DELETED:
371                // TODO deleting non-hierarchy fragments is done by the database
372                // itself as their foreign key to hierarchy is ON DELETE CASCADE
373                batch.deletes.add(new RowId(rowId));
374                fragment.setDetached();
375                // modified map cleared at end of loop
376                break;
377            case DELETED_DEPENDENT:
378                batch.deletesDependent.add(new RowId(rowId));
379                fragment.setDetached();
380                break;
381            case PRISTINE:
382                // cannot happen, but has been observed :(
383                log.error("Found PRISTINE fragment in modified map: " + fragment);
384                break;
385            default:
386                throw new RuntimeException(fragment.toString());
387            }
388        }
389        modified.clear();
390
391        // flush selections caches
392        for (SelectionContext sel : selections) {
393            sel.postSave();
394        }
395
396        return batch;
397    }
398
399    /** Updates a change token in the main fragment, and returns the condition to check. */
400    protected Map<String, Serializable> updateChangeToken(SimpleFragment hier) {
401        Long oldToken = (Long) hier.get(Model.MAIN_CHANGE_TOKEN_KEY);
402        Long newToken;
403        if (oldToken == null) {
404            // document without change token, just created
405            newToken = Model.INITIAL_CHANGE_TOKEN;
406        } else {
407            newToken = BaseDocument.updateChangeToken(oldToken);
408        }
409        hier.put(Model.MAIN_CHANGE_TOKEN_KEY, newToken);
410        return Collections.singletonMap(Model.MAIN_CHANGE_TOKEN_KEY, oldToken);
411    }
412
413    private boolean complexProp(SimpleFragment fragment) {
414        return complexProp((Boolean) fragment.get(Model.HIER_CHILD_ISPROPERTY_KEY));
415    }
416
417    private boolean complexProp(Boolean isProperty) {
418        return Boolean.TRUE.equals(isProperty);
419    }
420
421    private SelectionContext getHierSelectionContext(boolean complexProp) {
422        return complexProp ? hierComplex : hierNonComplex;
423    }
424
425    /**
426     * Finds the documents having been modified.
427     * <p>
428     * A document is modified if any of its direct fragments (same id) is modified, or if any of its complex property
429     * fragments having it as an ancestor is created, modified or deleted.
430     * <p>
431     * Created and deleted documents aren't considered modified.
432     *
433     * @return the set of modified documents
434     * @since 9.1
435     */
436    protected Set<Serializable> findModifiedDocuments() {
437        Set<Serializable> modifiedDocIds = new HashSet<>();
438        Set<Serializable> deletedDocIds = new HashSet<>();
439        for (Fragment fragment : modified.values()) {
440            Serializable docId = getContainingDocument(fragment.getId());
441            boolean complexProp = !fragment.getId().equals(docId);
442            switch (fragment.getState()) {
443            case MODIFIED:
444                modifiedDocIds.add(docId);
445                break;
446            case CREATED:
447                modifiedDocIds.add(docId);
448                break;
449            case DELETED:
450            case DELETED_DEPENDENT:
451                if (complexProp) {
452                    modifiedDocIds.add(docId);
453                } else if (Model.HIER_TABLE_NAME.equals(fragment.row.tableName)) {
454                    deletedDocIds.add(docId);
455                }
456                break;
457            default:
458            }
459        }
460        modifiedDocIds.removeAll(deletedDocIds);
461        modifiedDocIds.removeAll(createdIds);
462        return modifiedDocIds;
463    }
464
465    /**
466     * Finds the documents having dirty text or dirty binaries that have to be reindexed as fulltext.
467     *
468     * @param dirtyStrings set of ids, updated by this method
469     * @param dirtyBinaries set of ids, updated by this method
470     */
471    protected void findDirtyDocuments(Set<Serializable> dirtyStrings, Set<Serializable> dirtyBinaries) {
472        // deleted documents, for which we don't need to reindex anything
473        Set<Serializable> deleted = new HashSet<>();
474        for (Fragment fragment : modified.values()) {
475            Serializable docId = getContainingDocument(fragment.getId());
476            String tableName = fragment.row.tableName;
477            State state = fragment.getState();
478            switch (state) {
479            case DELETED:
480            case DELETED_DEPENDENT:
481                if (Model.HIER_TABLE_NAME.equals(tableName) && fragment.getId().equals(docId)) {
482                    // deleting the document, record this
483                    deleted.add(docId);
484                }
485                if (isDeleted(docId)) {
486                    break;
487                }
488                // this is a deleted fragment of a complex property
489                // from a document that has not been completely deleted
490                //$FALL-THROUGH$
491            case CREATED:
492                PropertyType t = model.getFulltextInfoForFragment(tableName);
493                if (t == null) {
494                    break;
495                }
496                if (t == PropertyType.STRING || t == PropertyType.BOOLEAN) {
497                    dirtyStrings.add(docId);
498                }
499                if (t == PropertyType.BINARY || t == PropertyType.BOOLEAN) {
500                    dirtyBinaries.add(docId);
501                }
502                break;
503            case MODIFIED:
504                Collection<String> keys;
505                if (model.isCollectionFragment(tableName)) {
506                    keys = Collections.singleton(null);
507                } else {
508                    keys = ((SimpleFragment) fragment).getDirtyKeys();
509                }
510                for (String key : keys) {
511                    PropertyType type = model.getFulltextFieldType(tableName, key);
512                    if (type == PropertyType.STRING || type == PropertyType.ARRAY_STRING) {
513                        dirtyStrings.add(docId);
514                    } else if (type == PropertyType.BINARY || type == PropertyType.ARRAY_BINARY) {
515                        dirtyBinaries.add(docId);
516                    }
517                }
518                break;
519            default:
520            }
521        }
522        dirtyStrings.removeAll(deleted);
523        dirtyBinaries.removeAll(deleted);
524    }
525
526    /**
527     * Marks locally all the invalidations gathered by a {@link Mapper} operation (like a version restore).
528     */
529    protected void markInvalidated(VCSInvalidations invalidations) {
530        if (invalidations.modified != null) {
531            for (RowId rowId : invalidations.modified) {
532                Fragment fragment = getIfPresent(rowId);
533                if (fragment != null) {
534                    setFragmentPristine(fragment);
535                    fragment.setInvalidatedModified();
536                }
537            }
538            for (SelectionContext sel : selections) {
539                sel.markInvalidated(invalidations.modified);
540            }
541        }
542        if (invalidations.deleted != null) {
543            for (RowId rowId : invalidations.deleted) {
544                Fragment fragment = getIfPresent(rowId);
545                if (fragment != null) {
546                    setFragmentPristine(fragment);
547                    fragment.setInvalidatedDeleted();
548                }
549            }
550        }
551        // TODO XXX transactionInvalidations.add(invalidations);
552    }
553
554    // called from Fragment
555    protected void setFragmentModified(Fragment fragment) {
556        RowId rowId = fragment.row;
557        pristine.remove(rowId);
558        modified.put(rowId, fragment);
559    }
560
561    // also called from Fragment
562    protected void setFragmentPristine(Fragment fragment) {
563        RowId rowId = fragment.row;
564        modified.remove(rowId);
565        pristine.put(rowId, fragment);
566    }
567
568    /**
569     * Post-transaction invalidations notification.
570     * <p>
571     * Called post-transaction by session commit/rollback or transactionless save.
572     */
573    public void sendInvalidationsToOthers() {
574        VCSInvalidations invalidations = new VCSInvalidations();
575        for (SelectionContext sel : selections) {
576            sel.gatherInvalidations(invalidations);
577        }
578        mapper.sendInvalidations(invalidations);
579    }
580
581    /**
582     * Applies all invalidations accumulated.
583     * <p>
584     * Called pre-transaction by start or transactionless save;
585     */
586    public void processReceivedInvalidations() {
587        VCSInvalidations invals = mapper.receiveInvalidations();
588        if (invals == null) {
589            return;
590        }
591
592        processCacheInvalidations(invals);
593    }
594
595    private void processCacheInvalidations(VCSInvalidations invalidations) {
596        if (invalidations == null) {
597            return;
598        }
599        if (invalidations.all) {
600            clearLocalCaches();
601        }
602        if (invalidations.modified != null) {
603            for (RowId rowId : invalidations.modified) {
604                Fragment fragment = pristine.remove(rowId);
605                if (fragment != null) {
606                    fragment.setInvalidatedModified();
607                }
608            }
609            for (SelectionContext sel : selections) {
610                sel.processReceivedInvalidations(invalidations.modified);
611            }
612        }
613        if (invalidations.deleted != null) {
614            for (RowId rowId : invalidations.deleted) {
615                Fragment fragment = pristine.remove(rowId);
616                if (fragment != null) {
617                    fragment.setInvalidatedDeleted();
618                }
619            }
620        }
621    }
622
623    public void checkInvalidationsConflict() {
624        // synchronized (receivedInvalidations) {
625        // if (receivedInvalidations.modified != null) {
626        // for (RowId rowId : receivedInvalidations.modified) {
627        // if (transactionInvalidations.contains(rowId)) {
628        // throw new ConcurrentModificationException(
629        // "Updating a concurrently modified value: "
630        // + new RowId(rowId));
631        // }
632        // }
633        // }
634        //
635        // if (receivedInvalidations.deleted != null) {
636        // for (RowId rowId : receivedInvalidations.deleted) {
637        // if (transactionInvalidations.contains(rowId)) {
638        // throw new ConcurrentModificationException(
639        // "Updating a concurrently deleted value: "
640        // + new RowId(rowId));
641        // }
642        // }
643        // }
644        // }
645    }
646
647    /**
648     * Gets a fragment, if present in the context.
649     * <p>
650     * Called by {@link #get}, and by the {@link Mapper} to reuse known selection fragments.
651     *
652     * @param rowId the fragment id
653     * @return the fragment, or {@code null} if not found
654     */
655    protected Fragment getIfPresent(RowId rowId) {
656        cacheCount.inc();
657        Fragment fragment = pristine.get(rowId);
658        if (fragment == null) {
659            fragment = modified.get(rowId);
660        }
661        if (fragment != null) {
662            cacheHitCount.inc();
663        }
664        return fragment;
665    }
666
667    /**
668     * Gets a fragment.
669     * <p>
670     * If it's not in the context, fetch it from the mapper. If it's not in the database, returns {@code null} or an
671     * absent fragment.
672     * <p>
673     * Deleted fragments may be returned.
674     *
675     * @param rowId the fragment id
676     * @param allowAbsent {@code true} to return an absent fragment as an object instead of {@code null}
677     * @return the fragment, or {@code null} if none is found and {@code allowAbsent} was {@code false}
678     */
679    protected Fragment get(RowId rowId, boolean allowAbsent) {
680        Fragment fragment = getIfPresent(rowId);
681        if (fragment == null) {
682            fragment = getFromMapper(rowId, allowAbsent, false);
683        }
684        return fragment;
685    }
686
687    /**
688     * Gets a fragment from the context or the mapper cache or the underlying database.
689     *
690     * @param rowId the fragment id
691     * @param allowAbsent {@code true} to return an absent fragment as an object instead of {@code null}
692     * @param cacheOnly only check memory, not the database
693     * @return the fragment, or when {@code allowAbsent} is {@code false}, a {@code null} if not found
694     */
695    protected Fragment getFromMapper(RowId rowId, boolean allowAbsent, boolean cacheOnly) {
696        List<Fragment> fragments = getFromMapper(Collections.singleton(rowId), allowAbsent, cacheOnly);
697        return fragments.isEmpty() ? null : fragments.get(0);
698    }
699
700    /**
701     * Gets a collection of fragments from the mapper. No order is kept between the inputs and outputs.
702     * <p>
703     * Fragments not found are not returned if {@code allowAbsent} is {@code false}.
704     */
705    protected List<Fragment> getFromMapper(Collection<RowId> rowIds, boolean allowAbsent, boolean cacheOnly) {
706        List<Fragment> res = new ArrayList<>(rowIds.size());
707
708        // find fragments we really want to fetch
709        List<RowId> todo = new ArrayList<>(rowIds.size());
710        for (RowId rowId : rowIds) {
711            if (isIdNew(rowId.id)) {
712                // the id has not been saved, so nothing exists yet in the
713                // database
714                // rowId is not a row -> will use an absent fragment
715                Fragment fragment = getFragmentFromFetchedRow(rowId, allowAbsent);
716                if (fragment != null) {
717                    res.add(fragment);
718                }
719            } else {
720                todo.add(rowId);
721            }
722        }
723        if (todo.isEmpty()) {
724            return res;
725        }
726
727        // fetch these fragments in bulk
728        List<? extends RowId> rows = mapper.read(todo, cacheOnly);
729        res.addAll(getFragmentsFromFetchedRows(rows, allowAbsent));
730
731        return res;
732    }
733
734    /**
735     * Gets a list of fragments.
736     * <p>
737     * If a fragment is not in the context, fetch it from the mapper. If it's not in the database, use an absent
738     * fragment or skip it.
739     * <p>
740     * Deleted fragments are skipped.
741     *
742     * @param rowIds the row ids
743     * @param allowAbsent {@code true} to return an absent fragment as an object instead of skipping it
744     * @return the fragments, in arbitrary order (no {@code null}s)
745     */
746    public List<Fragment> getMulti(Collection<RowId> rowIds, boolean allowAbsent) {
747        if (rowIds.isEmpty()) {
748            return Collections.emptyList();
749        }
750
751        // find those already in the context
752        List<Fragment> res = new ArrayList<>(rowIds.size());
753        List<RowId> todo = new LinkedList<>();
754        for (RowId rowId : rowIds) {
755            Fragment fragment = getIfPresent(rowId);
756            if (fragment == null) {
757                todo.add(rowId);
758            } else {
759                State state = fragment.getState();
760                if (state != State.DELETED && state != State.DELETED_DEPENDENT
761                        && (state != State.ABSENT || allowAbsent)) {
762                    res.add(fragment);
763                }
764            }
765        }
766        if (todo.isEmpty()) {
767            return res;
768        }
769
770        // fetch missing ones, return union
771        List<Fragment> fetched = getFromMapper(todo, allowAbsent, false);
772        res.addAll(fetched);
773        return res;
774    }
775
776    /**
777     * Turns the given rows (just fetched from the mapper) into fragments and record them in the context.
778     * <p>
779     * For each row, if the context already contains a fragment with the given id, it is returned instead of building a
780     * new one.
781     * <p>
782     * Deleted fragments are skipped.
783     * <p>
784     * If a simple {@link RowId} is passed, it means that an absent row was found by the mapper. An absent fragment will
785     * be returned, unless {@code allowAbsent} is {@code false} in which case it will be skipped.
786     *
787     * @param rowIds the list of rows or row ids
788     * @param allowAbsent {@code true} to return an absent fragment as an object instead of {@code null}
789     * @return the list of fragments
790     */
791    protected List<Fragment> getFragmentsFromFetchedRows(List<? extends RowId> rowIds, boolean allowAbsent) {
792        List<Fragment> fragments = new ArrayList<>(rowIds.size());
793        for (RowId rowId : rowIds) {
794            Fragment fragment = getFragmentFromFetchedRow(rowId, allowAbsent);
795            if (fragment != null) {
796                fragments.add(fragment);
797            }
798        }
799        return fragments;
800    }
801
802    /**
803     * Turns the given row (just fetched from the mapper) into a fragment and record it in the context.
804     * <p>
805     * If the context already contains a fragment with the given id, it is returned instead of building a new one.
806     * <p>
807     * If the fragment was deleted, {@code null} is returned.
808     * <p>
809     * If a simple {@link RowId} is passed, it means that an absent row was found by the mapper. An absent fragment will
810     * be returned, unless {@code allowAbsent} is {@code false} in which case {@code null} will be returned.
811     *
812     * @param rowId the row or row id (may be {@code null})
813     * @param allowAbsent {@code true} to return an absent fragment as an object instead of {@code null}
814     * @return the fragment, or {@code null} if it was deleted
815     */
816    protected Fragment getFragmentFromFetchedRow(RowId rowId, boolean allowAbsent) {
817        if (rowId == null) {
818            return null;
819        }
820        Fragment fragment = getIfPresent(rowId);
821        if (fragment != null) {
822            // row is already known in the context, use it
823            State state = fragment.getState();
824            if (state == State.DELETED || state == State.DELETED_DEPENDENT) {
825                // row has been deleted in the context, ignore it
826                return null;
827            } else if (state == State.INVALIDATED_MODIFIED || state == State.INVALIDATED_DELETED) {
828                // XXX TODO
829                throw new IllegalStateException(state.toString());
830            } else {
831                // keep existing fragment
832                return fragment;
833            }
834        }
835        boolean isCollection = model.isCollectionFragment(rowId.tableName);
836        if (rowId instanceof Row) {
837            Row row = (Row) rowId;
838            if (isCollection) {
839                fragment = new CollectionFragment(row, State.PRISTINE, this);
840            } else {
841                fragment = new SimpleFragment(row, State.PRISTINE, this);
842                // add to applicable selections
843                for (SelectionContext sel : selections) {
844                    if (sel.applicable((SimpleFragment) fragment)) {
845                        sel.recordExisting((SimpleFragment) fragment, false);
846                    }
847                }
848            }
849            return fragment;
850        } else {
851            if (allowAbsent) {
852                if (isCollection) {
853                    Serializable[] empty = model.getCollectionFragmentType(rowId.tableName).getEmptyArray();
854                    Row row = new Row(rowId.tableName, rowId.id, empty);
855                    return new CollectionFragment(row, State.ABSENT, this);
856                } else {
857                    Row row = new Row(rowId.tableName, rowId.id);
858                    return new SimpleFragment(row, State.ABSENT, this);
859                }
860            } else {
861                return null;
862            }
863        }
864    }
865
866    public SimpleFragment createHierarchyFragment(Row row) {
867        SimpleFragment fragment = createSimpleFragment(row);
868        SelectionContext hierSel = getHierSelectionContext(complexProp(fragment));
869        hierSel.recordCreated(fragment);
870        // no children for this new node
871        Serializable id = fragment.getId();
872        hierComplex.newSelection(id);
873        hierNonComplex.newSelection(id);
874        // could add to seriesProxies and seriesVersions as well
875        return fragment;
876    }
877
878    private SimpleFragment createVersionFragment(Row row) {
879        SimpleFragment fragment = createSimpleFragment(row);
880        seriesVersions.recordCreated(fragment);
881        // no proxies for this new version
882        if (model.proxiesEnabled) {
883            targetProxies.newSelection(fragment.getId());
884        }
885        return fragment;
886    }
887
888    public void createdProxyFragment(SimpleFragment fragment) {
889        if (model.proxiesEnabled) {
890            seriesProxies.recordCreated(fragment);
891            targetProxies.recordCreated(fragment);
892        }
893    }
894
895    public void removedProxyTarget(SimpleFragment fragment) {
896        if (model.proxiesEnabled) {
897            targetProxies.recordRemoved(fragment);
898        }
899    }
900
901    public void addedProxyTarget(SimpleFragment fragment) {
902        if (model.proxiesEnabled) {
903            targetProxies.recordCreated(fragment);
904        }
905    }
906
907    private SimpleFragment createSimpleFragment(Row row) {
908        if (pristine.containsKey(row) || modified.containsKey(row)) {
909            throw new ConcurrentUpdateException("Duplicate id: " + row.id);
910        }
911        return new SimpleFragment(row, State.CREATED, this);
912    }
913
914    /**
915     * Removes a property node and its children.
916     * <p>
917     * There's less work to do than when we have to remove a generic document node (less selections, and we can assume
918     * the depth is small so recurse).
919     */
920    public void removePropertyNode(SimpleFragment hierFragment) {
921        // collect children
922        Deque<SimpleFragment> todo = new LinkedList<>();
923        List<SimpleFragment> children = new LinkedList<>();
924        todo.add(hierFragment);
925        while (!todo.isEmpty()) {
926            SimpleFragment fragment = todo.removeFirst();
927            todo.addAll(getChildren(fragment.getId(), null, true)); // complex
928            children.add(fragment);
929        }
930        Collections.reverse(children);
931        // iterate on children depth first
932        for (SimpleFragment fragment : children) {
933            // remove from context
934            boolean primary = fragment == hierFragment;
935            removeFragmentAndDependents(fragment, primary);
936            // remove from selections
937            // removed from its parent selection
938            hierComplex.recordRemoved(fragment);
939            // no children anymore
940            hierComplex.recordRemovedSelection(fragment.getId());
941        }
942    }
943
944    private void removeFragmentAndDependents(SimpleFragment hierFragment, boolean primary) {
945        Serializable id = hierFragment.getId();
946        for (String fragmentName : model.getTypeFragments(new IdWithTypes(hierFragment))) {
947            RowId rowId = new RowId(fragmentName, id);
948            Fragment fragment = get(rowId, true); // may read it
949            State state = fragment.getState();
950            if (state != State.DELETED && state != State.DELETED_DEPENDENT) {
951                removeFragment(fragment, primary && hierFragment == fragment);
952            }
953        }
954    }
955
956    /**
957     * Gets descendants infos from a given root node. This includes information about the root node itself.
958     *
959     * @since 9.2
960     */
961    public List<NodeInfo> getNodeAndDescendantsInfo(SimpleFragment hierFragment) {
962        // get root info
963        Serializable rootId = hierFragment.getId();
964        SimpleFragment versionFragment;
965        SimpleFragment proxyFragment;
966        if (Model.PROXY_TYPE.equals(hierFragment.getString(Model.MAIN_PRIMARY_TYPE_KEY))) {
967            versionFragment = null;
968            proxyFragment = (SimpleFragment) get(new RowId(Model.PROXY_TABLE_NAME, rootId), true);
969        } else if (Boolean.TRUE.equals(hierFragment.get(Model.MAIN_IS_VERSION_KEY))) {
970            versionFragment = (SimpleFragment) get(new RowId(Model.VERSION_TABLE_NAME, rootId), true);
971            proxyFragment = null;
972        } else {
973            versionFragment = null;
974            proxyFragment = null;
975        }
976        NodeInfo rootInfo = new NodeInfo(hierFragment, versionFragment, proxyFragment);
977        List<NodeInfo> infos = mapper.getDescendantsInfo(rootId);
978        infos.add(rootInfo);
979        return infos;
980    }
981
982    /**
983     * Removes a document node and its children.
984     * <p>
985     * Assumes a full flush was done.
986     */
987    public void removeNode(SimpleFragment hierFragment, List<NodeInfo> nodeInfos) {
988        Serializable rootId = hierFragment.getId();
989
990        // get root info before deletion. may be a version or proxy
991        SimpleFragment versionFragment;
992        if (Model.PROXY_TYPE.equals(hierFragment.getString(Model.MAIN_PRIMARY_TYPE_KEY))) {
993            versionFragment = null;
994        } else if (Boolean.TRUE.equals(hierFragment.get(Model.MAIN_IS_VERSION_KEY))) {
995            versionFragment = (SimpleFragment) get(new RowId(Model.VERSION_TABLE_NAME, rootId), true);
996        } else {
997            versionFragment = null;
998        }
999
1000        // remove with descendants, and generate cache invalidations
1001        mapper.remove(rootId, nodeInfos);
1002
1003        // remove from context and selections
1004        for (NodeInfo info : nodeInfos) {
1005            for (String fragmentName : model.getTypeFragments(new IdWithTypes(info))) {
1006                RowId rowId = new RowId(fragmentName, info.id);
1007                removedFragment(rowId); // remove from context
1008            }
1009            removeFromSelections(info);
1010        }
1011
1012        // recompute version series if needed
1013        // only done for root of deletion as versions are not fileable
1014        Serializable versionSeriesId = versionFragment == null ? null
1015                : versionFragment.get(Model.VERSION_VERSIONABLE_KEY);
1016        if (versionSeriesId != null) {
1017            recomputeVersionSeries(versionSeriesId);
1018        }
1019    }
1020
1021    /**
1022     * Remove node from children/proxies selections.
1023     */
1024    private void removeFromSelections(NodeInfo info) {
1025        Serializable id = info.id;
1026        if (Model.PROXY_TYPE.equals(info.primaryType)) {
1027            seriesProxies.recordRemoved(id, info.versionSeriesId);
1028            targetProxies.recordRemoved(id, info.targetId);
1029        }
1030        if (info.versionSeriesId != null && info.targetId == null) {
1031            // version
1032            seriesVersions.recordRemoved(id, info.versionSeriesId);
1033        }
1034
1035        hierComplex.recordRemoved(info.id, info.parentId);
1036        hierNonComplex.recordRemoved(info.id, info.parentId);
1037
1038        // remove complete selections
1039        if (complexProp(info.isProperty)) {
1040            // no more a parent
1041            hierComplex.recordRemovedSelection(id);
1042            // is never a parent of non-complex children
1043        } else {
1044            // no more a parent
1045            hierComplex.recordRemovedSelection(id);
1046            hierNonComplex.recordRemovedSelection(id);
1047            // no more a version series
1048            if (model.proxiesEnabled) {
1049                seriesProxies.recordRemovedSelection(id);
1050            }
1051            seriesVersions.recordRemovedSelection(id);
1052            // no more a target
1053            if (model.proxiesEnabled) {
1054                targetProxies.recordRemovedSelection(id);
1055            }
1056        }
1057    }
1058
1059    /**
1060     * Deletes a fragment from the context. May generate a database DELETE if primary is {@code true}, otherwise
1061     * consider that database removal will be a cascade-induced consequence of another DELETE.
1062     */
1063    public void removeFragment(Fragment fragment, boolean primary) {
1064        RowId rowId = fragment.row;
1065        switch (fragment.getState()) {
1066        case ABSENT:
1067        case INVALIDATED_DELETED:
1068            pristine.remove(rowId);
1069            break;
1070        case CREATED:
1071            modified.remove(rowId);
1072            break;
1073        case PRISTINE:
1074        case INVALIDATED_MODIFIED:
1075            pristine.remove(rowId);
1076            modified.put(rowId, fragment);
1077            break;
1078        case MODIFIED:
1079            // already in modified
1080            break;
1081        case DETACHED:
1082        case DELETED:
1083        case DELETED_DEPENDENT:
1084            break;
1085        }
1086        fragment.setDeleted(primary);
1087    }
1088
1089    /**
1090     * Cleans up after a fragment has been removed in the database.
1091     *
1092     * @param rowId the row id
1093     */
1094    private void removedFragment(RowId rowId) {
1095        Fragment fragment = getIfPresent(rowId);
1096        if (fragment == null) {
1097            return;
1098        }
1099        switch (fragment.getState()) {
1100        case ABSENT:
1101        case PRISTINE:
1102        case INVALIDATED_MODIFIED:
1103        case INVALIDATED_DELETED:
1104            pristine.remove(rowId);
1105            break;
1106        case CREATED:
1107        case MODIFIED:
1108        case DELETED:
1109        case DELETED_DEPENDENT:
1110            // should not happen
1111            log.error("Removed fragment is in invalid state: " + fragment);
1112            modified.remove(rowId);
1113            break;
1114        case DETACHED:
1115            break;
1116        }
1117        fragment.setDetached();
1118    }
1119
1120    /**
1121     * Recomputes isLatest / isLatestMajor on all versions.
1122     */
1123    public void recomputeVersionSeries(Serializable versionSeriesId) {
1124        List<SimpleFragment> versFrags = seriesVersions.getSelectionFragments(versionSeriesId, null);
1125        Collections.sort(versFrags, VER_CREATED_COMPARATOR);
1126        Collections.reverse(versFrags);
1127        boolean isLatest = true;
1128        boolean isLatestMajor = true;
1129        for (SimpleFragment vsf : versFrags) {
1130
1131            // isLatestVersion
1132            vsf.put(Model.VERSION_IS_LATEST_KEY, Boolean.valueOf(isLatest));
1133            isLatest = false;
1134
1135            // isLatestMajorVersion
1136            SimpleFragment vh = getHier(vsf.getId(), true);
1137            boolean isMajor = Long.valueOf(0).equals(vh.get(Model.MAIN_MINOR_VERSION_KEY));
1138            vsf.put(Model.VERSION_IS_LATEST_MAJOR_KEY, Boolean.valueOf(isMajor && isLatestMajor));
1139            if (isMajor) {
1140                isLatestMajor = false;
1141            }
1142        }
1143    }
1144
1145    /**
1146     * Gets the version ids for a version series, ordered by creation time.
1147     */
1148    public List<Serializable> getVersionIds(Serializable versionSeriesId) {
1149        List<SimpleFragment> fragments = seriesVersions.getSelectionFragments(versionSeriesId, null);
1150        Collections.sort(fragments, VER_CREATED_COMPARATOR);
1151        return fragmentsIds(fragments);
1152    }
1153
1154    // called only when proxies enabled
1155    public List<Serializable> getSeriesProxyIds(Serializable versionSeriesId) {
1156        List<SimpleFragment> fragments = seriesProxies.getSelectionFragments(versionSeriesId, null);
1157        return fragmentsIds(fragments);
1158    }
1159
1160    // called only when proxies enabled
1161    public List<Serializable> getTargetProxyIds(Serializable targetId) {
1162        List<SimpleFragment> fragments = targetProxies.getSelectionFragments(targetId, null);
1163        return fragmentsIds(fragments);
1164    }
1165
1166    private List<Serializable> fragmentsIds(List<? extends Fragment> fragments) {
1167        return fragments.stream().map(Fragment::getId).collect(Collectors.toList());
1168    }
1169
1170    // called only when proxies enabled
1171    public Set<Serializable> getTargetProxies(Set<Serializable> ids) {
1172        // we don't read and create full selections to avoid trashing the selection cache
1173        // and because we're only interested in their ids
1174        Set<Serializable> proxyIds = new HashSet<>();
1175        // find what selections we already know about and for which ones we will have to ask the database
1176        List<Serializable> todo = new ArrayList<>();
1177        for (Serializable id : ids) {
1178            Selection selection = targetProxies.getSelectionOrNull(id);
1179            Set<Serializable> selectionProxyIds = selection == null ? null : selection.getIds();
1180            if (selectionProxyIds == null) {
1181                todo.add(id);
1182            } else {
1183                proxyIds.addAll(selectionProxyIds);
1184            }
1185        }
1186        if (!todo.isEmpty()) {
1187            // ask the database
1188            proxyIds.addAll(targetProxies.getSelectionIds(todo));
1189        }
1190        return proxyIds;
1191    }
1192
1193    /*
1194     * ----- Hierarchy -----
1195     */
1196
1197    public static class PathAndId {
1198        public final String path;
1199
1200        public final Serializable id;
1201
1202        public PathAndId(String path, Serializable id) {
1203            this.path = path;
1204            this.id = id;
1205        }
1206    }
1207
1208    /**
1209     * Gets the path by recursing up the hierarchy.
1210     */
1211    public String getPath(SimpleFragment hierFragment) {
1212        PathAndId pathAndId = getPathOrMissingParentId(hierFragment, true);
1213        return pathAndId.path;
1214    }
1215
1216    /**
1217     * Gets the full path, or the closest parent id which we don't have in cache.
1218     * <p>
1219     * If {@code fetch} is {@code true}, returns the full path.
1220     * <p>
1221     * If {@code fetch} is {@code false}, does not touch the mapper, only the context, therefore may return a missing
1222     * parent id instead of the path.
1223     *
1224     * @param fetch {@code true} if we can use the database, {@code false} if only caches should be used
1225     */
1226    public PathAndId getPathOrMissingParentId(SimpleFragment hierFragment, boolean fetch) {
1227        LinkedList<String> list = new LinkedList<>();
1228        Serializable parentId;
1229        while (true) {
1230            String name = hierFragment.getString(Model.HIER_CHILD_NAME_KEY);
1231            if (name == null) {
1232                // (empty string for normal databases, null for Oracle)
1233                name = "";
1234            }
1235            list.addFirst(name);
1236            parentId = hierFragment.get(Model.HIER_PARENT_KEY);
1237            if (parentId == null) {
1238                // root
1239                break;
1240            }
1241            // recurse in the parent
1242            RowId rowId = new RowId(Model.HIER_TABLE_NAME, parentId);
1243            hierFragment = (SimpleFragment) getIfPresent(rowId);
1244            if (hierFragment == null) {
1245                // try in mapper cache
1246                hierFragment = (SimpleFragment) getFromMapper(rowId, false, true);
1247                if (hierFragment == null) {
1248                    if (!fetch) {
1249                        return new PathAndId(null, parentId);
1250                    }
1251                    hierFragment = (SimpleFragment) getFromMapper(rowId, true, false);
1252                }
1253            }
1254        }
1255        String path;
1256        if (list.size() == 1) {
1257            String name = list.peek();
1258            if (name.isEmpty()) {
1259                // root, special case
1260                path = "/";
1261            } else {
1262                // placeless document, no initial slash
1263                path = name;
1264            }
1265        } else {
1266            path = String.join("/", list);
1267        }
1268        return new PathAndId(path, null);
1269    }
1270
1271    /**
1272     * Finds the id of the enclosing non-complex-property node.
1273     *
1274     * @param id the id
1275     * @return the id of the containing document, or {@code null} if there is no parent or the parent has been deleted.
1276     */
1277    public Serializable getContainingDocument(Serializable id) {
1278        Serializable pid = id;
1279        while (true) {
1280            if (pid == null) {
1281                // no parent
1282                return null;
1283            }
1284            SimpleFragment p = getHier(pid, false);
1285            if (p == null) {
1286                // can happen if the fragment has been deleted
1287                return null;
1288            }
1289            if (!complexProp(p)) {
1290                return pid;
1291            }
1292            pid = p.get(Model.HIER_PARENT_KEY);
1293        }
1294    }
1295
1296    // also called by Selection
1297    protected SimpleFragment getHier(Serializable id, boolean allowAbsent) {
1298        RowId rowId = new RowId(Model.HIER_TABLE_NAME, id);
1299        return (SimpleFragment) get(rowId, allowAbsent);
1300    }
1301
1302    private boolean isOrderable(Serializable parentId, boolean complexProp) {
1303        if (complexProp) {
1304            return true;
1305        }
1306        SimpleFragment parent = getHier(parentId, true);
1307        String typeName = parent.getString(Model.MAIN_PRIMARY_TYPE_KEY);
1308        return model.getDocumentTypeFacets(typeName).contains(FacetNames.ORDERABLE);
1309    }
1310
1311    /** Recursively checks if any of a fragment's parents has been deleted. */
1312    // needed because we don't recursively clear caches when doing a delete
1313    public boolean isDeleted(Serializable id) {
1314        while (id != null) {
1315            SimpleFragment fragment = getHier(id, false);
1316            State state;
1317            if (fragment == null || (state = fragment.getState()) == State.ABSENT || state == State.DELETED
1318                    || state == State.DELETED_DEPENDENT || state == State.INVALIDATED_DELETED) {
1319                return true;
1320            }
1321            id = fragment.get(Model.HIER_PARENT_KEY);
1322        }
1323        return false;
1324    }
1325
1326    /**
1327     * Gets the next pos value for a new child in a folder.
1328     *
1329     * @param nodeId the folder node id
1330     * @param complexProp whether to deal with complex properties or regular children
1331     * @return the next pos, or {@code null} if not orderable
1332     */
1333    public Long getNextPos(Serializable nodeId, boolean complexProp) {
1334        if (!isOrderable(nodeId, complexProp)) {
1335            return null;
1336        }
1337        long max = -1;
1338        for (SimpleFragment fragment : getChildren(nodeId, null, complexProp)) {
1339            Long pos = (Long) fragment.get(Model.HIER_CHILD_POS_KEY);
1340            if (pos != null && pos.longValue() > max) {
1341                max = pos.longValue();
1342            }
1343        }
1344        return Long.valueOf(max + 1);
1345    }
1346
1347    /**
1348     * Order a child before another.
1349     *
1350     * @param parentId the parent id
1351     * @param sourceId the node id to move
1352     * @param destId the node id before which to place the source node, if {@code null} then move the source to the end
1353     */
1354    public void orderBefore(Serializable parentId, Serializable sourceId, Serializable destId) {
1355        boolean complexProp = false;
1356        if (!isOrderable(parentId, complexProp)) {
1357            // TODO throw exception?
1358            return;
1359        }
1360        if (sourceId.equals(destId)) {
1361            return;
1362        }
1363        // This is optimized by assuming the number of children is small enough
1364        // to be manageable in-memory.
1365        // fetch children and relevant nodes
1366        List<SimpleFragment> fragments = getChildren(parentId, null, complexProp);
1367        // renumber fragments
1368        int i = 0;
1369        SimpleFragment source = null; // source if seen
1370        Long destPos = null;
1371        for (SimpleFragment fragment : fragments) {
1372            Serializable id = fragment.getId();
1373            if (id.equals(destId)) {
1374                destPos = Long.valueOf(i);
1375                i++;
1376                if (source != null) {
1377                    source.put(Model.HIER_CHILD_POS_KEY, destPos);
1378                }
1379            }
1380            Long setPos;
1381            if (id.equals(sourceId)) {
1382                i--;
1383                source = fragment;
1384                setPos = destPos;
1385            } else {
1386                setPos = Long.valueOf(i);
1387            }
1388            if (setPos != null) {
1389                if (!setPos.equals(fragment.get(Model.HIER_CHILD_POS_KEY))) {
1390                    fragment.put(Model.HIER_CHILD_POS_KEY, setPos);
1391                }
1392            }
1393            i++;
1394        }
1395        if (destId == null) {
1396            Long setPos = Long.valueOf(i);
1397            if (!setPos.equals(source.get(Model.HIER_CHILD_POS_KEY))) { // NOSONAR
1398                source.put(Model.HIER_CHILD_POS_KEY, setPos);
1399            }
1400        }
1401    }
1402
1403    public SimpleFragment getChildHierByName(Serializable parentId, String name, boolean complexProp) {
1404        return getHierSelectionContext(complexProp).getSelectionFragment(parentId, name);
1405    }
1406
1407    /**
1408     * Gets hier fragments for children.
1409     */
1410    public List<SimpleFragment> getChildren(Serializable parentId, String name, boolean complexProp) {
1411        List<SimpleFragment> fragments = getHierSelectionContext(complexProp).getSelectionFragments(parentId, name);
1412        if (isOrderable(parentId, complexProp)) {
1413            // sort children in order
1414            Collections.sort(fragments, POS_COMPARATOR);
1415        }
1416        return fragments;
1417    }
1418
1419    /** Checks that we don't move/copy under ourselves. */
1420    protected void checkNotUnder(Serializable parentId, Serializable id, String op) {
1421        Serializable pid = parentId;
1422        do {
1423            if (pid.equals(id)) {
1424                throw new DocumentExistsException(
1425                        "Cannot " + op + " a node under itself: " + parentId + " is under " + id);
1426            }
1427            SimpleFragment p = getHier(pid, false);
1428            if (p == null) {
1429                // cannot happen
1430                throw new NuxeoException("No parent: " + pid);
1431            }
1432            pid = p.get(Model.HIER_PARENT_KEY);
1433        } while (pid != null);
1434    }
1435
1436    /** Checks that a name is free. Cannot check concurrent sessions though. */
1437    protected void checkFreeName(Serializable parentId, String name, boolean complexProp) {
1438        Fragment fragment = getChildHierByName(parentId, name, complexProp);
1439        if (fragment != null) {
1440            throw new DocumentExistsException("Destination name already exists: " + name);
1441        }
1442    }
1443
1444    /**
1445     * Move a child to a new parent with a new name.
1446     *
1447     * @param source the source
1448     * @param parentId the destination parent id
1449     * @param name the new name
1450     */
1451    public void move(Node source, Serializable parentId, String name) {
1452        // a save() has already been done by the caller when doing
1453        // an actual move (different parents)
1454        Serializable id = source.getId();
1455        SimpleFragment hierFragment = source.getHierFragment();
1456        Serializable oldParentId = hierFragment.get(Model.HIER_PARENT_KEY);
1457        String oldName = hierFragment.getString(Model.HIER_CHILD_NAME_KEY);
1458        if (!oldParentId.equals(parentId)) {
1459            checkNotUnder(parentId, id, "move");
1460        } else if (oldName.equals(name)) {
1461            // null move
1462            return;
1463        }
1464        boolean complexProp = complexProp(hierFragment);
1465        checkFreeName(parentId, name, complexProp);
1466        /*
1467         * Do the move.
1468         */
1469        if (!oldName.equals(name)) {
1470            hierFragment.put(Model.HIER_CHILD_NAME_KEY, name);
1471        }
1472        // cache management
1473        getHierSelectionContext(complexProp).recordRemoved(hierFragment);
1474        hierFragment.put(Model.HIER_PARENT_KEY, parentId);
1475        getHierSelectionContext(complexProp).recordExisting(hierFragment, true);
1476        // path invalidated
1477        source.path = null;
1478    }
1479
1480    /**
1481     * Copy a child to a new parent with a new name.
1482     *
1483     * @param source the source of the copy
1484     * @param parentId the destination parent id
1485     * @param name the new name
1486     * @param afterRecordCopy a consumer called on ids of nodes that were records before copy
1487     * @return the id of the copy
1488     */
1489    public Serializable copy(Node source, Serializable parentId, String name, Consumer<Serializable> afterRecordCopy) {
1490        Serializable id = source.getId();
1491        SimpleFragment hierFragment = source.getHierFragment();
1492        Serializable oldParentId = hierFragment.get(Model.HIER_PARENT_KEY);
1493        if (oldParentId != null && !oldParentId.equals(parentId)) {
1494            checkNotUnder(parentId, id, "copy");
1495        }
1496        checkFreeName(parentId, name, complexProp(hierFragment));
1497        // do the copy
1498        Long pos = getNextPos(parentId, false);
1499        // no special children (comments, annotations...) kept on document copy
1500        boolean excludeSpecialChildren = true;
1501        boolean excludeACL = false;
1502        CopyResult copyResult = mapper.copy(new IdWithTypes(source), parentId, name, null, excludeSpecialChildren,
1503                excludeACL);
1504        Serializable newId = copyResult.copyId;
1505        // read new child in this session (updates children Selection)
1506        SimpleFragment copy = getHier(newId, false);
1507        // invalidate child in other sessions' children Selection
1508        markInvalidated(copyResult.invalidations);
1509        // read new proxies in this session (updates Selections)
1510        List<RowId> rowIds = new ArrayList<>();
1511        for (Serializable proxyId : copyResult.proxyIds) {
1512            rowIds.add(new RowId(Model.PROXY_TABLE_NAME, proxyId));
1513        }
1514        // multi-fetch will register the new fragments with the Selections
1515        List<Fragment> fragments = getMulti(rowIds, true);
1516        // invalidate Selections in other sessions
1517        for (Fragment fragment : fragments) {
1518            seriesProxies.recordExisting((SimpleFragment) fragment, true);
1519            targetProxies.recordExisting((SimpleFragment) fragment, true);
1520        }
1521        // version copy fixup
1522        if (source.isVersion()) {
1523            copy.put(Model.MAIN_IS_VERSION_KEY, null);
1524        }
1525        // pos fixup
1526        copy.put(Model.HIER_CHILD_POS_KEY, pos);
1527        // record fixup
1528        if (afterRecordCopy != null) {
1529            for (Serializable recordId : copyResult.recordIds) {
1530                afterRecordCopy.accept(recordId);
1531            }
1532        }
1533        return newId;
1534    }
1535
1536    /**
1537     * Checks in a node (creates a version).
1538     *
1539     * @param node the node to check in
1540     * @param label the version label
1541     * @param checkinComment the version description
1542     * @return the created version id
1543     */
1544    public Serializable checkIn(Node node, String label, String checkinComment) {
1545        Boolean checkedIn = (Boolean) node.hierFragment.get(Model.MAIN_CHECKED_IN_KEY);
1546        if (Boolean.TRUE.equals(checkedIn)) {
1547            throw new NuxeoException("Already checked in");
1548        }
1549        if (label == null) {
1550            // use version major + minor as label
1551            Serializable major = node.getSimpleProperty(Model.MAIN_MAJOR_VERSION_PROP).getValue();
1552            Serializable minor = node.getSimpleProperty(Model.MAIN_MINOR_VERSION_PROP).getValue();
1553            if (major == null) {
1554                major = "0";
1555            }
1556            if (minor == null) {
1557                minor = "0";
1558            }
1559            label = major + "." + minor;
1560        }
1561
1562        /*
1563         * Do the copy without non-complex children, with null parent.
1564         */
1565        Serializable id = node.getId();
1566        // complex (special) children are snapshot when checking in
1567        boolean excludeSpecialChildren = false;
1568        // except in legacy mode, we don't copy the live ACL when creating a version
1569        boolean excludeACL = versionAclMode != VersionAclMode.LEGACY;
1570        CopyResult res = mapper.copy(new IdWithTypes(node), null, null, null, excludeSpecialChildren, excludeACL);
1571        Serializable newId = res.copyId;
1572        markInvalidated(res.invalidations);
1573        // add version as a new child of its parent
1574        SimpleFragment verHier = getHier(newId, false);
1575        verHier.put(Model.MAIN_IS_VERSION_KEY, Boolean.TRUE);
1576        boolean isMajor = Long.valueOf(0).equals(verHier.get(Model.MAIN_MINOR_VERSION_KEY));
1577
1578        // create a "version" row for our new version
1579        Row row = new Row(Model.VERSION_TABLE_NAME, newId);
1580        row.putNew(Model.VERSION_VERSIONABLE_KEY, id);
1581        row.putNew(Model.VERSION_CREATED_KEY, new GregorianCalendar()); // now
1582        row.putNew(Model.VERSION_LABEL_KEY, label);
1583        row.putNew(Model.VERSION_DESCRIPTION_KEY, checkinComment);
1584        row.putNew(Model.VERSION_IS_LATEST_KEY, Boolean.TRUE);
1585        row.putNew(Model.VERSION_IS_LATEST_MAJOR_KEY, Boolean.valueOf(isMajor));
1586        createVersionFragment(row);
1587
1588        // update the original node to reflect that it's checked in
1589        node.hierFragment.put(Model.MAIN_CHECKED_IN_KEY, Boolean.TRUE);
1590        node.hierFragment.put(Model.MAIN_BASE_VERSION_KEY, newId);
1591
1592        recomputeVersionSeries(id);
1593
1594        return newId;
1595    }
1596
1597    /**
1598     * Checks out a node.
1599     *
1600     * @param node the node to check out
1601     */
1602    public void checkOut(Node node) {
1603        Boolean checkedIn = (Boolean) node.hierFragment.get(Model.MAIN_CHECKED_IN_KEY);
1604        if (!Boolean.TRUE.equals(checkedIn)) {
1605            throw new NuxeoException("Already checked out");
1606        }
1607        // update the node to reflect that it's checked out
1608        node.hierFragment.put(Model.MAIN_CHECKED_IN_KEY, Boolean.FALSE);
1609    }
1610
1611    /**
1612     * Restores a node to a given version.
1613     * <p>
1614     * The restored node is checked in.
1615     *
1616     * @param node the node
1617     * @param version the version to restore on this node
1618     */
1619    public void restoreVersion(Node node, Node version) {
1620        Serializable versionableId = node.getId();
1621        Serializable versionId = version.getId();
1622
1623        // clear complex properties
1624        List<SimpleFragment> children = getChildren(versionableId, null, true);
1625        // copy to avoid concurrent modifications
1626        for (SimpleFragment child : children.toArray(new SimpleFragment[children.size()])) {
1627            removePropertyNode(child);
1628        }
1629        session.flush(); // flush deletes
1630
1631        // copy the version values
1632        Row overwriteRow = new Row(Model.HIER_TABLE_NAME, versionableId);
1633        SimpleFragment versionHier = version.getHierFragment();
1634        for (String key : model.getFragmentKeysType(Model.HIER_TABLE_NAME).keySet()) {
1635            // keys we don't copy from version when restoring
1636            if (key.equals(Model.HIER_PARENT_KEY) || key.equals(Model.HIER_CHILD_NAME_KEY)
1637                    || key.equals(Model.HIER_CHILD_POS_KEY) || key.equals(Model.HIER_CHILD_ISPROPERTY_KEY)
1638                    || key.equals(Model.MAIN_PRIMARY_TYPE_KEY) || key.equals(Model.MAIN_CHECKED_IN_KEY)
1639                    || key.equals(Model.MAIN_BASE_VERSION_KEY) || key.equals(Model.MAIN_IS_VERSION_KEY)
1640                    || key.equals(Model.MAIN_IS_RECORD_KEY) || key.equals(Model.MAIN_RETAIN_UNTIL_KEY)
1641                    || key.equals(Model.MAIN_HAS_LEGAL_HOLD_KEY)) {
1642                continue;
1643            }
1644            overwriteRow.putNew(key, versionHier.get(key));
1645        }
1646        overwriteRow.putNew(Model.MAIN_CHECKED_IN_KEY, Boolean.TRUE);
1647        overwriteRow.putNew(Model.MAIN_BASE_VERSION_KEY, versionId);
1648        overwriteRow.putNew(Model.MAIN_IS_VERSION_KEY, null);
1649        overwriteRow.putNew(Model.MAIN_IS_RECORD_KEY, null);
1650        overwriteRow.putNew(Model.MAIN_RETAIN_UNTIL_KEY, null);
1651        overwriteRow.putNew(Model.MAIN_HAS_LEGAL_HOLD_KEY, null);
1652        // exclude special children to avoid duplicates on restore
1653        boolean excludeSpecialChildren = true;
1654        boolean excludeACL = true;
1655        CopyResult res = mapper.copy(new IdWithTypes(version), node.getParentId(), null, overwriteRow,
1656                excludeSpecialChildren, excludeACL);
1657        markInvalidated(res.invalidations);
1658    }
1659
1660}