001/*
002 * (C) Copyright 2014-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.dbs;
020
021import static java.lang.Boolean.TRUE;
022import static org.nuxeo.ecm.core.action.DeletionAction.ACTION_NAME;
023import static org.nuxeo.ecm.core.api.AbstractSession.DISABLED_ISLATESTVERSION_PROPERTY;
024import static org.nuxeo.ecm.core.api.CoreSession.BINARY_FULLTEXT_MAIN_KEY;
025import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.FACETED_TAG;
026import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.FACETED_TAG_LABEL;
027import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_CHANGE_TOKEN;
028import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.INITIAL_SYS_CHANGE_TOKEN;
029import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_BEGIN;
030import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_CREATOR;
031import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_END;
032import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_GRANT;
033import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_PERMISSION;
034import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_STATUS;
035import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACE_USER;
036import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL;
037import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACL_NAME;
038import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ACP;
039import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ANCESTOR_IDS;
040import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_BASE_VERSION_ID;
041import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_CHANGE_TOKEN;
042import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_BINARY;
043import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_JOBID;
044import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SCORE;
045import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_FULLTEXT_SIMPLE;
046import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_HAS_LEGAL_HOLD;
047import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_ID;
048import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_CHECKED_IN;
049import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_LATEST_MAJOR_VERSION;
050import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_LATEST_VERSION;
051import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_PROXY;
052import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_RECORD;
053import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_RETENTION_ACTIVE;
054import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_TRASHED;
055import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_IS_VERSION;
056import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_LIFECYCLE_POLICY;
057import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_LIFECYCLE_STATE;
058import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_LOCK_CREATED;
059import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_LOCK_OWNER;
060import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_MAJOR_VERSION;
061import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_MINOR_VERSION;
062import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_MIXIN_TYPES;
063import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_NAME;
064import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PARENT_ID;
065import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PATH_INTERNAL;
066import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_POS;
067import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PRIMARY_TYPE;
068import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_IDS;
069import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_TARGET_ID;
070import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_PROXY_VERSION_SERIES_ID;
071import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_READ_ACL;
072import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_RETAIN_UNTIL;
073import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_SYS_CHANGE_TOKEN;
074import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_CREATED;
075import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_DESCRIPTION;
076import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_LABEL;
077import static org.nuxeo.ecm.core.storage.dbs.DBSDocument.KEY_VERSION_SERIES_ID;
078
079import java.io.IOException;
080import java.io.Serializable;
081import java.text.Normalizer;
082import java.util.ArrayList;
083import java.util.Arrays;
084import java.util.Calendar;
085import java.util.Collection;
086import java.util.Collections;
087import java.util.Comparator;
088import java.util.GregorianCalendar;
089import java.util.HashMap;
090import java.util.HashSet;
091import java.util.Iterator;
092import java.util.LinkedList;
093import java.util.List;
094import java.util.Map;
095import java.util.Map.Entry;
096import java.util.NoSuchElementException;
097import java.util.Objects;
098import java.util.Set;
099import java.util.function.Consumer;
100import java.util.regex.Matcher;
101import java.util.regex.Pattern;
102import java.util.stream.Collectors;
103import java.util.stream.Stream;
104
105import org.apache.commons.lang3.StringUtils;
106import org.apache.commons.lang3.mutable.Mutable;
107import org.apache.commons.lang3.mutable.MutableObject;
108import org.apache.commons.logging.Log;
109import org.apache.commons.logging.LogFactory;
110import org.nuxeo.ecm.core.api.Blob;
111import org.nuxeo.ecm.core.api.CoreSession;
112import org.nuxeo.ecm.core.api.DocumentExistsException;
113import org.nuxeo.ecm.core.api.DocumentNotFoundException;
114import org.nuxeo.ecm.core.api.IterableQueryResult;
115import org.nuxeo.ecm.core.api.NuxeoException;
116import org.nuxeo.ecm.core.api.NuxeoPrincipal;
117import org.nuxeo.ecm.core.api.PartialList;
118import org.nuxeo.ecm.core.api.PropertyException;
119import org.nuxeo.ecm.core.api.ScrollResult;
120import org.nuxeo.ecm.core.api.VersionModel;
121import org.nuxeo.ecm.core.api.lock.LockManager;
122import org.nuxeo.ecm.core.api.repository.FulltextConfiguration;
123import org.nuxeo.ecm.core.api.security.ACE;
124import org.nuxeo.ecm.core.api.security.ACL;
125import org.nuxeo.ecm.core.api.security.ACP;
126import org.nuxeo.ecm.core.api.security.impl.ACLImpl;
127import org.nuxeo.ecm.core.api.security.impl.ACPImpl;
128import org.nuxeo.ecm.core.blob.BlobInfo;
129import org.nuxeo.ecm.core.bulk.BulkService;
130import org.nuxeo.ecm.core.bulk.message.BulkCommand;
131import org.nuxeo.ecm.core.model.BaseSession;
132import org.nuxeo.ecm.core.model.Document;
133import org.nuxeo.ecm.core.model.Session;
134import org.nuxeo.ecm.core.query.QueryFilter;
135import org.nuxeo.ecm.core.query.QueryParseException;
136import org.nuxeo.ecm.core.query.sql.NXQL;
137import org.nuxeo.ecm.core.query.sql.SQLQueryParser;
138import org.nuxeo.ecm.core.query.sql.model.Operand;
139import org.nuxeo.ecm.core.query.sql.model.OrderByClause;
140import org.nuxeo.ecm.core.query.sql.model.OrderByExpr;
141import org.nuxeo.ecm.core.query.sql.model.OrderByList;
142import org.nuxeo.ecm.core.query.sql.model.Reference;
143import org.nuxeo.ecm.core.query.sql.model.SQLQuery;
144import org.nuxeo.ecm.core.query.sql.model.SelectClause;
145import org.nuxeo.ecm.core.schema.DocumentType;
146import org.nuxeo.ecm.core.schema.FacetNames;
147import org.nuxeo.ecm.core.schema.SchemaManager;
148import org.nuxeo.ecm.core.schema.types.ListTypeImpl;
149import org.nuxeo.ecm.core.schema.types.Type;
150import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
151import org.nuxeo.ecm.core.schema.types.primitives.DateType;
152import org.nuxeo.ecm.core.schema.types.primitives.StringType;
153import org.nuxeo.ecm.core.storage.BaseDocument;
154import org.nuxeo.ecm.core.storage.ExpressionEvaluator;
155import org.nuxeo.ecm.core.storage.QueryOptimizer;
156import org.nuxeo.ecm.core.storage.State;
157import org.nuxeo.ecm.core.storage.StateHelper;
158import org.nuxeo.runtime.api.Framework;
159import org.nuxeo.runtime.metrics.MetricsService;
160import org.nuxeo.runtime.transaction.TransactionHelper;
161
162import io.dropwizard.metrics5.MetricName;
163import io.dropwizard.metrics5.MetricRegistry;
164import io.dropwizard.metrics5.SharedMetricRegistries;
165import io.dropwizard.metrics5.Timer;
166
167/**
168 * Implementation of a {@link Session} for Document-Based Storage.
169 *
170 * @since 5.9.4
171 */
172public class DBSSession extends BaseSession {
173
174    private static final Log log = LogFactory.getLog(DBSSession.class);
175
176    protected static final Set<String> KEYS_RETENTION_HOLD_AND_PROXIES = new HashSet<>(Arrays.asList(KEY_RETAIN_UNTIL,
177            KEY_HAS_LEGAL_HOLD, KEY_IS_RETENTION_ACTIVE, KEY_IS_PROXY, KEY_PROXY_TARGET_ID, KEY_PROXY_IDS));
178
179    protected final DBSTransactionState transaction;
180
181    protected final boolean fulltextStoredInBlob;
182
183    protected final boolean fulltextSearchDisabled;
184
185    protected final boolean changeTokenEnabled;
186
187    protected final MetricRegistry registry = SharedMetricRegistries.getOrCreate(MetricsService.class.getName());
188
189    private final Timer saveTimer;
190
191    private final Timer queryTimer;
192
193    private static final String LOG_MIN_DURATION_KEY = "org.nuxeo.dbs.query.log_min_duration_ms";
194
195    private long LOG_MIN_DURATION_NS = -1 * 1000000;
196
197    protected boolean isLatestVersionDisabled = false;
198
199    public DBSSession(DBSRepository repository) {
200        super(repository);
201        transaction = new DBSTransactionState(repository, this);
202        FulltextConfiguration fulltextConfiguration = repository.getFulltextConfiguration();
203        fulltextStoredInBlob = fulltextConfiguration != null && fulltextConfiguration.fulltextStoredInBlob;
204        fulltextSearchDisabled = fulltextConfiguration == null || fulltextConfiguration.fulltextSearchDisabled;
205        changeTokenEnabled = repository.isChangeTokenEnabled();
206
207        saveTimer = registry.timer(MetricName.build("nuxeo", "repositories", "repository", "save")
208                                             .tagged("repository", repository.getName()));
209        queryTimer = registry.timer(MetricName.build("nuxeo", "repositories", "repository", "query")
210                                              .tagged("repository", repository.getName()));
211        LOG_MIN_DURATION_NS = Long.parseLong(Framework.getProperty(LOG_MIN_DURATION_KEY, "-1")) * 1000000;
212        isLatestVersionDisabled = Framework.isBooleanPropertyTrue(DISABLED_ISLATESTVERSION_PROPERTY);
213    }
214
215    @Override
216    public String getRepositoryName() {
217        return repository.getName();
218    }
219
220    @Override
221    public void destroy() {
222        transaction.close();
223    }
224
225    @SuppressWarnings("resource") // timerContext closed by stop() in finally
226    @Override
227    public void save() {
228        final Timer.Context timerContext = saveTimer.time();
229        try {
230            transaction.save();
231            if (!TransactionHelper.isTransactionActiveOrMarkedRollback()) {
232                transaction.commit();
233            }
234        } finally {
235            timerContext.stop();
236        }
237    }
238
239    protected String getRootId() {
240        return transaction.getRootId();
241    }
242
243    /*
244     * Normalize using NFC to avoid decomposed characters (like 'e' + COMBINING ACUTE ACCENT instead of LATIN SMALL
245     * LETTER E WITH ACUTE). NFKC (normalization using compatibility decomposition) is not used, because compatibility
246     * decomposition turns some characters (LATIN SMALL LIGATURE FFI, TRADE MARK SIGN, FULLWIDTH SOLIDUS) into a series
247     * of characters ('f'+'f'+'i', 'T'+'M', '/') that cannot be re-composed into the original, and therefore loses
248     * information.
249     */
250    protected String normalize(String path) {
251        return Normalizer.normalize(path, Normalizer.Form.NFC);
252    }
253
254    @Override
255    public Document resolvePath(String path) {
256        // TODO move checks and normalize higher in call stack
257        if (path == null) {
258            throw new DocumentNotFoundException("Null path");
259        }
260        int len = path.length();
261        if (len == 0) {
262            throw new DocumentNotFoundException("Empty path");
263        }
264        if (path.charAt(0) != '/') {
265            throw new DocumentNotFoundException("Relative path: " + path);
266        }
267        if (len > 1 && path.charAt(len - 1) == '/') {
268            // remove final slash
269            path = path.substring(0, len - 1);
270            len--;
271        }
272        path = normalize(path);
273
274        if (len == 1) {
275            return getRootDocument();
276        }
277        DBSDocumentState docState = null;
278        String parentId = getRootId();
279        String[] names = path.split("/", -1);
280        for (int i = 1; i < names.length; i++) {
281            String name = names[i];
282            if (name.length() == 0) {
283                throw new DocumentNotFoundException("Path with empty component: " + path);
284            }
285            docState = transaction.getChildState(parentId, name);
286            if (docState == null) {
287                throw new DocumentNotFoundException(path);
288            }
289            parentId = docState.getId();
290        }
291        return getDocument(docState);
292    }
293
294    protected String getDocumentIdByPath(String path) {
295        // TODO move checks and normalize higher in call stack
296        if (path == null) {
297            throw new DocumentNotFoundException("Null path");
298        }
299        int len = path.length();
300        if (len == 0) {
301            throw new DocumentNotFoundException("Empty path");
302        }
303        if (path.charAt(0) != '/') {
304            throw new DocumentNotFoundException("Relative path: " + path);
305        }
306        if (len > 1 && path.charAt(len - 1) == '/') {
307            // remove final slash
308            path = path.substring(0, len - 1);
309            len--;
310        }
311        path = normalize(path);
312
313        if (len == 1) {
314            return getRootId();
315        }
316        DBSDocumentState docState = null;
317        String parentId = getRootId();
318        String[] names = path.split("/", -1);
319        for (int i = 1; i < names.length; i++) {
320            String name = names[i];
321            if (name.length() == 0) {
322                throw new DocumentNotFoundException("Path with empty component: " + path);
323            }
324            // TODO XXX add getChildId method
325            docState = transaction.getChildState(parentId, name);
326            if (docState == null) {
327                return null;
328            }
329            parentId = docState.getId();
330        }
331        return docState.getId(); // NOSONAR
332    }
333
334    protected Document getChild(String parentId, String name) {
335        name = normalize(name);
336        DBSDocumentState docState = transaction.getChildState(parentId, name);
337        DBSDocument doc = getDocument(docState);
338        if (doc == null) {
339            throw new DocumentNotFoundException(name);
340        }
341        return doc;
342    }
343
344    protected List<Document> getChildren(String parentId) {
345        List<DBSDocumentState> docStates = transaction.getChildrenStates(parentId);
346        if (isOrderable(parentId)) {
347            // sort children in order
348            docStates.sort(POS_COMPARATOR);
349        }
350        List<Document> children = new ArrayList<>(docStates.size());
351        for (DBSDocumentState docState : docStates) {
352            try {
353                children.add(getDocument(docState));
354            } catch (DocumentNotFoundException e) {
355                // ignore error retrieving one of the children
356                // (Unknown document type)
357                continue;
358            }
359        }
360        return children;
361    }
362
363    protected List<String> getChildrenIds(String parentId) {
364        // We want all children, the filter flags are null
365        boolean excludeSpecialChildren = false;
366        boolean excludeRegularChildren = false;
367        return getChildrenIds(parentId, excludeSpecialChildren, excludeRegularChildren);
368    }
369
370    protected List<String> getChildrenIds(String parentId, boolean excludeSpecialChildren,
371            boolean excludeRegularChildren) {
372        if (isOrderable(parentId)) {
373            // TODO get only id and pos, not full state
374            // TODO state not for update
375            List<DBSDocumentState> docStates = //
376                    transaction.getChildrenStates(parentId, excludeSpecialChildren, excludeRegularChildren);
377            docStates.sort(POS_COMPARATOR);
378            List<String> children = new ArrayList<>(docStates.size());
379            for (DBSDocumentState docState : docStates) {
380                children.add(docState.getId());
381            }
382            return children;
383        } else {
384            return transaction.getChildrenIds(parentId, excludeSpecialChildren, excludeRegularChildren);
385        }
386    }
387
388    protected boolean hasChildren(String parentId) {
389        return transaction.hasChildren(parentId);
390
391    }
392
393    @Override
394    public Document getDocumentByUUID(String id) {
395        Document doc = getDocument(id);
396        if (doc != null) {
397            return doc;
398        }
399        // exception required by API
400        throw new DocumentNotFoundException(id);
401    }
402
403    @Override
404    public Document getRootDocument() {
405        return getDocument(getRootId());
406    }
407
408    @Override
409    public Document getNullDocument() {
410        return new DBSDocument(null, null, this, true);
411    }
412
413    protected DBSDocument getDocument(String id) {
414        DBSDocumentState docState = transaction.getStateForUpdate(id);
415        return getDocument(docState);
416    }
417
418    protected List<Document> getDocuments(List<String> ids) {
419        List<DBSDocumentState> docStates = transaction.getStatesForUpdate(ids);
420        List<Document> docs = new ArrayList<>(ids.size());
421        for (DBSDocumentState docState : docStates) {
422            try {
423                docs.add(getDocument(docState));
424            } catch (DocumentNotFoundException e) {
425                // unknown type in db or null proxy target, ignore
426                continue;
427            }
428        }
429        return docs;
430    }
431
432    protected DBSDocument getDocument(DBSDocumentState docState) {
433        return getDocument(docState, true);
434    }
435
436    protected DBSDocument getDocument(DBSDocumentState docState, boolean readonly) {
437        if (docState == null) {
438            return null;
439        }
440        boolean isVersion = TRUE.equals(docState.get(KEY_IS_VERSION));
441
442        String typeName = docState.getPrimaryType();
443        SchemaManager schemaManager = Framework.getService(SchemaManager.class);
444        DocumentType type = schemaManager.getDocumentType(typeName);
445        if (type == null) {
446            throw new DocumentNotFoundException("Unknown document type: " + typeName);
447        }
448
449        boolean isProxy = TRUE.equals(docState.get(KEY_IS_PROXY));
450        if (isProxy) {
451            String targetId = (String) docState.get(KEY_PROXY_TARGET_ID);
452            DBSDocumentState targetState = transaction.getStateForUpdate(targetId);
453            if (targetState == null) {
454                throw new DocumentNotFoundException("Proxy has null target");
455            }
456        }
457
458        if (isVersion) {
459            return new DBSDocument(docState, type, this, readonly);
460        } else {
461            return new DBSDocument(docState, type, this, false);
462        }
463    }
464
465    protected boolean hasChild(String parentId, String name) {
466        name = normalize(name);
467        return transaction.hasChild(parentId, name);
468    }
469
470    public Document createChild(String id, String parentId, String name, Long pos, String typeName) {
471        name = normalize(name);
472        DBSDocumentState docState = createChildState(id, parentId, name, pos, typeName);
473        return getDocument(docState);
474    }
475
476    protected DBSDocumentState createChildState(String id, String parentId, String name, Long pos, String typeName) {
477        if (pos == null && parentId != null) {
478            pos = getNextPos(parentId);
479        }
480        return transaction.createChild(id, parentId, name, pos, typeName);
481    }
482
483    protected boolean isOrderable(String id) {
484        State state = transaction.getStateForRead(id);
485        String typeName = (String) state.get(KEY_PRIMARY_TYPE);
486        SchemaManager schemaManager = Framework.getService(SchemaManager.class);
487        return schemaManager.getDocumentType(typeName).getFacets().contains(FacetNames.ORDERABLE);
488    }
489
490    protected Long getNextPos(String parentId) {
491        if (!isOrderable(parentId)) {
492            return null;
493        }
494        long max = -1;
495        for (DBSDocumentState docState : transaction.getChildrenStates(parentId)) {
496            Long pos = (Long) docState.get(KEY_POS);
497            if (pos != null && pos > max) {
498                max = pos;
499            }
500        }
501        return max + 1;
502    }
503
504    protected void orderBefore(String parentId, String sourceId, String destId) {
505        if (!isOrderable(parentId)) {
506            // TODO throw exception?
507            return;
508        }
509        if (sourceId.equals(destId)) {
510            return;
511        }
512        // This is optimized by assuming the number of children is small enough
513        // to be manageable in-memory.
514        // fetch children
515        List<DBSDocumentState> docStates = transaction.getChildrenStates(parentId);
516        // sort children in order
517        docStates.sort(POS_COMPARATOR);
518        // renumber
519        int i = 0;
520        DBSDocumentState source = null; // source if seen
521        Long destPos = null;
522        for (DBSDocumentState docState : docStates) {
523            Serializable id = docState.getId();
524            if (id.equals(destId)) {
525                destPos = Long.valueOf(i);
526                i++;
527                if (source != null) {
528                    source.put(KEY_POS, destPos);
529                }
530            }
531            Long setPos;
532            if (id.equals(sourceId)) {
533                i--;
534                source = docState;
535                setPos = destPos;
536            } else {
537                setPos = Long.valueOf(i);
538            }
539            if (setPos != null) {
540                if (!setPos.equals(docState.get(KEY_POS))) {
541                    docState.put(KEY_POS, setPos);
542                }
543            }
544            i++;
545        }
546        if (destId == null) {
547            Long setPos = Long.valueOf(i);
548            if (!setPos.equals(source.get(KEY_POS))) { // NOSONAR
549                source.put(KEY_POS, setPos);
550            }
551        }
552    }
553
554    protected void checkOut(String id) {
555        DBSDocumentState docState = transaction.getStateForUpdate(id);
556        if (!TRUE.equals(docState.get(KEY_IS_CHECKED_IN))) {
557            throw new NuxeoException("Already checked out");
558        }
559        docState.put(KEY_IS_CHECKED_IN, null);
560    }
561
562    protected Document checkIn(String id, String label, String checkinComment) {
563        transaction.save();
564        DBSDocumentState docState = transaction.getStateForUpdate(id);
565        if (TRUE.equals(docState.get(KEY_IS_CHECKED_IN))) {
566            throw new NuxeoException("Already checked in");
567        }
568        if (label == null) {
569            // use version major + minor as label
570            Long major = (Long) docState.get(KEY_MAJOR_VERSION);
571            Long minor = (Long) docState.get(KEY_MINOR_VERSION);
572            if (major == null || minor == null) {
573                label = "";
574            } else {
575                label = major + "." + minor;
576            }
577        }
578
579        // copy into a version and create a snapshot of special children but not the regular children
580        boolean excludeSpecialChildren = false;
581        boolean excludeRegularChildren = true;
582        String versionId = //
583                copyRecurse(id, null, new LinkedList<>(), null, excludeSpecialChildren, excludeRegularChildren);
584        DBSDocumentState verState = transaction.getStateForUpdate(versionId);
585        String verId = verState.getId();
586        verState.put(KEY_PARENT_ID, null);
587        verState.put(KEY_ANCESTOR_IDS, null);
588        verState.put(KEY_IS_VERSION, TRUE);
589        verState.put(KEY_VERSION_SERIES_ID, id);
590        verState.put(KEY_VERSION_CREATED, new GregorianCalendar()); // now
591        verState.put(KEY_VERSION_LABEL, label);
592        verState.put(KEY_VERSION_DESCRIPTION, checkinComment);
593        verState.put(KEY_IS_LATEST_VERSION, TRUE);
594        verState.put(KEY_IS_CHECKED_IN, null);
595        verState.put(KEY_BASE_VERSION_ID, null);
596        boolean isMajor = Long.valueOf(0).equals(verState.get(KEY_MINOR_VERSION));
597        verState.put(KEY_IS_LATEST_MAJOR_VERSION, isMajor ? TRUE : null);
598        // except in legacy mode, we don't copy the live ACL when creating a version
599        if (versionAclMode != VersionAclMode.LEGACY) {
600            verState.put(KEY_ACP, null);
601        }
602
603        // update the doc to mark it checked in
604        docState.put(KEY_IS_CHECKED_IN, TRUE);
605        docState.put(KEY_BASE_VERSION_ID, verId);
606
607        if (!isLatestVersionDisabled) {
608            recomputeVersionSeries(id);
609        }
610
611        // update read acls
612        transaction.updateTreeReadAcls(verId);
613        transaction.save();
614
615        return getDocument(verId);
616    }
617
618    /**
619     * Recomputes isLatest / isLatestMajor on all versions.
620     */
621    protected void recomputeVersionSeries(String versionSeriesId) {
622        List<DBSDocumentState> docStates = transaction.getKeyValuedStates(KEY_VERSION_SERIES_ID, versionSeriesId,
623                KEY_IS_VERSION, TRUE);
624        docStates.sort(VERSION_CREATED_COMPARATOR);
625        Collections.reverse(docStates);
626        boolean isLatest = true;
627        boolean isLatestMajor = true;
628        for (DBSDocumentState docState : docStates) {
629            // isLatestVersion
630            docState.put(KEY_IS_LATEST_VERSION, isLatest ? TRUE : null);
631            isLatest = false;
632            // isLatestMajorVersion
633            boolean isMajor = Long.valueOf(0).equals(docState.get(KEY_MINOR_VERSION));
634            docState.put(KEY_IS_LATEST_MAJOR_VERSION, isMajor && isLatestMajor ? TRUE : null);
635            if (isMajor) {
636                isLatestMajor = false;
637            }
638        }
639    }
640
641    protected void restoreVersion(Document doc, Document version) {
642        String docId = doc.getUUID();
643        String versionId = version.getUUID();
644
645        DBSDocumentState docState = transaction.getStateForUpdate(docId);
646
647        if (TRUE.equals(docState.get(KEY_IS_RECORD))) {
648            getDocumentBlobManager().notifyBeforeRemove(doc);
649        }
650
651        State versionState = transaction.getStateForRead(versionId);
652
653        // clear all data
654        for (String key : docState.state.keyArray()) {
655            if (!keepWhenRestore(key)) {
656                docState.put(key, null);
657            }
658        }
659        // update from version
660        for (Entry<String, Serializable> en : versionState.entrySet()) {
661            String key = en.getKey();
662            if (!keepWhenRestore(key)) {
663                docState.put(key, StateHelper.deepCopy(en.getValue()));
664            }
665        }
666        docState.put(KEY_IS_VERSION, null);
667        docState.put(KEY_IS_CHECKED_IN, TRUE);
668        docState.put(KEY_BASE_VERSION_ID, versionId);
669        docState.put(KEY_IS_RECORD, null);
670        docState.put(KEY_RETAIN_UNTIL, null);
671        docState.put(KEY_HAS_LEGAL_HOLD, null);
672
673        if (TRUE.equals(versionState.get(KEY_IS_RECORD))) {
674            notifyAfterCopy(doc);
675        }
676    }
677
678    // keys we don't copy from version when restoring
679    protected boolean keepWhenRestore(String key) {
680        switch (key) {
681        // these are placeful stuff
682        case KEY_ID:
683        case KEY_PARENT_ID:
684        case KEY_ANCESTOR_IDS:
685        case KEY_NAME:
686        case KEY_POS:
687        case KEY_PRIMARY_TYPE:
688        case KEY_ACP:
689        case KEY_READ_ACL:
690            // these are version-specific
691        case KEY_VERSION_CREATED:
692        case KEY_VERSION_DESCRIPTION:
693        case KEY_VERSION_LABEL:
694        case KEY_VERSION_SERIES_ID:
695        case KEY_IS_LATEST_VERSION:
696        case KEY_IS_LATEST_MAJOR_VERSION:
697            // these will be updated after restore
698        case KEY_IS_VERSION:
699        case KEY_IS_CHECKED_IN:
700        case KEY_BASE_VERSION_ID:
701            // record
702        case KEY_IS_RECORD:
703        case KEY_RETAIN_UNTIL:
704        case KEY_HAS_LEGAL_HOLD:
705            return true;
706        }
707        return false;
708    }
709
710    @Override
711    public Document copy(Document source, Document parent, String name) {
712        transaction.save();
713        if (name == null) {
714            name = source.getName();
715        }
716        name = findFreeName(parent, name);
717        String sourceId = source.getUUID();
718        String parentId = parent.getUUID();
719        State sourceState = transaction.getStateForRead(sourceId);
720        State parentState = transaction.getStateForRead(parentId);
721        String oldParentId = (String) sourceState.get(KEY_PARENT_ID);
722        Object[] parentAncestorIds = (Object[]) parentState.get(KEY_ANCESTOR_IDS);
723        LinkedList<String> ancestorIds = new LinkedList<>();
724        if (parentAncestorIds != null) {
725            for (Object id : parentAncestorIds) {
726                ancestorIds.add((String) id);
727            }
728        }
729        ancestorIds.add(parentId);
730        if (oldParentId != null && !oldParentId.equals(parentId)) {
731            if (ancestorIds.contains(sourceId)) {
732                throw new DocumentExistsException(
733                        "Cannot copy a node under itself: " + parentId + " is under " + sourceId);
734
735            }
736            // checkNotUnder(parentId, sourceId, "copy");
737        }
738        // do the copy
739        Long pos = getNextPos(parentId);
740        boolean excludeSpecialChildren = true;
741        boolean excludeRegularChildren = false;
742        String copyId = //
743                copyRecurse(sourceId, parentId, ancestorIds, name, excludeSpecialChildren, excludeRegularChildren);
744        DBSDocumentState copyState = transaction.getStateForUpdate(copyId);
745        // version copy fixup
746        if (source.isVersion()) {
747            copyState.put(KEY_IS_VERSION, null);
748        }
749        // pos fixup
750        copyState.put(KEY_POS, pos);
751
752        // update read acls
753        transaction.updateTreeReadAcls(copyId);
754
755        return getDocument(copyState);
756    }
757
758    protected String copyRecurse(String sourceId, String parentId, LinkedList<String> ancestorIds, String name,
759            boolean excludeSpecialChildren, boolean excludeRegularChildren) {
760        String copyId = copy(sourceId, parentId, ancestorIds, name);
761        ancestorIds.addLast(copyId);
762        for (String childId : getChildrenIds(sourceId, excludeSpecialChildren, excludeRegularChildren)) {
763            // don't exclude regular children when recursing
764            copyRecurse(childId, copyId, ancestorIds, null, excludeSpecialChildren, false);
765        }
766        ancestorIds.removeLast();
767        return copyId;
768    }
769
770    /**
771     * Copy source under parent, and set its ancestors.
772     */
773    protected String copy(String sourceId, String parentId, List<String> ancestorIds, String name) {
774        DBSDocumentState copy = transaction.copy(sourceId);
775        copy.put(KEY_PARENT_ID, parentId);
776        copy.put(KEY_ANCESTOR_IDS, ancestorIds.toArray(new Object[ancestorIds.size()]));
777        if (name != null) {
778            copy.put(KEY_NAME, name);
779        }
780        copy.put(KEY_BASE_VERSION_ID, null);
781        copy.put(KEY_IS_CHECKED_IN, null);
782        if (parentId != null) {
783            // reset version
784            copy.put(KEY_MAJOR_VERSION, null);
785            copy.put(KEY_MINOR_VERSION, null);
786        }
787        if (TRUE.equals(copy.get(KEY_IS_RECORD))) {
788            // unset record on the copy
789            copy.put(KEY_IS_RECORD, null);
790            copy.put(KEY_RETAIN_UNTIL, null);
791            copy.put(KEY_HAS_LEGAL_HOLD, null);
792            DBSDocument doc = getDocument(copy);
793            notifyAfterCopy(doc);
794        }
795        return copy.getId();
796    }
797
798    protected static final Pattern dotDigitsPattern = Pattern.compile("(.*)\\.[0-9]+$");
799
800    protected String findFreeName(Document parent, String name) {
801        if (hasChild(parent.getUUID(), name)) {
802            Matcher m = dotDigitsPattern.matcher(name);
803            if (m.matches()) {
804                // remove trailing dot and digits
805                name = m.group(1);
806            }
807            // add dot + unique digits
808            name += "." + System.currentTimeMillis();
809        }
810        return name;
811    }
812
813    /** Checks that we don't move/copy under ourselves. */
814    protected void checkNotUnder(String parentId, String id, String op) {
815        // TODO use ancestors
816        String pid = parentId;
817        do {
818            if (pid.equals(id)) {
819                throw new DocumentExistsException(
820                        "Cannot " + op + " a node under itself: " + parentId + " is under " + id);
821            }
822            State state = transaction.getStateForRead(pid);
823            if (state == null) {
824                // cannot happen
825                throw new NuxeoException("No parent: " + pid);
826            }
827            pid = (String) state.get(KEY_PARENT_ID);
828        } while (pid != null);
829    }
830
831    @Override
832    public Document move(Document source, Document parent, String name) {
833        String oldName = source.getName();
834        if (name == null) {
835            name = oldName;
836        }
837        String sourceId = source.getUUID();
838        String parentId = parent.getUUID();
839        DBSDocumentState sourceState = transaction.getStateForUpdate(sourceId);
840        String oldParentId = (String) sourceState.get(KEY_PARENT_ID);
841
842        // simple case of a rename
843        if (Objects.equals(oldParentId, parentId)) {
844            if (!oldName.equals(name)) {
845                if (hasChild(parentId, name)) {
846                    throw new DocumentExistsException("Destination name already exists: " + name);
847                }
848                // do the move
849                sourceState.put(KEY_NAME, name);
850                // no ancestors to change
851            }
852            return source;
853        } else {
854            // if not just a simple rename, flush
855            transaction.save();
856            if (hasChild(parentId, name)) {
857                throw new DocumentExistsException("Destination name already exists: " + name);
858            }
859        }
860
861        // prepare new ancestor ids
862        State parentState = transaction.getStateForRead(parentId);
863        Object[] parentAncestorIds = (Object[]) parentState.get(KEY_ANCESTOR_IDS);
864        List<String> ancestorIdsList = new ArrayList<>();
865        if (parentAncestorIds != null) {
866            for (Object id : parentAncestorIds) {
867                ancestorIdsList.add((String) id);
868            }
869        }
870        ancestorIdsList.add(parentId);
871        Object[] ancestorIds = ancestorIdsList.toArray(new Object[ancestorIdsList.size()]);
872
873        if (ancestorIdsList.contains(sourceId)) {
874            throw new DocumentExistsException("Cannot move a node under itself: " + parentId + " is under " + sourceId);
875        }
876
877        // do the move
878        sourceState.put(KEY_NAME, name);
879        sourceState.put(KEY_PARENT_ID, parentId);
880
881        // update ancestors on all sub-children
882        Object[] oldAncestorIds = (Object[]) sourceState.get(KEY_ANCESTOR_IDS);
883        int ndel = oldAncestorIds == null ? 0 : oldAncestorIds.length;
884        transaction.updateAncestors(sourceId, ndel, ancestorIds);
885
886        // update read acls
887        transaction.updateTreeReadAcls(sourceId);
888
889        return source;
890    }
891
892    /**
893     * Removes a document.
894     * <p>
895     * We also have to update everything impacted by "relations":
896     * <ul>
897     * <li>parent-child relations: delete all subchildren recursively,
898     * <li>proxy-target relations: if a proxy is removed, update the target's PROXY_IDS; and if a target is removed,
899     * raise an error if a proxy still exists for that target.
900     * </ul>
901     */
902    protected void remove(String rootId, NuxeoPrincipal principal) {
903        transaction.save();
904
905        State rootState = transaction.getStateForRead(rootId);
906        String versionSeriesId;
907        if (TRUE.equals(rootState.get(KEY_IS_VERSION))) {
908            versionSeriesId = (String) rootState.get(KEY_VERSION_SERIES_ID);
909        } else {
910            versionSeriesId = null;
911        }
912
913        // find all sub-docs
914        Set<String> removedIds = new HashSet<>();
915        Set<String> undeletableIds = new HashSet<>();
916        Set<String> targetIds = new HashSet<>();
917        Map<String, Object[]> targetProxies = new HashMap<>();
918        Calendar now = Calendar.getInstance();
919
920        Consumer<State> collector = state -> {
921            String id = (String) state.get(KEY_ID);
922            removedIds.add(id);
923            if (TRUE.equals(state.get(KEY_HAS_LEGAL_HOLD))) {
924                undeletableIds.add(id);
925            } else {
926                Calendar retainUntil = (Calendar) state.get(KEY_RETAIN_UNTIL);
927                if (retainUntil != null && now.before(retainUntil)) {
928                    undeletableIds.add(id);
929                }
930            }
931            if (TRUE.equals(state.get(KEY_IS_RETENTION_ACTIVE))) {
932                undeletableIds.add(id);
933            }
934            if (TRUE.equals(state.get(KEY_IS_PROXY))) {
935                String targetId = (String) state.get(KEY_PROXY_TARGET_ID);
936                targetIds.add(targetId);
937            }
938            Object[] proxyIds = (Object[]) state.get(KEY_PROXY_IDS);
939            if (proxyIds != null) {
940                targetProxies.put(id, proxyIds);
941            }
942        };
943        collector.accept(rootState); // add the root node too
944        try (Stream<State> states = transaction.getDescendants(rootId, KEYS_RETENTION_HOLD_AND_PROXIES, 0)) {
945            states.forEach(collector);
946        }
947
948        // if a subdocument is under retention / hold, removal fails
949        if (!undeletableIds.isEmpty()) {
950            // in tests we may want to delete everything
951            boolean allowDeleteUndeletable = Framework.isBooleanPropertyTrue(PROP_ALLOW_DELETE_UNDELETABLE_DOCUMENTS);
952            if (!allowDeleteUndeletable) {
953                if (undeletableIds.contains(rootId)) {
954                    throw new DocumentExistsException("Cannot remove " + rootId + ", it is under retention / hold");
955                } else {
956                    throw new DocumentExistsException("Cannot remove " + rootId + ", subdocument "
957                            + undeletableIds.iterator().next() + " is under retention / hold");
958                }
959            }
960        }
961
962        // if a proxy target is removed, check that all proxies to it
963        // are removed
964        for (Entry<String, Object[]> en : targetProxies.entrySet()) {
965            String targetId = en.getKey();
966            for (Object proxyId : en.getValue()) {
967                if (!removedIds.contains(proxyId)) {
968                    throw new DocumentExistsException("Cannot remove " + rootId + ", subdocument " + targetId
969                            + " is the target of proxy " + proxyId);
970                }
971            }
972        }
973
974        // remove root doc
975        transaction.removeStates(Collections.singleton(rootId));
976        // Check that the ids to remove is not only the root id
977        if (removedIds.size() > 1) {
978            String nxql = String.format("SELECT * FROM Document, Relation WHERE ecm:ancestorId = '%s'", rootId);
979            BulkCommand command = new BulkCommand.Builder(ACTION_NAME, nxql, principal.getName())
980                                                 .repository(getRepositoryName())
981                                                 .build();
982            Framework.getService(BulkService.class).submit(command);
983        }
984
985        // fix proxies back-pointers on proxy targets
986        for (String targetId : targetIds) {
987            if (removedIds.contains(targetId)) {
988                // the target was also removed, skip
989                continue;
990            }
991            DBSDocumentState target = transaction.getStateForUpdate(targetId);
992            if (target != null) {
993                removeBackProxyIds(target, removedIds);
994            }
995        }
996
997        // recompute version series if needed
998        // only done for root of deletion as versions are not fileable
999        if (versionSeriesId != null) {
1000            recomputeVersionSeries(versionSeriesId);
1001        }
1002    }
1003
1004    @Override
1005    public Document createProxy(Document doc, Document folder) {
1006        if (doc == null) {
1007            throw new NullPointerException();
1008        }
1009        String id = doc.getUUID();
1010        String targetId;
1011        String versionSeriesId;
1012        if (doc.isVersion()) {
1013            targetId = id;
1014            versionSeriesId = doc.getVersionSeriesId();
1015        } else if (doc.isProxy()) {
1016            // copy the proxy
1017            State state = transaction.getStateForRead(id);
1018            targetId = (String) state.get(KEY_PROXY_TARGET_ID);
1019            versionSeriesId = (String) state.get(KEY_PROXY_VERSION_SERIES_ID);
1020        } else {
1021            // working copy (live document)
1022            targetId = id;
1023            versionSeriesId = targetId;
1024        }
1025
1026        String parentId = folder.getUUID();
1027        String name = findFreeName(folder, doc.getName());
1028        Long pos = parentId == null ? null : getNextPos(parentId);
1029
1030        DBSDocumentState docState = addProxyState(null, parentId, name, pos, targetId, versionSeriesId);
1031        return getDocument(docState);
1032    }
1033
1034    protected DBSDocumentState addProxyState(String id, String parentId, String name, Long pos, String targetId,
1035            String versionSeriesId) {
1036        DBSDocumentState target = transaction.getStateForUpdate(targetId);
1037        String typeName = (String) target.get(KEY_PRIMARY_TYPE);
1038
1039        DBSDocumentState proxy = transaction.createChild(id, parentId, name, pos, typeName);
1040        String proxyId = proxy.getId();
1041        proxy.put(KEY_IS_PROXY, TRUE);
1042        proxy.put(KEY_PROXY_TARGET_ID, targetId);
1043        proxy.put(KEY_PROXY_VERSION_SERIES_ID, versionSeriesId);
1044        if (changeTokenEnabled) {
1045            proxy.put(KEY_SYS_CHANGE_TOKEN, INITIAL_SYS_CHANGE_TOKEN);
1046            proxy.put(KEY_CHANGE_TOKEN, INITIAL_CHANGE_TOKEN);
1047        }
1048
1049        // copy target state to proxy
1050        transaction.updateProxy(target, proxyId);
1051
1052        // add back-reference to proxy on target
1053        addBackProxyId(target, proxyId);
1054
1055        return transaction.getStateForUpdate(proxyId);
1056    }
1057
1058    protected void addBackProxyId(DBSDocumentState docState, String id) {
1059        Object[] proxyIds = (Object[]) docState.get(KEY_PROXY_IDS);
1060        Object[] newProxyIds;
1061        if (proxyIds == null) {
1062            newProxyIds = new Object[] { id };
1063        } else {
1064            newProxyIds = new Object[proxyIds.length + 1];
1065            System.arraycopy(proxyIds, 0, newProxyIds, 0, proxyIds.length);
1066            newProxyIds[proxyIds.length] = id;
1067        }
1068        docState.put(KEY_PROXY_IDS, newProxyIds);
1069    }
1070
1071    protected void removeBackProxyId(DBSDocumentState docState, String id) {
1072        removeBackProxyIds(docState, Collections.singleton(id));
1073    }
1074
1075    protected void removeBackProxyIds(DBSDocumentState docState, Set<String> ids) {
1076        Object[] proxyIds = (Object[]) docState.get(KEY_PROXY_IDS);
1077        if (proxyIds == null) {
1078            return;
1079        }
1080        List<Object> keepIds = new ArrayList<>(proxyIds.length);
1081        for (Object pid : proxyIds) {
1082            if (!ids.contains(pid)) {
1083                keepIds.add(pid);
1084            }
1085        }
1086        Object[] newProxyIds = keepIds.isEmpty() ? null : keepIds.toArray(new Object[keepIds.size()]);
1087        docState.put(KEY_PROXY_IDS, newProxyIds);
1088    }
1089
1090    @Override
1091    public List<Document> getProxies(Document doc, Document folder) {
1092        List<DBSDocumentState> docStates;
1093        String docId = doc.getUUID();
1094        if (doc.isVersion()) {
1095            docStates = transaction.getKeyValuedStates(KEY_PROXY_TARGET_ID, docId);
1096        } else {
1097            String versionSeriesId;
1098            if (doc.isProxy()) {
1099                State state = transaction.getStateForRead(docId);
1100                versionSeriesId = (String) state.get(KEY_PROXY_VERSION_SERIES_ID);
1101            } else {
1102                versionSeriesId = docId;
1103            }
1104            docStates = transaction.getKeyValuedStates(KEY_PROXY_VERSION_SERIES_ID, versionSeriesId);
1105        }
1106
1107        String parentId = folder == null ? null : folder.getUUID();
1108        List<Document> documents = new ArrayList<>(docStates.size());
1109        for (DBSDocumentState docState : docStates) {
1110            // filter by parent
1111            if (parentId != null && !parentId.equals(docState.getParentId())) {
1112                continue;
1113            }
1114            documents.add(getDocument(docState));
1115        }
1116        return documents;
1117    }
1118
1119    @Override
1120    public List<Document> getProxies(Document doc) {
1121        State state = transaction.getStateForRead(doc.getUUID());
1122        Object[] proxyIds = (Object[]) state.get(KEY_PROXY_IDS);
1123        if (proxyIds != null) {
1124            List<String> ids = Arrays.stream(proxyIds).map(String::valueOf).collect(Collectors.toList());
1125            return getDocuments(ids);
1126        }
1127        return Collections.emptyList();
1128    }
1129
1130    @Override
1131    public void setProxyTarget(Document proxy, Document target) {
1132        String proxyId = proxy.getUUID();
1133        String targetId = target.getUUID();
1134        DBSDocumentState proxyState = transaction.getStateForUpdate(proxyId);
1135        String oldTargetId = (String) proxyState.get(KEY_PROXY_TARGET_ID);
1136
1137        // update old target's back-pointers: remove proxy id
1138        DBSDocumentState oldTargetState = transaction.getStateForUpdate(oldTargetId);
1139        removeBackProxyId(oldTargetState, proxyId);
1140        // update new target's back-pointers: add proxy id
1141        DBSDocumentState targetState = transaction.getStateForUpdate(targetId);
1142        addBackProxyId(targetState, proxyId);
1143        // set new target
1144        proxyState.put(KEY_PROXY_TARGET_ID, targetId);
1145    }
1146
1147    @Override
1148    public Document importDocument(String id, Document parent, String name, String typeName,
1149            Map<String, Serializable> properties) {
1150        String parentId = parent == null ? null : parent.getUUID();
1151        boolean isProxy = typeName.equals(CoreSession.IMPORT_PROXY_TYPE);
1152        Map<String, Serializable> props = new HashMap<>();
1153        Long pos = null; // TODO pos
1154        DBSDocumentState docState;
1155        if (isProxy) {
1156            // check that target exists and find its typeName
1157            String targetId = (String) properties.get(CoreSession.IMPORT_PROXY_TARGET_ID);
1158            if (targetId == null) {
1159                throw new NuxeoException("Cannot import proxy " + id + " with null target");
1160            }
1161            State targetState = transaction.getStateForRead(targetId);
1162            if (targetState == null) {
1163                throw new DocumentNotFoundException("Cannot import proxy " + id + " with missing target " + targetId);
1164            }
1165            String versionSeriesId = (String) properties.get(CoreSession.IMPORT_PROXY_VERSIONABLE_ID);
1166            docState = addProxyState(id, parentId, name, pos, targetId, versionSeriesId);
1167        } else {
1168            // version & live document
1169            props.put(KEY_LIFECYCLE_POLICY, properties.get(CoreSession.IMPORT_LIFECYCLE_POLICY));
1170            props.put(KEY_LIFECYCLE_STATE, properties.get(CoreSession.IMPORT_LIFECYCLE_STATE));
1171
1172            Serializable importLockOwnerProp = properties.get(CoreSession.IMPORT_LOCK_OWNER);
1173            if (importLockOwnerProp != null) {
1174                props.put(KEY_LOCK_OWNER, importLockOwnerProp);
1175            }
1176            Serializable importLockCreatedProp = properties.get(CoreSession.IMPORT_LOCK_CREATED);
1177            if (importLockCreatedProp != null) {
1178                props.put(KEY_LOCK_CREATED, importLockCreatedProp);
1179            }
1180
1181            Boolean isRecord = trueOrNull(properties.get(CoreSession.IMPORT_IS_RECORD));
1182            props.put(KEY_IS_RECORD, isRecord);
1183            if (TRUE.equals(isRecord)) {
1184                Calendar retainUntil = (Calendar) properties.get(CoreSession.IMPORT_RETAIN_UNTIL);
1185                if (retainUntil != null) {
1186                    props.put(KEY_RETAIN_UNTIL, retainUntil);
1187                }
1188                Boolean hasLegalHold = trueOrNull(properties.get(CoreSession.IMPORT_HAS_LEGAL_HOLD));
1189                props.put(KEY_HAS_LEGAL_HOLD, hasLegalHold);
1190            }
1191
1192            Serializable isRetentionActiveProp = properties.get(CoreSession.IMPORT_IS_RETENTION_ACTIVE);
1193            if (TRUE.equals(isRetentionActiveProp)) {
1194                props.put(KEY_IS_RETENTION_ACTIVE, TRUE);
1195            }
1196
1197            props.put(KEY_MAJOR_VERSION, properties.get(CoreSession.IMPORT_VERSION_MAJOR));
1198            props.put(KEY_MINOR_VERSION, properties.get(CoreSession.IMPORT_VERSION_MINOR));
1199            Boolean isVersion = trueOrNull(properties.get(CoreSession.IMPORT_IS_VERSION));
1200            props.put(KEY_IS_VERSION, isVersion);
1201            if (TRUE.equals(isVersion)) {
1202                // version
1203                props.put(KEY_VERSION_SERIES_ID, properties.get(CoreSession.IMPORT_VERSION_VERSIONABLE_ID));
1204                props.put(KEY_VERSION_CREATED, properties.get(CoreSession.IMPORT_VERSION_CREATED));
1205                props.put(KEY_VERSION_LABEL, properties.get(CoreSession.IMPORT_VERSION_LABEL));
1206                props.put(KEY_VERSION_DESCRIPTION, properties.get(CoreSession.IMPORT_VERSION_DESCRIPTION));
1207                // TODO maybe these should be recomputed at end of import:
1208                props.put(KEY_IS_LATEST_VERSION, trueOrNull(properties.get(CoreSession.IMPORT_VERSION_IS_LATEST)));
1209                props.put(KEY_IS_LATEST_MAJOR_VERSION,
1210                        trueOrNull(properties.get(CoreSession.IMPORT_VERSION_IS_LATEST_MAJOR)));
1211            } else {
1212                // live document
1213                props.put(KEY_BASE_VERSION_ID, properties.get(CoreSession.IMPORT_BASE_VERSION_ID));
1214                props.put(KEY_IS_CHECKED_IN, trueOrNull(properties.get(CoreSession.IMPORT_CHECKED_IN)));
1215            }
1216            docState = createChildState(id, parentId, name, pos, typeName);
1217        }
1218        for (Entry<String, Serializable> entry : props.entrySet()) {
1219            docState.put(entry.getKey(), entry.getValue());
1220        }
1221        return getDocument(docState, false); // not readonly
1222    }
1223
1224    protected static Boolean trueOrNull(Object value) {
1225        return TRUE.equals(value) ? TRUE : null;
1226    }
1227
1228    @Override
1229    public Document getVersion(String versionSeriesId, VersionModel versionModel) {
1230        DBSDocumentState docState = getVersionByLabel(versionSeriesId, versionModel.getLabel());
1231        if (docState == null) {
1232            return null;
1233        }
1234        versionModel.setDescription((String) docState.get(KEY_VERSION_DESCRIPTION));
1235        versionModel.setCreated((Calendar) docState.get(KEY_VERSION_CREATED));
1236        return getDocument(docState);
1237    }
1238
1239    protected DBSDocumentState getVersionByLabel(String versionSeriesId, String label) {
1240        List<DBSDocumentState> docStates = transaction.getKeyValuedStates(KEY_VERSION_SERIES_ID, versionSeriesId,
1241                KEY_IS_VERSION, TRUE);
1242        for (DBSDocumentState docState : docStates) {
1243            if (label.equals(docState.get(KEY_VERSION_LABEL))) {
1244                return docState;
1245            }
1246        }
1247        return null;
1248    }
1249
1250    protected List<String> getVersionsIds(String versionSeriesId) {
1251        // order by creation date
1252        List<DBSDocumentState> docStates = transaction.getKeyValuedStates(KEY_VERSION_SERIES_ID, versionSeriesId,
1253                KEY_IS_VERSION, TRUE);
1254        docStates.sort(VERSION_CREATED_COMPARATOR);
1255        List<String> ids = new ArrayList<>(docStates.size());
1256        for (DBSDocumentState docState : docStates) {
1257            ids.add(docState.getId());
1258        }
1259        return ids;
1260    }
1261
1262    protected Document getLastVersion(String versionSeriesId) {
1263        List<DBSDocumentState> docStates = transaction.getKeyValuedStates(KEY_VERSION_SERIES_ID, versionSeriesId,
1264                KEY_IS_VERSION, TRUE);
1265        // find latest one
1266        Calendar latest = null;
1267        DBSDocumentState latestState = null;
1268        for (DBSDocumentState docState : docStates) {
1269            Calendar created = (Calendar) docState.get(KEY_VERSION_CREATED);
1270            if (latest == null || created.compareTo(latest) > 0) {
1271                latest = created;
1272                latestState = docState;
1273            }
1274        }
1275        return latestState == null ? null : getDocument(latestState);
1276    }
1277
1278    private static final Comparator<DBSDocumentState> VERSION_CREATED_COMPARATOR = (s1, s2) -> {
1279        Calendar c1 = (Calendar) s1.get(KEY_VERSION_CREATED);
1280        Calendar c2 = (Calendar) s2.get(KEY_VERSION_CREATED);
1281        if (c1 == null && c2 == null) {
1282            // coherent sort
1283            return s1.hashCode() - s2.hashCode();
1284        }
1285        if (c1 == null) {
1286            return 1;
1287        }
1288        if (c2 == null) {
1289            return -1;
1290        }
1291        return c1.compareTo(c2);
1292    };
1293
1294    private static final Comparator<DBSDocumentState> POS_COMPARATOR = (s1, s2) -> {
1295        Long p1 = (Long) s1.get(KEY_POS);
1296        Long p2 = (Long) s2.get(KEY_POS);
1297        if (p1 == null && p2 == null) {
1298            // coherent sort
1299            return s1.hashCode() - s2.hashCode();
1300        }
1301        if (p1 == null) {
1302            return 1;
1303        }
1304        if (p2 == null) {
1305            return -1;
1306        }
1307        return p1.compareTo(p2);
1308    };
1309
1310    @Override
1311    public void updateReadACLs(Collection<String> docIds) {
1312        transaction.updateReadACLs(docIds);
1313    }
1314
1315    @Override
1316    public boolean isNegativeAclAllowed() {
1317        return false;
1318    }
1319
1320    @Override
1321    public ACP getACP(Document doc) {
1322        State state = transaction.getStateForRead(doc.getUUID());
1323        return memToAcp(state.get(KEY_ACP));
1324    }
1325
1326    @Override
1327    public void setACP(Document doc, ACP acp, boolean overwrite) {
1328        if (!overwrite && acp == null) {
1329            return;
1330        }
1331        checkNegativeAcl(acp);
1332        if (!overwrite) {
1333            acp = updateACP(getACP(doc), acp);
1334        }
1335        String id = doc.getUUID();
1336        DBSDocumentState docState = transaction.getStateForUpdate(id);
1337        docState.put(KEY_ACP, acpToMem(acp));
1338
1339        // update read acls
1340        transaction.updateTreeReadAcls(id);
1341    }
1342
1343    public static Serializable acpToMem(ACP acp) {
1344        if (acp == null) {
1345            return null;
1346        }
1347        ACL[] acls = acp.getACLs();
1348        if (acls.length == 0) {
1349            return null;
1350        }
1351        List<Serializable> aclList = new ArrayList<>(acls.length);
1352        for (ACL acl : acls) {
1353            String name = acl.getName();
1354            if (name.equals(ACL.INHERITED_ACL)) {
1355                continue;
1356            }
1357            ACE[] aces = acl.getACEs();
1358            List<Serializable> aceList = new ArrayList<>(aces.length);
1359            for (ACE ace : aces) {
1360                State aceMap = new State(6);
1361                aceMap.put(KEY_ACE_USER, ace.getUsername());
1362                aceMap.put(KEY_ACE_PERMISSION, ace.getPermission());
1363                aceMap.put(KEY_ACE_GRANT, ace.isGranted());
1364                String creator = ace.getCreator();
1365                if (creator != null) {
1366                    aceMap.put(KEY_ACE_CREATOR, creator);
1367                }
1368                Calendar begin = ace.getBegin();
1369                if (begin != null) {
1370                    aceMap.put(KEY_ACE_BEGIN, begin);
1371                }
1372                Calendar end = ace.getEnd();
1373                if (end != null) {
1374                    aceMap.put(KEY_ACE_END, end);
1375                }
1376                Long status = ace.getLongStatus();
1377                if (status != null) {
1378                    aceMap.put(KEY_ACE_STATUS, status);
1379                }
1380                aceList.add(aceMap);
1381            }
1382            if (aceList.isEmpty()) {
1383                continue;
1384            }
1385            State aclMap = new State(2);
1386            aclMap.put(KEY_ACL_NAME, name);
1387            aclMap.put(KEY_ACL, (Serializable) aceList);
1388            aclList.add(aclMap);
1389        }
1390        return (Serializable) aclList;
1391    }
1392
1393    protected static ACP memToAcp(Serializable acpSer) {
1394        if (acpSer == null) {
1395            return null;
1396        }
1397        @SuppressWarnings("unchecked")
1398        List<Serializable> aclList = (List<Serializable>) acpSer;
1399        ACP acp = new ACPImpl();
1400        for (Serializable aclSer : aclList) {
1401            State aclMap = (State) aclSer;
1402            String name = (String) aclMap.get(KEY_ACL_NAME);
1403            @SuppressWarnings("unchecked")
1404            List<Serializable> aceList = (List<Serializable>) aclMap.get(KEY_ACL);
1405            if (aceList == null) {
1406                continue;
1407            }
1408            ACL acl = new ACLImpl(name);
1409            for (Serializable aceSer : aceList) {
1410                State aceMap = (State) aceSer;
1411                String username = (String) aceMap.get(KEY_ACE_USER);
1412                String permission = (String) aceMap.get(KEY_ACE_PERMISSION);
1413                boolean granted = (boolean) aceMap.get(KEY_ACE_GRANT);
1414                String creator = (String) aceMap.get(KEY_ACE_CREATOR);
1415                Calendar begin = (Calendar) aceMap.get(KEY_ACE_BEGIN);
1416                Calendar end = (Calendar) aceMap.get(KEY_ACE_END);
1417                // status not read, ACE always computes it on read
1418                ACE ace = ACE.builder(username, permission)
1419                             .isGranted(granted)
1420                             .creator(creator)
1421                             .begin(begin)
1422                             .end(end)
1423                             .build();
1424                acl.add(ace);
1425            }
1426            acp.addACL(acl);
1427        }
1428        return acp;
1429    }
1430
1431    @Override
1432    public boolean isFulltextStoredInBlob() {
1433        return fulltextStoredInBlob;
1434    }
1435
1436    @Override
1437    public Map<String, String> getBinaryFulltext(String id) {
1438        State state = transaction.getStateForRead(id);
1439        String fulltext = (String) state.get(KEY_FULLTEXT_BINARY);
1440        if (fulltextStoredInBlob && fulltext != null) {
1441            DBSDocument doc = getDocument(id);
1442            if (doc == null) {
1443                // could not find doc (shouldn't happen)
1444                fulltext = null;
1445            } else {
1446                // fulltext is actually the blob key
1447                // now retrieve the actual fulltext from the blob content
1448                try {
1449                    BlobInfo blobInfo = new BlobInfo();
1450                    blobInfo.key = fulltext;
1451                    String xpath = BaseDocument.FULLTEXT_BINARYTEXT_PROP;
1452                    Blob blob = getDocumentBlobManager().readBlob(blobInfo, doc, xpath);
1453                    fulltext = blob.getString();
1454                } catch (IOException e) {
1455                    throw new PropertyException("Cannot read fulltext blob for doc: " + id, e);
1456                }
1457            }
1458        }
1459        return Collections.singletonMap(BINARY_FULLTEXT_MAIN_KEY, fulltext);
1460    }
1461
1462    @Override
1463    public void removeDocument(String id) {
1464        transaction.save();
1465
1466        DBSDocumentState docState = transaction.getStateForUpdate(id);
1467
1468        Calendar retainUntil = (Calendar) docState.get(KEY_RETAIN_UNTIL);
1469        if (retainUntil != null && Calendar.getInstance().before(retainUntil)) {
1470            throw new DocumentExistsException("Cannot remove " + id + ", it is under retention / hold");
1471        }
1472        if (TRUE.equals(docState.get(KEY_HAS_LEGAL_HOLD))) {
1473            throw new DocumentExistsException("Cannot remove " + id + ", it is under retention / hold");
1474        }
1475        if (TRUE.equals(docState.get(KEY_IS_RETENTION_ACTIVE))) {
1476            throw new DocumentExistsException("Cannot remove " + id + ", it is under active retention");
1477        }
1478
1479        // notify blob manager before removal
1480        try {
1481            DBSDocument doc = getDocument(docState);
1482            getDocumentBlobManager().notifyBeforeRemove(doc);
1483        } catch (DocumentNotFoundException e) {
1484            // unknown type in db or null proxy target
1485            // ignore blob manager notification
1486        }
1487
1488        // remove doc
1489        transaction.removeStates(Collections.singleton(id));
1490
1491    }
1492
1493    @Override
1494    public PartialList<Document> query(String query, String queryType, QueryFilter queryFilter, long countUpTo) {
1495        // query
1496        PartialList<String> pl = doQuery(query, queryType, queryFilter, (int) countUpTo);
1497
1498        // get Documents in bulk
1499        List<Document> docs = getDocuments(pl);
1500
1501        return new PartialList<>(docs, pl.totalSize());
1502    }
1503
1504    @SuppressWarnings("resource") // Time.Context closed by stop()
1505    protected PartialList<String> doQuery(String query, String queryType, QueryFilter queryFilter, int countUpTo) {
1506        final Timer.Context timerContext = queryTimer.time();
1507        try {
1508            Mutable<String> idKeyHolder = new MutableObject<>();
1509            PartialList<Map<String, Serializable>> pl = doQueryAndFetch(query, queryType, queryFilter, false, countUpTo,
1510                    idKeyHolder);
1511            String idKey = idKeyHolder.getValue();
1512            List<String> ids = new ArrayList<>(pl.size());
1513            for (Map<String, Serializable> map : pl) {
1514                String id = (String) map.get(idKey);
1515                ids.add(id);
1516            }
1517            return new PartialList<>(ids, pl.totalSize());
1518        } finally {
1519            long duration = timerContext.stop();
1520            if (LOG_MIN_DURATION_NS >= 0 && duration > LOG_MIN_DURATION_NS) {
1521                String msg = String.format("duration_ms:\t%.2f\t%s %s\tquery\t%s", duration / 1000000.0, queryFilter,
1522                        countUpToAsString(countUpTo), query);
1523                if (log.isTraceEnabled()) {
1524                    log.info(msg, new Throwable("Slow query stack trace"));
1525                } else {
1526                    log.info(msg);
1527                }
1528            }
1529        }
1530    }
1531
1532    protected PartialList<Map<String, Serializable>> doQueryAndFetch(String query, String queryType,
1533            QueryFilter queryFilter, boolean distinctDocuments, int countUpTo, Mutable<String> idKeyHolder) {
1534        if ("NXTAG".equals(queryType)) {
1535            // for now don't try to implement tags
1536            // and return an empty list
1537            return new PartialList<>(Collections.<Map<String, Serializable>> emptyList(), 0);
1538        }
1539        if (!NXQL.NXQL.equals(queryType)) {
1540            throw new NuxeoException("No QueryMaker accepts query type: " + queryType);
1541        }
1542
1543        // transform the query according to the transformers defined by the
1544        // security policies
1545        SQLQuery sqlQuery = SQLQueryParser.parse(query);
1546        for (SQLQuery.Transformer transformer : queryFilter.getQueryTransformers()) {
1547            sqlQuery = transformer.transform(queryFilter.getPrincipal(), sqlQuery);
1548        }
1549
1550        SelectClause selectClause = sqlQuery.select;
1551        if (selectClause.isEmpty()) {
1552            // turned into SELECT ecm:uuid
1553            selectClause.add(new Reference(NXQL.ECM_UUID));
1554        }
1555        boolean selectStar = selectClause.count() == 1 && (selectClause.containsOperand(new Reference(NXQL.ECM_UUID)));
1556        if (selectStar) {
1557            distinctDocuments = true;
1558        } else if (selectClause.isDistinct()) {
1559            throw new QueryParseException("SELECT DISTINCT not supported on DBS");
1560        }
1561        if (idKeyHolder != null) {
1562            Operand operand = selectClause.operands().iterator().next();
1563            String idKey = operand instanceof Reference ? ((Reference) operand).name : NXQL.ECM_UUID;
1564            idKeyHolder.setValue(idKey);
1565        }
1566
1567        // Replace select clause for tags
1568        String ecmTag = selectClause.elements.keySet()
1569                                             .stream()
1570                                             .filter(k -> k.startsWith(NXQL.ECM_TAG))
1571                                             .findFirst()
1572                                             .orElse(null);
1573        String keyTag = null;
1574        if (ecmTag != null) {
1575            keyTag = FACETED_TAG + "/*1/" + FACETED_TAG_LABEL;
1576            selectClause.elements.replace(ecmTag, new Reference(keyTag));
1577        }
1578
1579        // Add useful select clauses, used for order by path
1580        selectClause.elements.putIfAbsent(NXQL.ECM_UUID, new Reference(NXQL.ECM_UUID));
1581        selectClause.elements.putIfAbsent(NXQL.ECM_PARENTID, new Reference(NXQL.ECM_PARENTID));
1582        selectClause.elements.putIfAbsent(NXQL.ECM_NAME, new Reference(NXQL.ECM_NAME));
1583
1584        QueryOptimizer optimizer = new DBSQueryOptimizer().withFacetFilter(queryFilter.getFacetFilter());
1585        sqlQuery = optimizer.optimize(sqlQuery);
1586        DBSExpressionEvaluator evaluator = new DBSExpressionEvaluator(this, sqlQuery, queryFilter.getPrincipals(),
1587                fulltextSearchDisabled);
1588
1589        int limit = (int) queryFilter.getLimit();
1590        int offset = (int) queryFilter.getOffset();
1591        if (offset < 0) {
1592            offset = 0;
1593        }
1594        if (limit < 0) {
1595            limit = 0;
1596        }
1597
1598        int repoLimit;
1599        int repoOffset;
1600        OrderByClause repoOrderByClause;
1601        OrderByClause orderByClause = sqlQuery.orderBy;
1602        boolean postFilter = isOrderByPath(orderByClause);
1603        if (postFilter) {
1604            // we have to merge ordering and batching between memory and
1605            // repository
1606            repoLimit = 0;
1607            repoOffset = 0;
1608            repoOrderByClause = null;
1609        } else {
1610            // fast case, we can use the repository query directly
1611            repoLimit = limit;
1612            repoOffset = offset;
1613            repoOrderByClause = orderByClause;
1614        }
1615
1616        // query the repository
1617        PartialList<Map<String, Serializable>> projections = transaction.queryAndFetch(evaluator, repoOrderByClause,
1618                distinctDocuments, repoLimit, repoOffset, countUpTo);
1619
1620        for (Map<String, Serializable> proj : projections) {
1621            if (proj.containsKey(keyTag)) {
1622                proj.put(ecmTag, proj.remove(keyTag));
1623            }
1624        }
1625
1626        if (postFilter) {
1627            // ORDER BY
1628            if (orderByClause != null) {
1629                doOrderBy(projections, orderByClause);
1630            }
1631            // LIMIT / OFFSET
1632            if (limit != 0) {
1633                int size = projections.size();
1634                int fromIndex = offset > size ? size : offset;
1635                int toIndex = fromIndex + limit > size ? size : fromIndex + limit;
1636                projections = projections.subList(fromIndex, toIndex);
1637            }
1638        }
1639
1640        return projections;
1641    }
1642
1643    /** Does an ORDER BY clause include ecm:path */
1644    protected boolean isOrderByPath(OrderByClause orderByClause) {
1645        if (orderByClause == null) {
1646            return false;
1647        }
1648        for (OrderByExpr ob : orderByClause.elements) {
1649            if (ob.reference.name.equals(NXQL.ECM_PATH)) {
1650                return true;
1651            }
1652        }
1653        return false;
1654    }
1655
1656    protected String getPath(Map<String, Serializable> projection) {
1657        String name = (String) projection.get(NXQL.ECM_NAME);
1658        String parentId = (String) projection.get(NXQL.ECM_PARENTID);
1659        State state;
1660        if (parentId == null || (state = transaction.getStateForRead(parentId)) == null) {
1661            if ("".equals(name)) {
1662                return "/"; // root
1663            } else {
1664                return name; // placeless, no slash
1665            }
1666        }
1667        LinkedList<String> list = new LinkedList<>();
1668        list.addFirst(name);
1669        for (;;) {
1670            name = (String) state.get(KEY_NAME);
1671            parentId = (String) state.get(KEY_PARENT_ID);
1672            list.addFirst(name);
1673            if (parentId == null || (state = transaction.getStateForRead(parentId)) == null) {
1674                return StringUtils.join(list, '/');
1675            }
1676        }
1677    }
1678
1679    protected void doOrderBy(List<Map<String, Serializable>> projections, OrderByClause orderByClause) {
1680        if (isOrderByPath(orderByClause)) {
1681            // add path info to do the sort
1682            for (Map<String, Serializable> projection : projections) {
1683                projection.put(ExpressionEvaluator.NXQL_ECM_PATH, getPath(projection));
1684            }
1685        }
1686        projections.sort(new OrderByComparator(orderByClause));
1687    }
1688
1689    public static class OrderByComparator implements Comparator<Map<String, Serializable>> {
1690
1691        protected final OrderByClause orderByClause;
1692
1693        public OrderByComparator(OrderByClause orderByClause) {
1694            // replace ecm:path with ecm:__path for evaluation
1695            // (we don't want to allow ecm:path to be usable anywhere else
1696            // and resolve to a null value)
1697            OrderByList obl = new OrderByList();
1698            for (OrderByExpr ob : orderByClause.elements) {
1699                if (ob.reference.name.equals(NXQL.ECM_PATH)) {
1700                    ob = new OrderByExpr(new Reference(ExpressionEvaluator.NXQL_ECM_PATH), ob.isDescending);
1701                }
1702                obl.add(ob);
1703            }
1704            this.orderByClause = new OrderByClause(obl);
1705        }
1706
1707        @Override
1708        @SuppressWarnings("unchecked")
1709        public int compare(Map<String, Serializable> map1, Map<String, Serializable> map2) {
1710            for (OrderByExpr ob : orderByClause.elements) {
1711                Reference ref = ob.reference;
1712                boolean desc = ob.isDescending;
1713                Object v1 = map1.get(ref.name);
1714                Object v2 = map2.get(ref.name);
1715                int cmp;
1716                if (v1 == null) {
1717                    cmp = v2 == null ? 0 : -1;
1718                } else if (v2 == null) {
1719                    cmp = 1;
1720                } else {
1721                    if (!(v1 instanceof Comparable)) {
1722                        throw new QueryParseException("Not a comparable: " + v1);
1723                    }
1724                    cmp = ((Comparable<Object>) v1).compareTo(v2);
1725                }
1726                if (desc) {
1727                    cmp = -cmp;
1728                }
1729                if (cmp != 0) {
1730                    return cmp;
1731                }
1732                // loop for lexicographical comparison
1733            }
1734            return 0;
1735        }
1736    }
1737
1738    @Override
1739    public IterableQueryResult queryAndFetch(String query, String queryType, QueryFilter queryFilter,
1740            boolean distinctDocuments, Object[] params) {
1741        final Timer.Context timerContext = queryTimer.time();
1742        try {
1743            PartialList<Map<String, Serializable>> pl = doQueryAndFetch(query, queryType, queryFilter,
1744                    distinctDocuments, -1, null);
1745            return new DBSQueryResult(pl);
1746        } finally {
1747            long duration = timerContext.stop();
1748            if (LOG_MIN_DURATION_NS >= 0 && duration > LOG_MIN_DURATION_NS) {
1749                String msg = String.format("duration_ms:\t%.2f\t%s\tqueryAndFetch\t%s", duration / 1000000.0,
1750                        queryFilter, query);
1751                if (log.isTraceEnabled()) {
1752                    log.info(msg, new Throwable("Slow query stack trace"));
1753                } else {
1754                    log.info(msg);
1755                }
1756            }
1757
1758        }
1759    }
1760
1761    @SuppressWarnings("resource") // Time.Context closed by stop()
1762    @Override
1763    public PartialList<Map<String, Serializable>> queryProjection(String query, String queryType,
1764            QueryFilter queryFilter, boolean distinctDocuments, long countUpTo, Object[] params) {
1765        final Timer.Context timerContext = queryTimer.time();
1766        try {
1767            return doQueryAndFetch(query, queryType, queryFilter, distinctDocuments, (int) countUpTo, null);
1768        } finally {
1769            long duration = timerContext.stop();
1770            if (LOG_MIN_DURATION_NS >= 0 && duration > LOG_MIN_DURATION_NS) {
1771                String msg = String.format("duration_ms:\t%.2f\t%s\tqueryProjection\t%s", duration / 1000000.0,
1772                        queryFilter, query);
1773                if (log.isTraceEnabled()) {
1774                    log.info(msg, new Throwable("Slow query stack trace"));
1775                } else {
1776                    log.info(msg);
1777                }
1778            }
1779
1780        }
1781    }
1782
1783    @Override
1784    public ScrollResult<String> scroll(String query, int batchSize, int keepAliveSeconds) {
1785        SQLQuery sqlQuery = SQLQueryParser.parse(query);
1786        SelectClause selectClause = sqlQuery.select;
1787        selectClause.add(new Reference(NXQL.ECM_UUID));
1788        sqlQuery = new DBSQueryOptimizer().optimize(sqlQuery);
1789        DBSExpressionEvaluator evaluator = new DBSExpressionEvaluator(this, sqlQuery, null, fulltextSearchDisabled);
1790        return transaction.scroll(evaluator, batchSize, keepAliveSeconds);
1791    }
1792
1793    @Override
1794    public ScrollResult<String> scroll(String query, QueryFilter queryFilter, int batchSize, int keepAliveSeconds) {
1795        SQLQuery sqlQuery = SQLQueryParser.parse(query);
1796        SelectClause selectClause = sqlQuery.select;
1797        selectClause.add(new Reference(NXQL.ECM_UUID));
1798        sqlQuery = new DBSQueryOptimizer().optimize(sqlQuery);
1799        for (SQLQuery.Transformer transformer : queryFilter.getQueryTransformers()) {
1800            sqlQuery = transformer.transform(queryFilter.getPrincipal(), sqlQuery);
1801        }
1802        DBSExpressionEvaluator evaluator = new DBSExpressionEvaluator(this, sqlQuery, queryFilter.getPrincipals(),
1803                fulltextSearchDisabled);
1804        return transaction.scroll(evaluator, batchSize, keepAliveSeconds);
1805    }
1806
1807    @Override
1808    public ScrollResult<String> scroll(String scrollId) {
1809        return transaction.scroll(scrollId);
1810    }
1811
1812    private String countUpToAsString(long countUpTo) {
1813        if (countUpTo > 0) {
1814            return String.format("count total results up to %d", countUpTo);
1815        }
1816        return countUpTo == -1 ? "count total results UNLIMITED" : "";
1817    }
1818
1819    protected static class DBSQueryResult implements IterableQueryResult, Iterator<Map<String, Serializable>> {
1820
1821        boolean closed;
1822
1823        protected List<Map<String, Serializable>> maps;
1824
1825        protected long totalSize;
1826
1827        protected long pos;
1828
1829        protected DBSQueryResult(PartialList<Map<String, Serializable>> pl) {
1830            this.maps = pl;
1831            this.totalSize = pl.totalSize();
1832        }
1833
1834        @Override
1835        public Iterator<Map<String, Serializable>> iterator() {
1836            return this;
1837        }
1838
1839        @Override
1840        public void close() {
1841            closed = true;
1842            pos = -1;
1843        }
1844
1845        @Override
1846        public boolean isLife() {
1847            return !closed;
1848        }
1849
1850        @Override
1851        public boolean mustBeClosed() {
1852            return false; // holds no resources
1853        }
1854
1855        @Override
1856        public long size() {
1857            return totalSize;
1858        }
1859
1860        @Override
1861        public long pos() {
1862            return pos;
1863        }
1864
1865        @Override
1866        public void skipTo(long pos) {
1867            if (pos < 0) {
1868                pos = 0;
1869            } else if (pos > totalSize) {
1870                pos = totalSize;
1871            }
1872            this.pos = pos;
1873        }
1874
1875        @Override
1876        public boolean hasNext() {
1877            return pos < totalSize;
1878        }
1879
1880        @Override
1881        public Map<String, Serializable> next() {
1882            if (closed || pos == totalSize) {
1883                throw new NoSuchElementException();
1884            }
1885            Map<String, Serializable> map = maps.get((int) pos);
1886            pos++;
1887            return map;
1888        }
1889
1890        @Override
1891        public void remove() {
1892            throw new UnsupportedOperationException();
1893        }
1894    }
1895
1896    public static String convToInternal(String name) {
1897        switch (name) {
1898        case NXQL.ECM_UUID:
1899            return KEY_ID;
1900        case NXQL.ECM_NAME:
1901            return KEY_NAME;
1902        case NXQL.ECM_POS:
1903            return KEY_POS;
1904        case NXQL.ECM_PARENTID:
1905            return KEY_PARENT_ID;
1906        case NXQL.ECM_MIXINTYPE:
1907            return KEY_MIXIN_TYPES;
1908        case NXQL.ECM_PRIMARYTYPE:
1909            return KEY_PRIMARY_TYPE;
1910        case NXQL.ECM_ISPROXY:
1911            return KEY_IS_PROXY;
1912        case NXQL.ECM_ISVERSION:
1913        case NXQL.ECM_ISVERSION_OLD:
1914            return KEY_IS_VERSION;
1915        case NXQL.ECM_LIFECYCLESTATE:
1916            return KEY_LIFECYCLE_STATE;
1917        case NXQL.ECM_LOCK_OWNER:
1918            return KEY_LOCK_OWNER;
1919        case NXQL.ECM_LOCK_CREATED:
1920            return KEY_LOCK_CREATED;
1921        case NXQL.ECM_PROXY_TARGETID:
1922            return KEY_PROXY_TARGET_ID;
1923        case NXQL.ECM_PROXY_VERSIONABLEID:
1924            return KEY_PROXY_VERSION_SERIES_ID;
1925        case NXQL.ECM_ISCHECKEDIN:
1926            return KEY_IS_CHECKED_IN;
1927        case NXQL.ECM_ISLATESTVERSION:
1928            return KEY_IS_LATEST_VERSION;
1929        case NXQL.ECM_ISLATESTMAJORVERSION:
1930            return KEY_IS_LATEST_MAJOR_VERSION;
1931        case NXQL.ECM_VERSIONLABEL:
1932            return KEY_VERSION_LABEL;
1933        case NXQL.ECM_VERSIONCREATED:
1934            return KEY_VERSION_CREATED;
1935        case NXQL.ECM_VERSIONDESCRIPTION:
1936            return KEY_VERSION_DESCRIPTION;
1937        case NXQL.ECM_VERSION_VERSIONABLEID:
1938            return KEY_VERSION_SERIES_ID;
1939        case NXQL.ECM_ISRECORD:
1940            return KEY_IS_RECORD;
1941        case NXQL.ECM_RETAINUNTIL:
1942            return KEY_RETAIN_UNTIL;
1943        case NXQL.ECM_HASLEGALHOLD:
1944            return KEY_HAS_LEGAL_HOLD;
1945        case NXQL.ECM_ANCESTORID:
1946        case ExpressionEvaluator.NXQL_ECM_ANCESTOR_IDS:
1947            return KEY_ANCESTOR_IDS;
1948        case ExpressionEvaluator.NXQL_ECM_PATH:
1949            return KEY_PATH_INTERNAL;
1950        case ExpressionEvaluator.NXQL_ECM_READ_ACL:
1951            return KEY_READ_ACL;
1952        case NXQL.ECM_FULLTEXT_JOBID:
1953            return KEY_FULLTEXT_JOBID;
1954        case NXQL.ECM_FULLTEXT_SCORE:
1955            return KEY_FULLTEXT_SCORE;
1956        case ExpressionEvaluator.NXQL_ECM_FULLTEXT_SIMPLE:
1957            return KEY_FULLTEXT_SIMPLE;
1958        case ExpressionEvaluator.NXQL_ECM_FULLTEXT_BINARY:
1959            return KEY_FULLTEXT_BINARY;
1960        case NXQL.ECM_ACL:
1961            return KEY_ACP;
1962        case DBSDocument.PROP_UID_MAJOR_VERSION:
1963        case DBSDocument.PROP_MAJOR_VERSION:
1964            return DBSDocument.KEY_MAJOR_VERSION;
1965        case DBSDocument.PROP_UID_MINOR_VERSION:
1966        case DBSDocument.PROP_MINOR_VERSION:
1967            return DBSDocument.KEY_MINOR_VERSION;
1968        case NXQL.ECM_ISTRASHED:
1969            return KEY_IS_TRASHED;
1970        case NXQL.ECM_FULLTEXT:
1971            throw new UnsupportedOperationException(name);
1972        }
1973        throw new QueryParseException("No such property: " + name);
1974    }
1975
1976    public static String convToInternalAce(String name) {
1977        switch (name) {
1978        case NXQL.ECM_ACL_NAME:
1979            return KEY_ACL_NAME;
1980        case NXQL.ECM_ACL_PRINCIPAL:
1981            return KEY_ACE_USER;
1982        case NXQL.ECM_ACL_PERMISSION:
1983            return KEY_ACE_PERMISSION;
1984        case NXQL.ECM_ACL_GRANT:
1985            return KEY_ACE_GRANT;
1986        case NXQL.ECM_ACL_CREATOR:
1987            return KEY_ACE_CREATOR;
1988        case NXQL.ECM_ACL_BEGIN:
1989            return KEY_ACE_BEGIN;
1990        case NXQL.ECM_ACL_END:
1991            return KEY_ACE_END;
1992        case NXQL.ECM_ACL_STATUS:
1993            return KEY_ACE_STATUS;
1994        }
1995        return null;
1996    }
1997
1998    public static String convToNXQL(String name) {
1999        switch (name) {
2000        case KEY_ID:
2001            return NXQL.ECM_UUID;
2002        case KEY_NAME:
2003            return NXQL.ECM_NAME;
2004        case KEY_POS:
2005            return NXQL.ECM_POS;
2006        case KEY_PARENT_ID:
2007            return NXQL.ECM_PARENTID;
2008        case KEY_MIXIN_TYPES:
2009            return NXQL.ECM_MIXINTYPE;
2010        case KEY_PRIMARY_TYPE:
2011            return NXQL.ECM_PRIMARYTYPE;
2012        case KEY_IS_PROXY:
2013            return NXQL.ECM_ISPROXY;
2014        case KEY_IS_VERSION:
2015            return NXQL.ECM_ISVERSION;
2016        case KEY_LIFECYCLE_STATE:
2017            return NXQL.ECM_LIFECYCLESTATE;
2018        case KEY_LOCK_OWNER:
2019            return NXQL.ECM_LOCK_OWNER;
2020        case KEY_LOCK_CREATED:
2021            return NXQL.ECM_LOCK_CREATED;
2022        case KEY_PROXY_TARGET_ID:
2023            return NXQL.ECM_PROXY_TARGETID;
2024        case KEY_PROXY_VERSION_SERIES_ID:
2025            return NXQL.ECM_PROXY_VERSIONABLEID;
2026        case KEY_IS_CHECKED_IN:
2027            return NXQL.ECM_ISCHECKEDIN;
2028        case KEY_IS_LATEST_VERSION:
2029            return NXQL.ECM_ISLATESTVERSION;
2030        case KEY_IS_LATEST_MAJOR_VERSION:
2031            return NXQL.ECM_ISLATESTMAJORVERSION;
2032        case KEY_VERSION_LABEL:
2033            return NXQL.ECM_VERSIONLABEL;
2034        case KEY_VERSION_CREATED:
2035            return NXQL.ECM_VERSIONCREATED;
2036        case KEY_VERSION_DESCRIPTION:
2037            return NXQL.ECM_VERSIONDESCRIPTION;
2038        case KEY_VERSION_SERIES_ID:
2039            return NXQL.ECM_VERSION_VERSIONABLEID;
2040        case KEY_IS_RECORD:
2041            return NXQL.ECM_ISRECORD;
2042        case KEY_RETAIN_UNTIL:
2043            return NXQL.ECM_RETAINUNTIL;
2044        case KEY_HAS_LEGAL_HOLD:
2045            return NXQL.ECM_HASLEGALHOLD;
2046        case KEY_MAJOR_VERSION:
2047            return "major_version"; // TODO XXX constant
2048        case KEY_MINOR_VERSION:
2049            return "minor_version";
2050        case KEY_FULLTEXT_SCORE:
2051            return NXQL.ECM_FULLTEXT_SCORE;
2052        case KEY_IS_TRASHED:
2053            return NXQL.ECM_ISTRASHED;
2054        case KEY_LIFECYCLE_POLICY:
2055        case KEY_ACP:
2056        case KEY_ANCESTOR_IDS:
2057        case KEY_BASE_VERSION_ID:
2058        case KEY_READ_ACL:
2059        case KEY_FULLTEXT_SIMPLE:
2060        case KEY_FULLTEXT_BINARY:
2061        case KEY_FULLTEXT_JOBID:
2062        case KEY_PATH_INTERNAL:
2063            return null;
2064        }
2065        throw new QueryParseException("No such property: " + name);
2066    }
2067
2068    protected static final Type STRING_ARRAY_TYPE = new ListTypeImpl("", "", StringType.INSTANCE);
2069
2070    public static Type getType(String name) {
2071        switch (name) {
2072        case KEY_IS_VERSION:
2073        case KEY_IS_CHECKED_IN:
2074        case KEY_IS_LATEST_VERSION:
2075        case KEY_IS_LATEST_MAJOR_VERSION:
2076        case KEY_IS_PROXY:
2077        case KEY_ACE_GRANT:
2078        case KEY_IS_TRASHED:
2079        case KEY_IS_RECORD:
2080        case KEY_HAS_LEGAL_HOLD:
2081            return BooleanType.INSTANCE;
2082        case KEY_VERSION_CREATED:
2083        case KEY_LOCK_CREATED:
2084        case KEY_ACE_BEGIN:
2085        case KEY_ACE_END:
2086        case KEY_RETAIN_UNTIL:
2087            return DateType.INSTANCE;
2088        case KEY_MIXIN_TYPES:
2089        case KEY_ANCESTOR_IDS:
2090        case KEY_PROXY_IDS:
2091        case KEY_READ_ACL:
2092            return STRING_ARRAY_TYPE;
2093        }
2094        return null;
2095    }
2096
2097    /**
2098     * Keys for boolean values that are stored as true or null (never false).
2099     *
2100     * @since 11.1
2101     */
2102    public static final Set<String> TRUE_OR_NULL_BOOLEAN_KEYS = Set.of( //
2103            KEY_IS_VERSION, //
2104            KEY_IS_CHECKED_IN, //
2105            KEY_IS_LATEST_VERSION, //
2106            KEY_IS_LATEST_MAJOR_VERSION, //
2107            KEY_IS_PROXY, //
2108            KEY_IS_TRASHED, //
2109            KEY_IS_RECORD, //
2110            KEY_HAS_LEGAL_HOLD);
2111
2112    /**
2113     * Keys for values that are ids.
2114     *
2115     * @since 11.1
2116     */
2117    public static final Set<String> ID_VALUES_KEYS = Set.of( //
2118            KEY_ID, //
2119            KEY_PARENT_ID, //
2120            KEY_ANCESTOR_IDS, //
2121            KEY_VERSION_SERIES_ID, //
2122            KEY_BASE_VERSION_ID, //
2123            KEY_PROXY_TARGET_ID, //
2124            KEY_PROXY_IDS, //
2125            KEY_PROXY_VERSION_SERIES_ID, //
2126            KEY_FULLTEXT_JOBID);
2127
2128    @Override
2129    public LockManager getLockManager() {
2130        // we have to ask the repository because the lock manager may be contributed
2131        return ((DBSRepository) repository).getLockManager();
2132    }
2133
2134    /**
2135     * Marks this document id as belonging to a user change.
2136     *
2137     * @since 9.2
2138     */
2139    public void markUserChange(String id) {
2140        if (changeTokenEnabled) {
2141            transaction.markUserChange(id);
2142        }
2143    }
2144
2145    /*
2146     * ----- Transaction management -----
2147     */
2148
2149    @Override
2150    public void start() {
2151        transaction.begin();
2152    }
2153
2154    @Override
2155    public void end() {
2156        // nothing
2157    }
2158
2159    @Override
2160    public void commit() {
2161        transaction.commit();
2162    }
2163
2164    @Override
2165    public void rollback() {
2166        transaction.rollback();
2167    }
2168
2169}