001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage;
020
021import java.io.IOException;
022import java.io.Serializable;
023import java.lang.reflect.Array;
024import java.util.ArrayDeque;
025import java.util.ArrayList;
026import java.util.Arrays;
027import java.util.Calendar;
028import java.util.Collection;
029import java.util.Collections;
030import java.util.Deque;
031import java.util.HashMap;
032import java.util.HashSet;
033import java.util.List;
034import java.util.Map;
035import java.util.Map.Entry;
036import java.util.Set;
037import java.util.function.Consumer;
038import java.util.regex.Pattern;
039
040import org.apache.commons.lang.ArrayUtils;
041import org.apache.commons.lang.StringUtils;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.DocumentNotFoundException;
044import org.nuxeo.ecm.core.api.Lock;
045import org.nuxeo.ecm.core.api.PropertyException;
046import org.nuxeo.ecm.core.api.model.DocumentPart;
047import org.nuxeo.ecm.core.api.model.Property;
048import org.nuxeo.ecm.core.api.model.PropertyNotFoundException;
049import org.nuxeo.ecm.core.api.model.impl.ComplexProperty;
050import org.nuxeo.ecm.core.api.model.impl.primitives.BlobProperty;
051import org.nuxeo.ecm.core.blob.BlobInfo;
052import org.nuxeo.ecm.core.blob.DocumentBlobManager;
053import org.nuxeo.ecm.core.model.Document;
054import org.nuxeo.ecm.core.schema.SchemaManager;
055import org.nuxeo.ecm.core.schema.TypeConstants;
056import org.nuxeo.ecm.core.schema.types.ComplexType;
057import org.nuxeo.ecm.core.schema.types.CompositeType;
058import org.nuxeo.ecm.core.schema.types.Field;
059import org.nuxeo.ecm.core.schema.types.ListType;
060import org.nuxeo.ecm.core.schema.types.Schema;
061import org.nuxeo.ecm.core.schema.types.SimpleTypeImpl;
062import org.nuxeo.ecm.core.schema.types.Type;
063import org.nuxeo.ecm.core.schema.types.primitives.BinaryType;
064import org.nuxeo.ecm.core.schema.types.primitives.BooleanType;
065import org.nuxeo.ecm.core.schema.types.primitives.DateType;
066import org.nuxeo.ecm.core.schema.types.primitives.DoubleType;
067import org.nuxeo.ecm.core.schema.types.primitives.IntegerType;
068import org.nuxeo.ecm.core.schema.types.primitives.LongType;
069import org.nuxeo.ecm.core.schema.types.primitives.StringType;
070import org.nuxeo.runtime.api.Framework;
071
072/**
073 * Base implementation for a Document.
074 * <p>
075 * Knows how to read and write values. It is generic in terms of a base State class from which one can read and write
076 * values.
077 *
078 * @since 7.3
079 */
080public abstract class BaseDocument<T extends StateAccessor> implements Document {
081
082    public static final String[] EMPTY_STRING_ARRAY = new String[0];
083
084    public static final String BLOB_NAME = "name";
085
086    public static final String BLOB_MIME_TYPE = "mime-type";
087
088    public static final String BLOB_ENCODING = "encoding";
089
090    public static final String BLOB_DIGEST = "digest";
091
092    public static final String BLOB_LENGTH = "length";
093
094    public static final String BLOB_DATA = "data";
095
096    public static final String DC_PREFIX = "dc:";
097
098    public static final String DC_ISSUED = "dc:issued";
099
100    // used instead of ecm:changeToken when change tokens are disabled
101    public static final String DC_MODIFIED = "dc:modified";
102
103    public static final String RELATED_TEXT_RESOURCES = "relatedtextresources";
104
105    public static final String RELATED_TEXT_ID = "relatedtextid";
106
107    public static final String RELATED_TEXT = "relatedtext";
108
109    public static final String FULLTEXT_JOBID_PROP = "ecm:fulltextJobId";
110
111    public static final String FULLTEXT_SIMPLETEXT_PROP = "ecm:simpleText";
112
113    public static final String FULLTEXT_BINARYTEXT_PROP = "ecm:binaryText";
114
115    public static final String MISC_LIFECYCLE_STATE_PROP = "ecm:lifeCycleState";
116
117    public static final String LOCK_OWNER_PROP = "ecm:lockOwner";
118
119    public static final String LOCK_CREATED_PROP = "ecm:lockCreated";
120
121    public static final Set<String> VERSION_WRITABLE_PROPS = new HashSet<String>(Arrays.asList( //
122            FULLTEXT_JOBID_PROP, //
123            FULLTEXT_BINARYTEXT_PROP, //
124            MISC_LIFECYCLE_STATE_PROP, //
125            LOCK_OWNER_PROP, //
126            LOCK_CREATED_PROP, //
127            DC_ISSUED, //
128            RELATED_TEXT_RESOURCES, //
129            RELATED_TEXT_ID, //
130            RELATED_TEXT //
131    ));
132
133    protected final static Pattern NON_CANONICAL_INDEX = Pattern.compile("[^/\\[\\]]+" // name
134            + "\\[(\\d+)\\]" // index in brackets
135    );
136
137    protected static final Runnable NO_DIRTY = () -> {
138    };
139
140    /**
141     * Gets the list of proxy schemas, if this is a proxy.
142     *
143     * @return the proxy schemas, or {@code null}
144     */
145    protected abstract List<Schema> getProxySchemas();
146
147    /**
148     * Gets a child state.
149     *
150     * @param state the parent state
151     * @param name the child name
152     * @param type the child's type
153     * @return the child state, or {@code null} if it doesn't exist
154     */
155    protected abstract T getChild(T state, String name, Type type) throws PropertyException;
156
157    /**
158     * Gets a child state into which we will want to write data.
159     * <p>
160     * Creates it if needed.
161     *
162     * @param state the parent state
163     * @param name the child name
164     * @param type the child's type
165     * @return the child state, never {@code null}
166     * @since 7.4
167     */
168    protected abstract T getChildForWrite(T state, String name, Type type) throws PropertyException;
169
170    /**
171     * Gets a child state which is a list.
172     *
173     * @param state the parent state
174     * @param name the child name
175     * @return the child state, never {@code null}
176     */
177    protected abstract List<T> getChildAsList(T state, String name) throws PropertyException;
178
179    /**
180     * Update a list.
181     *
182     * @param state the parent state
183     * @param name the child name
184     * @param field the list element type
185     * @param xpath the xpath of this list
186     * @param values the values
187     */
188    protected abstract void updateList(T state, String name, Field field, String xpath, List<Object> values)
189            throws PropertyException;
190
191    /**
192     * Update a list.
193     *
194     * @param state the parent state
195     * @param name the child name
196     * @param property the property
197     * @return the list of states to write
198     */
199    protected abstract List<T> updateList(T state, String name, Property property) throws PropertyException;
200
201    /**
202     * Finds the internal name to use to refer to this property.
203     */
204    protected abstract String internalName(String name);
205
206    /**
207     * Canonicalizes a Nuxeo xpath.
208     * <p>
209     * Replaces {@code a/foo[123]/b} with {@code a/123/b}
210     *
211     * @param xpath the xpath
212     * @return the canonicalized xpath.
213     */
214    protected static String canonicalXPath(String xpath) {
215        if (xpath.indexOf('[') > 0) {
216            xpath = NON_CANONICAL_INDEX.matcher(xpath).replaceAll("$1");
217        }
218        return xpath;
219    }
220
221    /** Copies the array with an appropriate class depending on the type. */
222    protected static Object[] typedArray(Type type, Object[] array) {
223        if (array == null) {
224            array = EMPTY_STRING_ARRAY;
225        }
226        Class<?> klass;
227        if (type instanceof StringType) {
228            klass = String.class;
229        } else if (type instanceof BooleanType) {
230            klass = Boolean.class;
231        } else if (type instanceof LongType) {
232            klass = Long.class;
233        } else if (type instanceof DoubleType) {
234            klass = Double.class;
235        } else if (type instanceof DateType) {
236            klass = Calendar.class;
237        } else if (type instanceof BinaryType) {
238            klass = String.class;
239        } else if (type instanceof IntegerType) {
240            throw new RuntimeException("Unimplemented primitive type: " + type.getClass().getName());
241        } else if (type instanceof SimpleTypeImpl) {
242            // simple type with constraints -- ignore constraints XXX
243            return typedArray(type.getSuperType(), array);
244        } else {
245            throw new RuntimeException("Invalid primitive type: " + type.getClass().getName());
246        }
247        int len = array.length;
248        Object[] copy = (Object[]) Array.newInstance(klass, len);
249        System.arraycopy(array, 0, copy, 0, len);
250        return copy;
251    }
252
253    protected static boolean isVersionWritableProperty(String name) {
254        return VERSION_WRITABLE_PROPS.contains(name) //
255                || name.startsWith(FULLTEXT_BINARYTEXT_PROP) //
256                || name.startsWith(FULLTEXT_SIMPLETEXT_PROP);
257    }
258
259    protected static void clearDirtyFlags(Property property) {
260        if (property.isContainer()) {
261            for (Property p : property) {
262                clearDirtyFlags(p);
263            }
264        }
265        property.clearDirtyFlags();
266    }
267
268    /**
269     * Checks for ignored writes. May throw.
270     */
271    protected boolean checkReadOnlyIgnoredWrite(Property property, T state) throws PropertyException {
272        String name = property.getField().getName().getPrefixedName();
273        if (!isReadOnly() || isVersionWritableProperty(name)) {
274            // do write
275            return false;
276        }
277        if (!isVersion()) {
278            throw new PropertyException("Cannot write readonly property: " + name);
279        }
280        if (!name.startsWith(DC_PREFIX) && !getTopLevelSchema(property).isVersionWritabe()) {
281            throw new PropertyException("Cannot set property on a version: " + name);
282        }
283        // ignore write if value can quickly be detected as unchanged
284        Object value = property.getValueForWrite();
285        Object oldValue;
286        Type type = property.getType();
287        if (type.isSimpleType()) {
288            oldValue = state.getSingle(name);
289        } else if (type.isListType() && ((ListType) type).getFieldType().isSimpleType()) {
290            oldValue = state.getArray(name);
291        } else {
292            // complex property or complex list, no quick way to detect changes
293            // do write
294            return false;
295        }
296        if (!ArrayUtils.isEquals(value, oldValue)) {
297            // do write
298            return false;
299        }
300        // ignore attempt to write identical value
301        return true;
302    }
303
304    /**
305     * Gets the {@link Schema} at the top-level of the type hierarchy for this {@link Property}.
306     *
307     * @since 9.3
308     */
309    protected Schema getTopLevelSchema(Property property) {
310        for (;;) {
311            Type type = property.getType();
312            if (type instanceof Schema) {
313                return (Schema) type;
314            }
315            property = property.getParent();
316        }
317    }
318
319    protected BlobInfo getBlobInfo(T state) throws PropertyException {
320        BlobInfo blobInfo = new BlobInfo();
321        blobInfo.key = (String) state.getSingle(BLOB_DATA);
322        blobInfo.filename = (String) state.getSingle(BLOB_NAME);
323        blobInfo.mimeType = (String) state.getSingle(BLOB_MIME_TYPE);
324        blobInfo.encoding = (String) state.getSingle(BLOB_ENCODING);
325        blobInfo.digest = (String) state.getSingle(BLOB_DIGEST);
326        blobInfo.length = (Long) state.getSingle(BLOB_LENGTH);
327        return blobInfo;
328    }
329
330    protected void setBlobInfo(T state, BlobInfo blobInfo) throws PropertyException {
331        state.setSingle(BLOB_DATA, blobInfo.key);
332        state.setSingle(BLOB_NAME, blobInfo.filename);
333        state.setSingle(BLOB_MIME_TYPE, blobInfo.mimeType);
334        state.setSingle(BLOB_ENCODING, blobInfo.encoding);
335        state.setSingle(BLOB_DIGEST, blobInfo.digest);
336        state.setSingle(BLOB_LENGTH, blobInfo.length);
337    }
338
339    /**
340     * Gets a value (may be complex/list) from the document at the given xpath.
341     */
342    protected Object getValueObject(T state, String xpath) throws PropertyException {
343        xpath = canonicalXPath(xpath);
344        String[] segments = xpath.split("/");
345
346        /*
347         * During this loop state may become null if we read an uninitialized complex property (DBS), in that case the
348         * code must treat it as reading uninitialized values for its children.
349         */
350        ComplexType parentType = getType();
351        for (int i = 0; i < segments.length; i++) {
352            String segment = segments[i];
353            Field field = parentType.getField(segment);
354            if (field == null && i == 0) {
355                // check facets
356                SchemaManager schemaManager = Framework.getService(SchemaManager.class);
357                for (String facet : getFacets()) {
358                    CompositeType facetType = schemaManager.getFacet(facet);
359                    field = facetType.getField(segment);
360                    if (field != null) {
361                        break;
362                    }
363                }
364            }
365            if (field == null && i == 0 && getProxySchemas() != null) {
366                // check proxy schemas
367                for (Schema schema : getProxySchemas()) {
368                    field = schema.getField(segment);
369                    if (field != null) {
370                        break;
371                    }
372                }
373            }
374            if (field == null) {
375                throw new PropertyNotFoundException(xpath, i == 0 ? null : "Unknown segment: " + segment);
376            }
377            String name = field.getName().getPrefixedName(); // normalize from segment
378            Type type = field.getType();
379
380            // check if we have a complex list index in the next position
381            if (i < segments.length - 1 && StringUtils.isNumeric(segments[i + 1])) {
382                int index = Integer.parseInt(segments[i + 1]);
383                i++;
384                if (!type.isListType() || ((ListType) type).getFieldType().isSimpleType()) {
385                    throw new PropertyNotFoundException(xpath, "Cannot use index after segment: " + segment);
386                }
387                List<T> list = state == null ? Collections.emptyList() : getChildAsList(state, name);
388                if (index >= list.size()) {
389                    throw new PropertyNotFoundException(xpath, "Index out of bounds: " + index);
390                }
391                // find complex list state
392                state = list.get(index);
393                parentType = (ComplexType) ((ListType) type).getFieldType();
394                if (i == segments.length - 1) {
395                    // last segment
396                    return getValueComplex(state, parentType);
397                } else {
398                    // not last segment
399                    continue;
400                }
401            }
402
403            if (i == segments.length - 1) {
404                // last segment
405                return state == null ? null : getValueField(state, field);
406            } else {
407                // not last segment
408                if (type.isSimpleType()) {
409                    // scalar
410                    throw new PropertyNotFoundException(xpath, "Segment must be last: " + segment);
411                } else if (type.isComplexType()) {
412                    // complex property
413                    state = state == null ? null : getChild(state, name, type);
414                    // here state can be null (DBS), continue loop with it, meaning uninitialized for read
415                    parentType = (ComplexType) type;
416                } else {
417                    // list
418                    ListType listType = (ListType) type;
419                    if (listType.isArray()) {
420                        // array of scalars
421                        throw new PropertyNotFoundException(xpath, "Segment must be last: " + segment);
422                    } else {
423                        // complex list but next segment was not numeric
424                        throw new PropertyNotFoundException(xpath, "Missing list index after segment: " + segment);
425                    }
426                }
427            }
428        }
429        throw new AssertionError("not reached");
430    }
431
432    protected Object getValueField(T state, Field field) throws PropertyException {
433        Type type = field.getType();
434        String name = field.getName().getPrefixedName();
435        name = internalName(name);
436        if (type.isSimpleType()) {
437            // scalar
438            return state.getSingle(name);
439        } else if (type.isComplexType()) {
440            // complex property
441            T childState = getChild(state, name, type);
442            if (childState == null) {
443                return null;
444            }
445            return getValueComplex(childState, (ComplexType) type);
446        } else {
447            // array or list
448            Type fieldType = ((ListType) type).getFieldType();
449            if (fieldType.isSimpleType()) {
450                // array
451                return state.getArray(name);
452            } else {
453                // complex list
454                List<T> childStates = getChildAsList(state, name);
455                List<Object> list = new ArrayList<>(childStates.size());
456                for (T childState : childStates) {
457                    Object value = getValueComplex(childState, (ComplexType) fieldType);
458                    list.add(value);
459                }
460                return list;
461            }
462        }
463    }
464
465    protected Object getValueComplex(T state, ComplexType complexType) throws PropertyException {
466        if (TypeConstants.isContentType(complexType)) {
467            return getValueBlob(state);
468        }
469        Map<String, Object> map = new HashMap<>();
470        for (Field field : complexType.getFields()) {
471            String name = field.getName().getPrefixedName();
472            Object value = getValueField(state, field);
473            map.put(name, value);
474        }
475        return map;
476    }
477
478    protected Blob getValueBlob(T state) throws PropertyException {
479        BlobInfo blobInfo = getBlobInfo(state);
480        DocumentBlobManager blobManager = Framework.getService(DocumentBlobManager.class);
481        try {
482            return blobManager.readBlob(blobInfo, getRepositoryName());
483        } catch (IOException e) {
484            throw new PropertyException("Cannot get blob info for: " + blobInfo.key, e);
485        }
486    }
487
488    /**
489     * Sets a value (may be complex/list) into the document at the given xpath.
490     */
491    protected void setValueObject(T state, String xpath, Object value) throws PropertyException {
492        xpath = canonicalXPath(xpath);
493        String[] segments = xpath.split("/");
494
495        ComplexType parentType = getType();
496        for (int i = 0; i < segments.length; i++) {
497            String segment = segments[i];
498            Field field = parentType.getField(segment);
499            if (field == null && i == 0) {
500                // check facets
501                SchemaManager schemaManager = Framework.getService(SchemaManager.class);
502                for (String facet : getFacets()) {
503                    CompositeType facetType = schemaManager.getFacet(facet);
504                    field = facetType.getField(segment);
505                    if (field != null) {
506                        break;
507                    }
508                }
509            }
510            if (field == null && i == 0 && getProxySchemas() != null) {
511                // check proxy schemas
512                for (Schema schema : getProxySchemas()) {
513                    field = schema.getField(segment);
514                    if (field != null) {
515                        break;
516                    }
517                }
518            }
519            if (field == null) {
520                throw new PropertyNotFoundException(xpath, i == 0 ? null : "Unknown segment: " + segment);
521            }
522            String name = field.getName().getPrefixedName(); // normalize from segment
523            Type type = field.getType();
524
525            // check if we have a complex list index in the next position
526            if (i < segments.length - 1 && StringUtils.isNumeric(segments[i + 1])) {
527                int index = Integer.parseInt(segments[i + 1]);
528                i++;
529                if (!type.isListType() || ((ListType) type).getFieldType().isSimpleType()) {
530                    throw new PropertyNotFoundException(xpath, "Cannot use index after segment: " + segment);
531                }
532                List<T> list = getChildAsList(state, name);
533                if (index >= list.size()) {
534                    throw new PropertyNotFoundException(xpath, "Index out of bounds: " + index);
535                }
536                // find complex list state
537                state = list.get(index);
538                field = ((ListType) type).getField();
539                if (i == segments.length - 1) {
540                    // last segment
541                    setValueComplex(state, field, xpath, value);
542                } else {
543                    // not last segment
544                    parentType = (ComplexType) field.getType();
545                }
546                continue;
547            }
548
549            if (i == segments.length - 1) {
550                // last segment
551                setValueField(state, field, xpath, value);
552            } else {
553                // not last segment
554                if (type.isSimpleType()) {
555                    // scalar
556                    throw new PropertyNotFoundException(xpath, "Segment must be last: " + segment);
557                } else if (type.isComplexType()) {
558                    // complex property
559                    state = getChildForWrite(state, name, type);
560                    parentType = (ComplexType) type;
561                } else {
562                    // list
563                    ListType listType = (ListType) type;
564                    if (listType.isArray()) {
565                        // array of scalars
566                        throw new PropertyNotFoundException(xpath, "Segment must be last: " + segment);
567                    } else {
568                        // complex list but next segment was not numeric
569                        throw new PropertyNotFoundException(xpath, "Missing list index after segment: " + segment);
570                    }
571                }
572            }
573        }
574    }
575
576    protected void setValueField(T state, Field field, String xpath, Object value) throws PropertyException {
577        Type type = field.getType();
578        String name = field.getName().getPrefixedName(); // normalize from map key
579        name = internalName(name);
580        // TODO we could check for read-only here
581        if (type.isSimpleType()) {
582            // scalar
583            state.setSingle(name, value);
584        } else if (type.isComplexType()) {
585            // complex property
586            T childState = getChildForWrite(state, name, type);
587            setValueComplex(childState, field, xpath, value);
588        } else {
589            // array or list
590            ListType listType = (ListType) type;
591            Type fieldType = listType.getFieldType();
592            if (fieldType.isSimpleType()) {
593                // array
594                if (value instanceof List) {
595                    value = ((List<?>) value).toArray(new Object[0]);
596                }
597                state.setArray(name, (Object[]) value);
598            } else {
599                // complex list
600                if (value != null && !(value instanceof List)) {
601                    throw new PropertyException(
602                            "Expected List value for: " + name + ", got " + value.getClass().getName() + " instead");
603                }
604                @SuppressWarnings("unchecked")
605                List<Object> values = value == null ? Collections.emptyList() : (List<Object>) value;
606                updateList(state, name, listType.getField(), xpath, values);
607            }
608        }
609    }
610
611    protected void setValueComplex(T state, Field field, String xpath, Object value) throws PropertyException {
612        ComplexType complexType = (ComplexType) field.getType();
613        if (TypeConstants.isContentType(complexType)) {
614            if (value != null && !(value instanceof Blob)) {
615                throw new PropertyException(
616                        "Expected Blob value for: " + xpath + ", got " + value.getClass().getName() + " instead");
617            }
618            setValueBlob(state, (Blob) value, xpath);
619            return;
620        }
621        if (value != null && !(value instanceof Map)) {
622            throw new PropertyException(
623                    "Expected Map value for: " + xpath + ", got " + value.getClass().getName() + " instead");
624        }
625        @SuppressWarnings("unchecked")
626        Map<String, Object> map = value == null ? Collections.emptyMap() : (Map<String, Object>) value;
627        Set<String> keys = new HashSet<>(map.keySet());
628        for (Field f : complexType.getFields()) {
629            String name = f.getName().getPrefixedName();
630            keys.remove(name);
631            value = map.get(name);
632            setValueField(state, f, xpath + '/' + name, value);
633        }
634        if (!keys.isEmpty()) {
635            throw new PropertyException("Unknown key: " + keys.iterator().next() + " for " + xpath);
636        }
637    }
638
639    protected void setValueBlob(T state, Blob blob, String xpath) throws PropertyException {
640        BlobInfo blobInfo = new BlobInfo();
641        if (blob != null) {
642            DocumentBlobManager blobManager = Framework.getService(DocumentBlobManager.class);
643            try {
644                blobInfo.key = blobManager.writeBlob(blob, this, xpath);
645            } catch (IOException e) {
646                throw new PropertyException("Cannot get blob info for: " + blob, e);
647            }
648            blobInfo.filename = blob.getFilename();
649            blobInfo.mimeType = blob.getMimeType();
650            blobInfo.encoding = blob.getEncoding();
651            blobInfo.digest = blob.getDigest();
652            blobInfo.length = blob.getLength() == -1 ? null : Long.valueOf(blob.getLength());
653        }
654        setBlobInfo(state, blobInfo);
655    }
656
657    /**
658     * Reads state into a complex property.
659     */
660    protected void readComplexProperty(T state, ComplexProperty complexProperty) throws PropertyException {
661        if (state == null) {
662            complexProperty.init(null);
663            return;
664        }
665        if (complexProperty instanceof BlobProperty) {
666            Blob blob = getValueBlob(state);
667            complexProperty.init((Serializable) blob);
668            return;
669        }
670        for (Property property : complexProperty) {
671            String name = property.getField().getName().getPrefixedName();
672            name = internalName(name);
673            Type type = property.getType();
674            if (type.isSimpleType()) {
675                // simple property
676                Object value = state.getSingle(name);
677                property.init((Serializable) value);
678            } else if (type.isComplexType()) {
679                // complex property
680                T childState = getChild(state, name, type);
681                readComplexProperty(childState, (ComplexProperty) property);
682                ((ComplexProperty) property).removePhantomFlag();
683            } else {
684                ListType listType = (ListType) type;
685                if (listType.getFieldType().isSimpleType()) {
686                    // array
687                    Object[] array = state.getArray(name);
688                    array = typedArray(listType.getFieldType(), array);
689                    property.init(array);
690                } else {
691                    // complex list
692                    Field listField = listType.getField();
693                    List<T> childStates = getChildAsList(state, name);
694                    // TODO property.init(null) if null children in DBS
695                    List<Object> list = new ArrayList<>(childStates.size());
696                    for (T childState : childStates) {
697                        ComplexProperty p = (ComplexProperty) complexProperty.getRoot().createProperty(property,
698                                listField, 0);
699                        readComplexProperty(childState, p);
700                        list.add(p.getValue());
701                    }
702                    property.init((Serializable) list);
703                }
704            }
705        }
706    }
707
708    protected static class BlobWriteInfo<T extends StateAccessor> {
709
710        public final T state;
711
712        public final Blob blob;
713
714        public final String xpath;
715
716        public BlobWriteInfo(T state, Blob blob, String xpath) {
717            this.state = state;
718            this.blob = blob;
719            this.xpath = xpath;
720        }
721    }
722
723    protected static class BlobWriteContext<T extends StateAccessor> implements WriteContext {
724
725        public final Map<BaseDocument<T>, List<BlobWriteInfo<T>>> blobWriteInfos = new HashMap<>();
726
727        public final Set<String> xpaths = new HashSet<>();
728
729        /**
730         * Records a change to a given xpath.
731         */
732        public void recordChange(String xpath) {
733            xpaths.add(xpath);
734        }
735
736        /**
737         * Records a blob update.
738         */
739        public void recordBlob(BaseDocument<T> doc, T state, Blob blob, String xpath) {
740            BlobWriteInfo<T> info = new BlobWriteInfo<T>(state, blob, xpath);
741            blobWriteInfos.computeIfAbsent(doc, k -> new ArrayList<>()).add(info);
742        }
743
744        @Override
745        public Set<String> getChanges() {
746            return xpaths;
747        }
748
749        // note, in the proxy case baseDoc may be different from the doc in the map
750        @Override
751        public void flush(Document baseDoc) {
752            // first, write all updated blobs
753            for (Entry<BaseDocument<T>, List<BlobWriteInfo<T>>> es : blobWriteInfos.entrySet()) {
754                BaseDocument<T> doc = es.getKey();
755                for (BlobWriteInfo<T> info : es.getValue()) {
756                    doc.setValueBlob(info.state, info.blob, info.xpath);
757                }
758            }
759            // then inform the blob manager about the changed xpaths
760            DocumentBlobManager blobManager = Framework.getService(DocumentBlobManager.class);
761            blobManager.notifyChanges(baseDoc, xpaths);
762        }
763    }
764
765    @Override
766    public WriteContext getWriteContext() {
767        return new BlobWriteContext<T>();
768    }
769
770    /**
771     * Writes state from a complex property.
772     *
773     * @return {@code true} if something changed
774     */
775    protected boolean writeComplexProperty(T state, ComplexProperty complexProperty, WriteContext writeContext)
776            throws PropertyException {
777        boolean writeAll = ((DocumentPart) complexProperty).getClearComplexPropertyBeforeSet();
778        return writeComplexProperty(state, complexProperty, null, writeAll, writeContext);
779    }
780
781    /**
782     * Writes state from a complex property.
783     * <p>
784     * Writes only properties that are dirty, unless writeAll is true in which case everything is written.
785     *
786     * @return {@code true} if something changed
787     */
788    protected boolean writeComplexProperty(T state, ComplexProperty complexProperty, String xpath, boolean writeAll,
789            WriteContext wc) throws PropertyException {
790        @SuppressWarnings("unchecked")
791        BlobWriteContext<T> writeContext = (BlobWriteContext<T>) wc;
792        if (complexProperty instanceof BlobProperty) {
793            Serializable value = ((BlobProperty) complexProperty).getValueForWrite();
794            if (value != null && !(value instanceof Blob)) {
795                throw new PropertyException("Cannot write a non-Blob value: " + value);
796            }
797            writeContext.recordBlob(this, state, (Blob) value, xpath);
798            return true;
799        }
800        boolean changed = false;
801        for (Property property : complexProperty) {
802            // write dirty properties, but also phantoms with non-null default values
803            // this is critical for DeltaLong updates to work, they need a non-null initial value
804            if (writeAll || property.isDirty()
805                    || (property.isPhantom() && property.getField().getDefaultValue() != null)) {
806                // do the write
807            } else {
808                continue;
809            }
810            String name = property.getField().getName().getPrefixedName();
811            name = internalName(name);
812            if (checkReadOnlyIgnoredWrite(property, state)) {
813                continue;
814            }
815            String xp = xpath == null ? name : xpath + '/' + name;
816            writeContext.recordChange(xp);
817            changed = true;
818
819            Type type = property.getType();
820            if (type.isSimpleType()) {
821                // simple property
822                Serializable value = property.getValueForWrite();
823                state.setSingle(name, value);
824            } else if (type.isComplexType()) {
825                // complex property
826                T childState = getChildForWrite(state, name, type);
827                writeComplexProperty(childState, (ComplexProperty) property, xp, writeAll, writeContext);
828            } else {
829                ListType listType = (ListType) type;
830                if (listType.getFieldType().isSimpleType()) {
831                    // array
832                    Serializable value = property.getValueForWrite();
833                    if (value instanceof List) {
834                        List<?> list = (List<?>) value;
835                        Object[] array;
836                        if (list.isEmpty()) {
837                            array = new Object[0];
838                        } else {
839                            // use properly-typed array, useful for mem backend that doesn't re-convert all types
840                            Class<?> klass = list.get(0).getClass();
841                            array = (Object[]) Array.newInstance(klass, list.size());
842                        }
843                        value = list.toArray(array);
844                    } else if (value instanceof Object[]) {
845                        Object[] ar = (Object[]) value;
846                        if (ar.length != 0) {
847                            // use properly-typed array, useful for mem backend that doesn't re-convert all types
848                            Class<?> klass = Object.class;
849                            for (Object o : ar) {
850                                if (o != null) {
851                                    klass = o.getClass();
852                                    break;
853                                }
854                            }
855                            Object[] array;
856                            if (ar.getClass().getComponentType() == klass) {
857                                array = ar;
858                            } else {
859                                // copy to array with proper component type
860                                array = (Object[]) Array.newInstance(klass, ar.length);
861                                System.arraycopy(ar, 0, array, 0, ar.length);
862                            }
863                            value = array;
864                        }
865                    } else if (value == null) {
866                        // ok
867                    } else {
868                        throw new IllegalStateException(value.toString());
869                    }
870                    state.setArray(name, (Object[]) value);
871                } else {
872                    // complex list
873                    // update it
874                    List<T> childStates = updateList(state, name, property);
875                    // write values
876                    int i = 0;
877                    for (Property childProperty : property.getChildren()) {
878                        T childState = childStates.get(i);
879                        String xpi = xp + '/' + i;
880                        boolean moved = childProperty.isMoved();
881                        boolean c = writeComplexProperty(childState, (ComplexProperty) childProperty, xpi,
882                                writeAll || moved, writeContext);
883                        if (c) {
884                            writeContext.recordChange(xpi);
885                        }
886                        i++;
887                    }
888                }
889            }
890        }
891        return changed;
892    }
893
894    /**
895     * Reads prefetched values.
896     */
897    protected Map<String, Serializable> readPrefetch(T state, ComplexType complexType, Set<String> xpaths)
898            throws PropertyException {
899        // augment xpaths with all prefixes, to cut short recursive search
900        Set<String> prefixes = new HashSet<>();
901        for (String xpath : xpaths) {
902            for (;;) {
903                // add as prefix
904                if (!prefixes.add(xpath)) {
905                    // already present, we can stop
906                    break;
907                }
908                // loop with its prefix
909                int i = xpath.lastIndexOf('/');
910                if (i == -1) {
911                    break;
912                }
913                xpath = xpath.substring(0, i);
914            }
915        }
916        Map<String, Serializable> prefetch = new HashMap<String, Serializable>();
917        readPrefetch(state, complexType, null, null, prefixes, prefetch);
918        return prefetch;
919    }
920
921    protected void readPrefetch(T state, ComplexType complexType, String xpathGeneric, String xpath,
922            Set<String> prefixes, Map<String, Serializable> prefetch) throws PropertyException {
923        if (TypeConstants.isContentType(complexType)) {
924            if (!prefixes.contains(xpathGeneric)) {
925                return;
926            }
927            Blob blob = getValueBlob(state);
928            prefetch.put(xpath, (Serializable) blob);
929            return;
930        }
931        for (Field field : complexType.getFields()) {
932            readPrefetchField(state, field, xpathGeneric, xpath, prefixes, prefetch);
933        }
934    }
935
936    protected void readPrefetchField(T state, Field field, String xpathGeneric, String xpath, Set<String> prefixes,
937            Map<String, Serializable> prefetch) {
938        String name = field.getName().getPrefixedName();
939        Type type = field.getType();
940        xpathGeneric = xpathGeneric == null ? name : xpathGeneric + '/' + name;
941        xpath = xpath == null ? name : xpath + '/' + name;
942        if (!prefixes.contains(xpathGeneric)) {
943            return;
944        }
945        if (type.isSimpleType()) {
946            // scalar
947            Object value = state.getSingle(name);
948            prefetch.put(xpath, (Serializable) value);
949        } else if (type.isComplexType()) {
950            // complex property
951            T childState = getChild(state, name, type);
952            if (childState != null) {
953                readPrefetch(childState, (ComplexType) type, xpathGeneric, xpath, prefixes, prefetch);
954            }
955        } else {
956            // array or list
957            ListType listType = (ListType) type;
958            if (listType.getFieldType().isSimpleType()) {
959                // array
960                Object[] value = state.getArray(name);
961                prefetch.put(xpath, value);
962            } else {
963                // complex list
964                List<T> childStates = getChildAsList(state, name);
965                Field listField = listType.getField();
966                xpathGeneric += "/*";
967                int i = 0;
968                for (T childState : childStates) {
969                    readPrefetch(childState, (ComplexType) listField.getType(), xpathGeneric, xpath + '/' + i++,
970                            prefixes, prefetch);
971                }
972            }
973        }
974    }
975
976    /**
977     * Visits all the blobs of this document and calls the passed blob visitor on each one.
978     */
979    protected void visitBlobs(T state, Consumer<BlobAccessor> blobVisitor, Runnable markDirty)
980            throws PropertyException {
981        Visit visit = new Visit(blobVisitor, markDirty);
982        // structural type
983        visit.visitBlobsComplex(state, getType());
984        // dynamic facets
985        SchemaManager schemaManager = Framework.getService(SchemaManager.class);
986        for (String facet : getFacets()) {
987            CompositeType facetType = schemaManager.getFacet(facet);
988            if (facetType != null) { // if not obsolete facet
989                visit.visitBlobsComplex(state, facetType);
990            }
991        }
992        // proxy schemas
993        if (getProxySchemas() != null) {
994            for (Schema schema : getProxySchemas()) {
995                visit.visitBlobsComplex(state, schema);
996            }
997        }
998    }
999
1000    protected class StateBlobAccessor implements BlobAccessor {
1001
1002        protected final Collection<String> path;
1003
1004        protected final T state;
1005
1006        protected final Runnable markDirty;
1007
1008        public StateBlobAccessor(Collection<String> path, T state, Runnable markDirty) {
1009            this.path = path;
1010            this.state = state;
1011            this.markDirty = markDirty;
1012        }
1013
1014        @Override
1015        public String getXPath() {
1016            return StringUtils.join(path, "/");
1017        }
1018
1019        @Override
1020        public Blob getBlob() throws PropertyException {
1021            return getValueBlob(state);
1022        }
1023
1024        @Override
1025        public void setBlob(Blob blob) throws PropertyException {
1026            markDirty.run();
1027            setValueBlob(state, blob, getXPath());
1028        }
1029    }
1030
1031    protected class Visit {
1032
1033        protected final Consumer<BlobAccessor> blobVisitor;
1034
1035        protected final Runnable markDirty;
1036
1037        protected final Deque<String> path;
1038
1039        public Visit(Consumer<BlobAccessor> blobVisitor, Runnable markDirty) {
1040            this.blobVisitor = blobVisitor;
1041            this.markDirty = markDirty;
1042            path = new ArrayDeque<>();
1043        }
1044
1045        public void visitBlobsComplex(T state, ComplexType complexType) throws PropertyException {
1046            if (TypeConstants.isContentType(complexType)) {
1047                blobVisitor.accept(new StateBlobAccessor(path, state, markDirty));
1048                return;
1049            }
1050            for (Field field : complexType.getFields()) {
1051                visitBlobsField(state, field);
1052            }
1053        }
1054
1055        protected void visitBlobsField(T state, Field field) throws PropertyException {
1056            Type type = field.getType();
1057            if (type.isSimpleType()) {
1058                // scalar
1059            } else if (type.isComplexType()) {
1060                // complex property
1061                String name = field.getName().getPrefixedName();
1062                T childState = getChild(state, name, type);
1063                if (childState != null) {
1064                    path.addLast(name);
1065                    visitBlobsComplex(childState, (ComplexType) type);
1066                    path.removeLast();
1067                }
1068            } else {
1069                // array or list
1070                Type fieldType = ((ListType) type).getFieldType();
1071                if (fieldType.isSimpleType()) {
1072                    // array
1073                } else {
1074                    // complex list
1075                    String name = field.getName().getPrefixedName();
1076                    path.addLast(name);
1077                    int i = 0;
1078                    for (T childState : getChildAsList(state, name)) {
1079                        path.addLast(String.valueOf(i++));
1080                        visitBlobsComplex(childState, (ComplexType) fieldType);
1081                        path.removeLast();
1082                    }
1083                    path.removeLast();
1084                }
1085            }
1086        }
1087    }
1088
1089    @Override
1090    public Lock getLock() {
1091        try {
1092            return getSession().getLockManager().getLock(getUUID());
1093        } catch (DocumentNotFoundException e) {
1094            return getDocumentLock();
1095        }
1096    }
1097
1098    @Override
1099    public Lock setLock(Lock lock) {
1100        if (lock == null) {
1101            throw new NullPointerException("Attempt to use null lock on: " + getUUID());
1102        }
1103        try {
1104            return getSession().getLockManager().setLock(getUUID(), lock);
1105        } catch (DocumentNotFoundException e) {
1106            return setDocumentLock(lock);
1107        }
1108    }
1109
1110    @Override
1111    public Lock removeLock(String owner) {
1112        try {
1113            return getSession().getLockManager().removeLock(getUUID(), owner);
1114        } catch (DocumentNotFoundException e) {
1115            return removeDocumentLock(owner);
1116        }
1117    }
1118
1119    /**
1120     * Gets the lock from this recently created and unsaved document.
1121     *
1122     * @return the lock, or {@code null} if no lock is set
1123     * @since 7.4
1124     */
1125    protected abstract Lock getDocumentLock();
1126
1127    /**
1128     * Sets a lock on this recently created and unsaved document.
1129     *
1130     * @param lock the lock to set
1131     * @return {@code null} if locking succeeded, or the existing lock if locking failed
1132     * @since 7.4
1133     */
1134    protected abstract Lock setDocumentLock(Lock lock);
1135
1136    /**
1137     * Removes a lock from this recently created and unsaved document.
1138     *
1139     * @param the owner to check, or {@code null} for no check
1140     * @return {@code null} if there was no lock or if removal succeeded, or a lock if it blocks removal due to owner
1141     *         mismatch
1142     * @since 7.4
1143     */
1144    protected abstract Lock removeDocumentLock(String owner);
1145
1146    // also used as a regexp for split
1147    public static final String TOKEN_SEP = "-";
1148
1149    /**
1150     * Builds the user-visible change token from low-level change token and system change token information.
1151     *
1152     * @param sysChangeToken the system change token
1153     * @param changeToken the change token
1154     * @return the user-visible change token
1155     * @since 9.2
1156     */
1157    public static String buildUserVisibleChangeToken(Long sysChangeToken, Long changeToken) {
1158        if (sysChangeToken == null || changeToken == null) {
1159            return null;
1160        }
1161        return sysChangeToken.toString() + TOKEN_SEP + changeToken.toString();
1162    }
1163
1164    /**
1165     * Validates that the passed user-visible change token is compatible with the current change token.
1166     *
1167     * @param sysChangeToken the system change token
1168     * @param changeToken the change token
1169     * @param userVisibleChangeToken the user-visible change token
1170     * @return {@code false} if the change token is not valid
1171     * @since 9.2
1172     */
1173    public static boolean validateUserVisibleChangeToken(Long sysChangeToken, Long changeToken,
1174            String userVisibleChangeToken) {
1175        if (sysChangeToken == null || changeToken == null) {
1176            return true;
1177        }
1178        // we only compare the change token, not the system change token, to allow background system updates
1179        String[] parts = userVisibleChangeToken.split(TOKEN_SEP);
1180        if (parts.length != 2) {
1181            return false; // invalid format
1182        }
1183        return parts[1].equals(changeToken.toString());
1184    }
1185
1186    /**
1187     * Validates that the passed user-visible change token is compatible with the current legacy change token.
1188     *
1189     * @param modified the {@code dc:modified} timestamp
1190     * @param userVisibleChangeToken the user-visible change token
1191     * @return {@code false} if the change token is not valid
1192     * @since 9.2
1193     */
1194    protected boolean validateLegacyChangeToken(Calendar modified, String userVisibleChangeToken) {
1195        if (modified == null) {
1196            return true;
1197        }
1198        return userVisibleChangeToken.equals(String.valueOf(modified.getTimeInMillis()));
1199    }
1200
1201    /**
1202     * Gets the legacy change token for the given timestamp.
1203     *
1204     * @param modified the {@code dc:modified} timestamp
1205     * @return the legacy change token
1206     * @since 9.2
1207     */
1208    protected String getLegacyChangeToken(Calendar modified) {
1209        if (modified == null) {
1210            return null;
1211        }
1212        return String.valueOf(modified.getTimeInMillis());
1213    }
1214
1215    /**
1216     * Updates a change token to its new value.
1217     *
1218     * @param changeToken the change token (not {@code null})
1219     * @return the updated change token
1220     * @since 9.2
1221     */
1222    public static Long updateChangeToken(Long changeToken) {
1223        return Long.valueOf(changeToken.longValue() + 1);
1224    }
1225
1226}