001/*
002 * (C) Copyright 2017-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.storage.mongodb;
020
021import static java.lang.Boolean.FALSE;
022import static org.nuxeo.ecm.core.storage.State.NOP;
023import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_EACH;
024import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID;
025import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_INC;
026import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_PULLALL;
027import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_PUSH;
028import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_SET;
029import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_UNSET;
030import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.ONE;
031
032import java.io.Serializable;
033import java.lang.reflect.Array;
034import java.util.ArrayList;
035import java.util.Arrays;
036import java.util.Calendar;
037import java.util.Collection;
038import java.util.Date;
039import java.util.HashSet;
040import java.util.List;
041import java.util.Map.Entry;
042import java.util.Set;
043import java.util.regex.Pattern;
044
045import org.bson.Document;
046import org.bson.conversions.Bson;
047import org.nuxeo.ecm.core.api.model.Delta;
048import org.nuxeo.ecm.core.storage.State;
049import org.nuxeo.ecm.core.storage.State.ListDiff;
050import org.nuxeo.ecm.core.storage.State.StateDiff;
051
052import com.mongodb.client.model.Filters;
053
054/**
055 * Converts between MongoDB types (bson) and DBS types (diff, state, list, serializable).
056 * <p>
057 * The MongoDB native "_id" can optionally be translated into a custom id in memory (usually "ecm:id"). Otherwise it is
058 * stripped from returned results.
059 *
060 * @since 9.1
061 */
062public class MongoDBConverter {
063
064    /** The key to use in memory to map the database native "_id". */
065    protected final String idKey;
066
067    /** The keys for booleans whose value is true or null (instead of false). */
068    protected final Set<String> trueOrNullBooleanKeys;
069
070    /** The keys whose values are ids and are stored as longs. */
071    protected final Set<String> idValuesKeys;
072
073    /**
074     * Constructor for a converter that does not map the MongoDB native "_id".
075     *
076     * @since 10.3
077     */
078    public MongoDBConverter() {
079        this(null, Set.of(), Set.of());
080    }
081
082    /**
083     * Constructor for a converter that also knows to optionally translate the native MongoDB "_id" into a custom id.
084     * <p>
085     * When {@code idValuesKeys} are provided, the ids are stored as longs.
086     *
087     * @param idKey the key to use to map the native "_id" in memory, if not {@code null}
088     * @param trueOrNullBooleanKeys the keys corresponding to boolean values that are only true or null (instead of
089     *            false)
090     * @param idValuesKeys the keys corresponding to values that are ids
091     */
092    public MongoDBConverter(String idKey, Set<String> trueOrNullBooleanKeys, Set<String> idValuesKeys) {
093        this.idKey = idKey;
094        this.trueOrNullBooleanKeys = trueOrNullBooleanKeys;
095        this.idValuesKeys = idValuesKeys;
096    }
097
098    /**
099     * Constructs a list of MongoDB updates from the given {@link StateDiff}.
100     * <p>
101     * We need a list because some cases need two operations to avoid conflicts.
102     */
103    public List<Document> diffToBson(StateDiff diff) {
104        UpdateBuilder updateBuilder = new UpdateBuilder();
105        return updateBuilder.build(diff);
106    }
107
108    public void putToBson(Document doc, String key, Object value) {
109        doc.put(keyToBson(key), valueToBson(key, value));
110    }
111
112    public String keyToBson(String key) {
113        if (idKey == null) {
114            return key;
115        } else {
116            return idKey.equals(key) ? MONGODB_ID : key;
117        }
118    }
119
120    public Object valueToBson(String key, Object value) {
121        if (value instanceof State) {
122            return stateToBson((State) value);
123        } else if (value instanceof List) {
124            @SuppressWarnings("unchecked")
125            List<Object> values = (List<Object>) value;
126            return listToBson(key, values);
127        } else if (value instanceof Object[]) {
128            return listToBson(key, Arrays.asList((Object[]) value));
129        } else {
130            return serializableToBson(key, value);
131        }
132    }
133
134    public Document stateToBson(State state) {
135        Document doc = new Document();
136        for (Entry<String, Serializable> en : state.entrySet()) {
137            Serializable value = en.getValue();
138            if (value != null) {
139                putToBson(doc, en.getKey(), value);
140            }
141        }
142        return doc;
143    }
144
145    public <T> List<Object> listToBson(String key, Collection<T> values) {
146        ArrayList<Object> objects = new ArrayList<>(values.size());
147        for (T value : values) {
148            objects.add(valueToBson(key, value));
149        }
150        return objects;
151    }
152
153    public Bson filterEq(String key, Object value) {
154        return Filters.eq(keyToBson(key), valueToBson(key, value));
155    }
156
157    public <T> Bson filterIn(String key, Collection<T> values) {
158        return Filters.in(keyToBson(key), listToBson(key, values));
159    }
160
161    public Serializable getFromBson(Document doc, String bsonKey, String key) {
162        return bsonToValue(key, doc.get(bsonKey));
163    }
164
165    public String bsonToKey(String key) {
166        if (idKey == null) {
167            return key;
168        } else {
169            return MONGODB_ID.equals(key) ? idKey : key;
170        }
171    }
172
173    public State bsonToState(Document doc) {
174        if (doc == null) {
175            return null;
176        }
177        State state = new State(doc.keySet().size());
178        for (String bsonKey : doc.keySet()) {
179            if (idKey == null && MONGODB_ID.equals(bsonKey)) {
180                // skip native id if it's not mapped to something
181                continue;
182            }
183            String key = bsonToKey(bsonKey);
184            state.put(key, getFromBson(doc, bsonKey, key));
185        }
186        return state;
187    }
188
189    public Serializable bsonToValue(String key, Object value) {
190        if (value instanceof List) {
191            @SuppressWarnings("unchecked")
192            List<Object> list = (List<Object>) value;
193            if (list.isEmpty()) {
194                return null;
195            } else {
196                Class<?> klass = Object.class;
197                for (Object o : list) {
198                    if (o != null) {
199                        klass = bsonToSerializableClass(key, o.getClass());
200                        break;
201                    }
202                }
203                if (Document.class.isAssignableFrom(klass)) {
204                    List<Serializable> l = new ArrayList<>(list.size());
205                    for (Object el : list) {
206                        l.add(bsonToState((Document) el));
207                    }
208                    return (Serializable) l;
209                } else {
210                    // turn the list into a properly-typed array
211                    Object[] ar = (Object[]) Array.newInstance(klass, list.size());
212                    int i = 0;
213                    for (Object el : list) {
214                        ar[i++] = bsonToSerializable(key, el);
215                    }
216                    return ar;
217                }
218            }
219        } else if (value instanceof Document) {
220            return bsonToState((Document) value);
221        } else {
222            return bsonToSerializable(key, value);
223        }
224    }
225
226    protected boolean valueIsId(String key) {
227        return key != null && idValuesKeys.contains(key);
228    }
229
230    public Object serializableToBson(String key, Object value) {
231        if (value instanceof Calendar) {
232            return ((Calendar) value).getTime();
233        }
234        if (valueIsId(key)) {
235            return idToBson(value);
236        }
237        if (FALSE.equals(value) && key != null && trueOrNullBooleanKeys.contains(key)) {
238            return null;
239        }
240        return value;
241    }
242
243    public Serializable bsonToSerializable(String key, Object val) {
244        if (val instanceof Date) {
245            Calendar cal = Calendar.getInstance();
246            cal.setTime((Date) val);
247            return cal;
248        }
249        if (valueIsId(key)) {
250            return bsonToId(val);
251        }
252        return (Serializable) val;
253    }
254
255    public Class<?> bsonToSerializableClass(String key, Class<?> klass) {
256        if (Date.class.isAssignableFrom(klass)) {
257            return Calendar.class;
258        }
259        if (valueIsId(key)) {
260            return String.class;
261        }
262        return klass;
263    }
264
265    // exactly 16 chars in lowercase hex
266    protected static final Pattern HEX_RE = Pattern.compile("[0-9a-f]{16}");
267
268    // convert hex id to long
269    protected Object idToBson(Object value) {
270        if (value == null) {
271            return null;
272        }
273        try {
274            String string = (String) value;
275            if (!HEX_RE.matcher(string).matches()) {
276                throw new NumberFormatException(string);
277            }
278            return Long.parseUnsignedLong(string, 16);
279        } catch (ClassCastException | NumberFormatException e) {
280            return "__invalid_id__" + value;
281        }
282    }
283
284    // convert long to hex id
285    protected String bsonToId(Object val) {
286        if (val == null) {
287            return null;
288        }
289        try {
290            String hex = Long.toHexString((Long) val);
291            int nz = 16 - hex.length();
292            if (nz > 0) {
293                hex = "0".repeat(nz) + hex;
294            }
295            return hex;
296        } catch (ClassCastException e) {
297            return "__invalid_id__" + val;
298        }
299    }
300
301    /**
302     * Update list builder to prevent several updates of the same field.
303     * <p>
304     * This happens if two operations act on two fields where one is a prefix of the other.
305     * <p>
306     * Example: Cannot update 'mylist.0.string' and 'mylist' at the same time (error 16837)
307     *
308     * @since 5.9.5
309     */
310    public class UpdateBuilder {
311
312        protected final Document set = new Document();
313
314        protected final Document unset = new Document();
315
316        protected final Document push = new Document();
317
318        protected final Document pull = new Document();
319
320        protected final Document inc = new Document();
321
322        protected final List<Document> updates = new ArrayList<>(10);
323
324        protected Document update;
325
326        protected Set<String> prefixKeys;
327
328        protected Set<String> keys;
329
330        public List<Document> build(StateDiff diff) {
331            processStateDiff(diff, null);
332            newUpdate();
333            for (Entry<String, Object> en : set.entrySet()) {
334                update(MONGODB_SET, en.getKey(), en.getValue());
335            }
336            for (Entry<String, Object> en : unset.entrySet()) {
337                update(MONGODB_UNSET, en.getKey(), en.getValue());
338            }
339            for (Entry<String, Object> en : push.entrySet()) {
340                update(MONGODB_PUSH, en.getKey(), en.getValue());
341            }
342            for (Entry<String, Object> en : pull.entrySet()) {
343                update(MONGODB_PULLALL, en.getKey(), en.getValue());
344            }
345            for (Entry<String, Object> en : inc.entrySet()) {
346                update(MONGODB_INC, en.getKey(), en.getValue());
347            }
348            return updates;
349        }
350
351        protected void processStateDiff(StateDiff diff, String prefix) {
352            String elemPrefix = prefix == null ? "" : prefix + '.';
353            for (Entry<String, Serializable> en : diff.entrySet()) {
354                String name = elemPrefix + en.getKey();
355                Serializable value = en.getValue();
356                if (value instanceof StateDiff) {
357                    processStateDiff((StateDiff) value, name);
358                } else if (value instanceof ListDiff) {
359                    processListDiff((ListDiff) value, name);
360                } else if (value instanceof Delta) {
361                    processDelta((Delta) value, name);
362                } else {
363                    // not a diff
364                    processValue(name, value);
365                }
366            }
367        }
368
369        protected void processListDiff(ListDiff listDiff, String prefix) {
370            if (listDiff.diff != null) {
371                String elemPrefix = prefix == null ? "" : prefix + '.';
372                int i = 0;
373                for (Object value : listDiff.diff) {
374                    String name = elemPrefix + i;
375                    if (value instanceof StateDiff) {
376                        processStateDiff((StateDiff) value, name);
377                    } else if (value != NOP) {
378                        // set value
379                        set.put(name, valueToBson(prefix, value));
380                    }
381                    i++;
382                }
383            }
384            if (listDiff.rpush != null) {
385                Object pushed;
386                if (listDiff.rpush.size() == 1) {
387                    // no need to use $each for one element
388                    pushed = valueToBson(prefix, listDiff.rpush.get(0));
389                } else {
390                    pushed = new Document(MONGODB_EACH, listToBson(prefix, listDiff.rpush));
391                }
392                push.put(prefix, pushed);
393            }
394            if (listDiff.pull != null) {
395                pull.put(prefix, valueToBson(prefix, listDiff.pull));
396            }
397        }
398
399        protected void processDelta(Delta delta, String prefix) {
400            // MongoDB can $inc a field that doesn't exist, it's treated as 0 BUT it doesn't work on null
401            // so we ensure (in diffToUpdates) that we never store a null but remove the field instead
402            Object incValue = valueToBson(prefix, delta.getDeltaValue());
403            inc.put(prefix, incValue);
404        }
405
406        protected void processValue(String name, Serializable value) {
407            if (value == null) {
408                // for null values, beyond the space saving,
409                // it's important to unset the field instead of setting the value to null
410                // because $inc does not work on nulls but works on non-existent fields
411                unset.put(name, ONE);
412            } else {
413                set.put(name, valueToBson(name, value));
414            }
415        }
416
417        protected void newUpdate() {
418            updates.add(update = new Document());
419            prefixKeys = new HashSet<>();
420            keys = new HashSet<>();
421        }
422
423        protected void update(String op, String bsonKey, Object val) {
424            checkForConflict(bsonKey);
425            Document map = (Document) update.get(op);
426            if (map == null) {
427                update.put(op, map = new Document());
428            }
429            map.put(bsonKey, val);
430        }
431
432        /**
433         * Checks if the key conflicts with one of the previous keys.
434         * <p>
435         * A conflict occurs if one key is equals to or is a prefix of the other.
436         */
437        protected void checkForConflict(String key) {
438            List<String> pKeys = getPrefixKeys(key);
439            if (conflictKeys(key, pKeys)) {
440                newUpdate();
441            }
442            prefixKeys.addAll(pKeys);
443            keys.add(key);
444        }
445
446        protected boolean conflictKeys(String key, List<String> subkeys) {
447            if (prefixKeys.contains(key)) {
448                return true;
449            }
450            for (String sk : subkeys) {
451                if (keys.contains(sk)) {
452                    return true;
453                }
454            }
455            return false;
456        }
457
458        /**
459         * return a list of parents key
460         * <p>
461         * foo.0.bar -&gt; [foo, foo.0, foo.0.bar]
462         */
463        protected List<String> getPrefixKeys(String key) {
464            List<String> ret = new ArrayList<>(10);
465            int i = 0;
466            while ((i = key.indexOf('.', i)) > 0) {
467                ret.add(key.substring(0, i++));
468            }
469            ret.add(key);
470            return ret;
471        }
472
473    }
474
475}