001/* 002 * (C) Copyright 2017-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.storage.mongodb; 020 021import static java.lang.Boolean.FALSE; 022import static org.nuxeo.ecm.core.storage.State.NOP; 023import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_EACH; 024import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID; 025import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_INC; 026import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_PULLALL; 027import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_PUSH; 028import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_SET; 029import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_UNSET; 030import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.ONE; 031 032import java.io.Serializable; 033import java.lang.reflect.Array; 034import java.util.ArrayList; 035import java.util.Arrays; 036import java.util.Calendar; 037import java.util.Collection; 038import java.util.Date; 039import java.util.HashSet; 040import java.util.List; 041import java.util.Map.Entry; 042import java.util.Set; 043import java.util.regex.Pattern; 044 045import org.bson.Document; 046import org.bson.conversions.Bson; 047import org.nuxeo.ecm.core.api.model.Delta; 048import org.nuxeo.ecm.core.storage.State; 049import org.nuxeo.ecm.core.storage.State.ListDiff; 050import org.nuxeo.ecm.core.storage.State.StateDiff; 051 052import com.mongodb.client.model.Filters; 053 054/** 055 * Converts between MongoDB types (bson) and DBS types (diff, state, list, serializable). 056 * <p> 057 * The MongoDB native "_id" can optionally be translated into a custom id in memory (usually "ecm:id"). Otherwise it is 058 * stripped from returned results. 059 * 060 * @since 9.1 061 */ 062public class MongoDBConverter { 063 064 /** The key to use in memory to map the database native "_id". */ 065 protected final String idKey; 066 067 /** The keys for booleans whose value is true or null (instead of false). */ 068 protected final Set<String> trueOrNullBooleanKeys; 069 070 /** The keys whose values are ids and are stored as longs. */ 071 protected final Set<String> idValuesKeys; 072 073 /** 074 * Constructor for a converter that does not map the MongoDB native "_id". 075 * 076 * @since 10.3 077 */ 078 public MongoDBConverter() { 079 this(null, Set.of(), Set.of()); 080 } 081 082 /** 083 * Constructor for a converter that also knows to optionally translate the native MongoDB "_id" into a custom id. 084 * <p> 085 * When {@code idValuesKeys} are provided, the ids are stored as longs. 086 * 087 * @param idKey the key to use to map the native "_id" in memory, if not {@code null} 088 * @param trueOrNullBooleanKeys the keys corresponding to boolean values that are only true or null (instead of 089 * false) 090 * @param idValuesKeys the keys corresponding to values that are ids 091 */ 092 public MongoDBConverter(String idKey, Set<String> trueOrNullBooleanKeys, Set<String> idValuesKeys) { 093 this.idKey = idKey; 094 this.trueOrNullBooleanKeys = trueOrNullBooleanKeys; 095 this.idValuesKeys = idValuesKeys; 096 } 097 098 /** 099 * Constructs a list of MongoDB updates from the given {@link StateDiff}. 100 * <p> 101 * We need a list because some cases need two operations to avoid conflicts. 102 */ 103 public List<Document> diffToBson(StateDiff diff) { 104 UpdateBuilder updateBuilder = new UpdateBuilder(); 105 return updateBuilder.build(diff); 106 } 107 108 public void putToBson(Document doc, String key, Object value) { 109 doc.put(keyToBson(key), valueToBson(key, value)); 110 } 111 112 public String keyToBson(String key) { 113 if (idKey == null) { 114 return key; 115 } else { 116 return idKey.equals(key) ? MONGODB_ID : key; 117 } 118 } 119 120 public Object valueToBson(String key, Object value) { 121 if (value instanceof State) { 122 return stateToBson((State) value); 123 } else if (value instanceof List) { 124 @SuppressWarnings("unchecked") 125 List<Object> values = (List<Object>) value; 126 return listToBson(key, values); 127 } else if (value instanceof Object[]) { 128 return listToBson(key, Arrays.asList((Object[]) value)); 129 } else { 130 return serializableToBson(key, value); 131 } 132 } 133 134 public Document stateToBson(State state) { 135 Document doc = new Document(); 136 for (Entry<String, Serializable> en : state.entrySet()) { 137 Serializable value = en.getValue(); 138 if (value != null) { 139 putToBson(doc, en.getKey(), value); 140 } 141 } 142 return doc; 143 } 144 145 public <T> List<Object> listToBson(String key, Collection<T> values) { 146 ArrayList<Object> objects = new ArrayList<>(values.size()); 147 for (T value : values) { 148 objects.add(valueToBson(key, value)); 149 } 150 return objects; 151 } 152 153 public Bson filterEq(String key, Object value) { 154 return Filters.eq(keyToBson(key), valueToBson(key, value)); 155 } 156 157 public <T> Bson filterIn(String key, Collection<T> values) { 158 return Filters.in(keyToBson(key), listToBson(key, values)); 159 } 160 161 public Serializable getFromBson(Document doc, String bsonKey, String key) { 162 return bsonToValue(key, doc.get(bsonKey)); 163 } 164 165 public String bsonToKey(String key) { 166 if (idKey == null) { 167 return key; 168 } else { 169 return MONGODB_ID.equals(key) ? idKey : key; 170 } 171 } 172 173 public State bsonToState(Document doc) { 174 if (doc == null) { 175 return null; 176 } 177 State state = new State(doc.keySet().size()); 178 for (String bsonKey : doc.keySet()) { 179 if (idKey == null && MONGODB_ID.equals(bsonKey)) { 180 // skip native id if it's not mapped to something 181 continue; 182 } 183 String key = bsonToKey(bsonKey); 184 state.put(key, getFromBson(doc, bsonKey, key)); 185 } 186 return state; 187 } 188 189 public Serializable bsonToValue(String key, Object value) { 190 if (value instanceof List) { 191 @SuppressWarnings("unchecked") 192 List<Object> list = (List<Object>) value; 193 if (list.isEmpty()) { 194 return null; 195 } else { 196 Class<?> klass = Object.class; 197 for (Object o : list) { 198 if (o != null) { 199 klass = bsonToSerializableClass(key, o.getClass()); 200 break; 201 } 202 } 203 if (Document.class.isAssignableFrom(klass)) { 204 List<Serializable> l = new ArrayList<>(list.size()); 205 for (Object el : list) { 206 l.add(bsonToState((Document) el)); 207 } 208 return (Serializable) l; 209 } else { 210 // turn the list into a properly-typed array 211 Object[] ar = (Object[]) Array.newInstance(klass, list.size()); 212 int i = 0; 213 for (Object el : list) { 214 ar[i++] = bsonToSerializable(key, el); 215 } 216 return ar; 217 } 218 } 219 } else if (value instanceof Document) { 220 return bsonToState((Document) value); 221 } else { 222 return bsonToSerializable(key, value); 223 } 224 } 225 226 protected boolean valueIsId(String key) { 227 return key != null && idValuesKeys.contains(key); 228 } 229 230 public Object serializableToBson(String key, Object value) { 231 if (value instanceof Calendar) { 232 return ((Calendar) value).getTime(); 233 } 234 if (valueIsId(key)) { 235 return idToBson(value); 236 } 237 if (FALSE.equals(value) && key != null && trueOrNullBooleanKeys.contains(key)) { 238 return null; 239 } 240 return value; 241 } 242 243 public Serializable bsonToSerializable(String key, Object val) { 244 if (val instanceof Date) { 245 Calendar cal = Calendar.getInstance(); 246 cal.setTime((Date) val); 247 return cal; 248 } 249 if (valueIsId(key)) { 250 return bsonToId(val); 251 } 252 return (Serializable) val; 253 } 254 255 public Class<?> bsonToSerializableClass(String key, Class<?> klass) { 256 if (Date.class.isAssignableFrom(klass)) { 257 return Calendar.class; 258 } 259 if (valueIsId(key)) { 260 return String.class; 261 } 262 return klass; 263 } 264 265 // exactly 16 chars in lowercase hex 266 protected static final Pattern HEX_RE = Pattern.compile("[0-9a-f]{16}"); 267 268 // convert hex id to long 269 protected Object idToBson(Object value) { 270 if (value == null) { 271 return null; 272 } 273 try { 274 String string = (String) value; 275 if (!HEX_RE.matcher(string).matches()) { 276 throw new NumberFormatException(string); 277 } 278 return Long.parseUnsignedLong(string, 16); 279 } catch (ClassCastException | NumberFormatException e) { 280 return "__invalid_id__" + value; 281 } 282 } 283 284 // convert long to hex id 285 protected String bsonToId(Object val) { 286 if (val == null) { 287 return null; 288 } 289 try { 290 String hex = Long.toHexString((Long) val); 291 int nz = 16 - hex.length(); 292 if (nz > 0) { 293 hex = "0".repeat(nz) + hex; 294 } 295 return hex; 296 } catch (ClassCastException e) { 297 return "__invalid_id__" + val; 298 } 299 } 300 301 /** 302 * Update list builder to prevent several updates of the same field. 303 * <p> 304 * This happens if two operations act on two fields where one is a prefix of the other. 305 * <p> 306 * Example: Cannot update 'mylist.0.string' and 'mylist' at the same time (error 16837) 307 * 308 * @since 5.9.5 309 */ 310 public class UpdateBuilder { 311 312 protected final Document set = new Document(); 313 314 protected final Document unset = new Document(); 315 316 protected final Document push = new Document(); 317 318 protected final Document pull = new Document(); 319 320 protected final Document inc = new Document(); 321 322 protected final List<Document> updates = new ArrayList<>(10); 323 324 protected Document update; 325 326 protected Set<String> prefixKeys; 327 328 protected Set<String> keys; 329 330 public List<Document> build(StateDiff diff) { 331 processStateDiff(diff, null); 332 newUpdate(); 333 for (Entry<String, Object> en : set.entrySet()) { 334 update(MONGODB_SET, en.getKey(), en.getValue()); 335 } 336 for (Entry<String, Object> en : unset.entrySet()) { 337 update(MONGODB_UNSET, en.getKey(), en.getValue()); 338 } 339 for (Entry<String, Object> en : push.entrySet()) { 340 update(MONGODB_PUSH, en.getKey(), en.getValue()); 341 } 342 for (Entry<String, Object> en : pull.entrySet()) { 343 update(MONGODB_PULLALL, en.getKey(), en.getValue()); 344 } 345 for (Entry<String, Object> en : inc.entrySet()) { 346 update(MONGODB_INC, en.getKey(), en.getValue()); 347 } 348 return updates; 349 } 350 351 protected void processStateDiff(StateDiff diff, String prefix) { 352 String elemPrefix = prefix == null ? "" : prefix + '.'; 353 for (Entry<String, Serializable> en : diff.entrySet()) { 354 String name = elemPrefix + en.getKey(); 355 Serializable value = en.getValue(); 356 if (value instanceof StateDiff) { 357 processStateDiff((StateDiff) value, name); 358 } else if (value instanceof ListDiff) { 359 processListDiff((ListDiff) value, name); 360 } else if (value instanceof Delta) { 361 processDelta((Delta) value, name); 362 } else { 363 // not a diff 364 processValue(name, value); 365 } 366 } 367 } 368 369 protected void processListDiff(ListDiff listDiff, String prefix) { 370 if (listDiff.diff != null) { 371 String elemPrefix = prefix == null ? "" : prefix + '.'; 372 int i = 0; 373 for (Object value : listDiff.diff) { 374 String name = elemPrefix + i; 375 if (value instanceof StateDiff) { 376 processStateDiff((StateDiff) value, name); 377 } else if (value != NOP) { 378 // set value 379 set.put(name, valueToBson(prefix, value)); 380 } 381 i++; 382 } 383 } 384 if (listDiff.rpush != null) { 385 Object pushed; 386 if (listDiff.rpush.size() == 1) { 387 // no need to use $each for one element 388 pushed = valueToBson(prefix, listDiff.rpush.get(0)); 389 } else { 390 pushed = new Document(MONGODB_EACH, listToBson(prefix, listDiff.rpush)); 391 } 392 push.put(prefix, pushed); 393 } 394 if (listDiff.pull != null) { 395 pull.put(prefix, valueToBson(prefix, listDiff.pull)); 396 } 397 } 398 399 protected void processDelta(Delta delta, String prefix) { 400 // MongoDB can $inc a field that doesn't exist, it's treated as 0 BUT it doesn't work on null 401 // so we ensure (in diffToUpdates) that we never store a null but remove the field instead 402 Object incValue = valueToBson(prefix, delta.getDeltaValue()); 403 inc.put(prefix, incValue); 404 } 405 406 protected void processValue(String name, Serializable value) { 407 if (value == null) { 408 // for null values, beyond the space saving, 409 // it's important to unset the field instead of setting the value to null 410 // because $inc does not work on nulls but works on non-existent fields 411 unset.put(name, ONE); 412 } else { 413 set.put(name, valueToBson(name, value)); 414 } 415 } 416 417 protected void newUpdate() { 418 updates.add(update = new Document()); 419 prefixKeys = new HashSet<>(); 420 keys = new HashSet<>(); 421 } 422 423 protected void update(String op, String bsonKey, Object val) { 424 checkForConflict(bsonKey); 425 Document map = (Document) update.get(op); 426 if (map == null) { 427 update.put(op, map = new Document()); 428 } 429 map.put(bsonKey, val); 430 } 431 432 /** 433 * Checks if the key conflicts with one of the previous keys. 434 * <p> 435 * A conflict occurs if one key is equals to or is a prefix of the other. 436 */ 437 protected void checkForConflict(String key) { 438 List<String> pKeys = getPrefixKeys(key); 439 if (conflictKeys(key, pKeys)) { 440 newUpdate(); 441 } 442 prefixKeys.addAll(pKeys); 443 keys.add(key); 444 } 445 446 protected boolean conflictKeys(String key, List<String> subkeys) { 447 if (prefixKeys.contains(key)) { 448 return true; 449 } 450 for (String sk : subkeys) { 451 if (keys.contains(sk)) { 452 return true; 453 } 454 } 455 return false; 456 } 457 458 /** 459 * return a list of parents key 460 * <p> 461 * foo.0.bar -> [foo, foo.0, foo.0.bar] 462 */ 463 protected List<String> getPrefixKeys(String key) { 464 List<String> ret = new ArrayList<>(10); 465 int i = 0; 466 while ((i = key.indexOf('.', i)) > 0) { 467 ret.add(key.substring(0, i++)); 468 } 469 ret.add(key); 470 return ret; 471 } 472 473 } 474 475}