001/* 002 * (C) Copyright 2017-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.storage.mongodb; 020 021import static org.nuxeo.ecm.core.storage.State.NOP; 022import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_EACH; 023import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_ID; 024import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_INC; 025import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_PUSH; 026import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_SET; 027import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.MONGODB_UNSET; 028import static org.nuxeo.ecm.core.storage.mongodb.MongoDBRepository.ONE; 029 030import java.io.Serializable; 031import java.lang.reflect.Array; 032import java.util.ArrayList; 033import java.util.Arrays; 034import java.util.Calendar; 035import java.util.Date; 036import java.util.HashSet; 037import java.util.List; 038import java.util.Map.Entry; 039import java.util.Set; 040 041import org.bson.Document; 042import org.nuxeo.ecm.core.api.model.Delta; 043import org.nuxeo.ecm.core.storage.State; 044import org.nuxeo.ecm.core.storage.State.ListDiff; 045import org.nuxeo.ecm.core.storage.State.StateDiff; 046 047/** 048 * Converts between MongoDB types (bson) and DBS types (diff, state, list, serializable). 049 * <p> 050 * The MongoDB native "_id" can optionally be translated into a custom id in memory (usually "ecm:id"). Otherwise it is 051 * stripped from returned results. 052 * 053 * @since 9.1 054 */ 055public class MongoDBConverter { 056 057 /** The key to use in memory to map the database native "_id". */ 058 protected final String idKey; 059 060 /** 061 * Constructor for a converter that does not map the MongoDB native "_id". 062 * 063 * @since 10.3 064 */ 065 public MongoDBConverter() { 066 this(null); 067 } 068 069 /** 070 * Constructor for a converter that also knows to optionally translate the native MongoDB "_id" into a custom id. 071 * 072 * @param idKey the key to use to map the native "_id" in memory, if not {@code null} 073 */ 074 public MongoDBConverter(String idKey) { 075 this.idKey = idKey; 076 } 077 078 /** 079 * Constructs a list of MongoDB updates from the given {@link StateDiff}. 080 * <p> 081 * We need a list because some cases need two operations to avoid conflicts. 082 */ 083 public List<Document> diffToBson(StateDiff diff) { 084 UpdateBuilder updateBuilder = new UpdateBuilder(); 085 return updateBuilder.build(diff); 086 } 087 088 public String keyToBson(String key) { 089 if (idKey == null) { 090 return key; 091 } else { 092 return idKey.equals(key) ? MONGODB_ID : key; 093 } 094 } 095 096 public Object valueToBson(Object value) { 097 if (value instanceof State) { 098 return stateToBson((State) value); 099 } else if (value instanceof List) { 100 @SuppressWarnings("unchecked") 101 List<Object> values = (List<Object>) value; 102 return listToBson(values); 103 } else if (value instanceof Object[]) { 104 return listToBson(Arrays.asList((Object[]) value)); 105 } else { 106 return serializableToBson(value); 107 } 108 } 109 110 public Document stateToBson(State state) { 111 Document doc = new Document(); 112 for (Entry<String, Serializable> en : state.entrySet()) { 113 Object val = valueToBson(en.getValue()); 114 if (val != null) { 115 doc.put(keyToBson(en.getKey()), val); 116 } 117 } 118 return doc; 119 } 120 121 public List<Object> listToBson(List<Object> values) { 122 ArrayList<Object> objects = new ArrayList<>(values.size()); 123 for (Object value : values) { 124 objects.add(valueToBson(value)); 125 } 126 return objects; 127 } 128 129 public String bsonToKey(String key) { 130 if (idKey == null) { 131 return key; 132 } else { 133 return MONGODB_ID.equals(key) ? idKey : key; 134 } 135 } 136 137 public State bsonToState(Document doc) { 138 if (doc == null) { 139 return null; 140 } 141 State state = new State(doc.keySet().size()); 142 for (String key : doc.keySet()) { 143 if (idKey == null && MONGODB_ID.equals(key)) { 144 // skip native id if it's not mapped to something 145 continue; 146 } 147 state.put(bsonToKey(key), bsonToValue(doc.get(key))); 148 } 149 return state; 150 } 151 152 public Serializable bsonToValue(Object value) { 153 if (value instanceof List) { 154 @SuppressWarnings("unchecked") 155 List<Object> list = (List<Object>) value; 156 if (list.isEmpty()) { 157 return null; 158 } else { 159 Class<?> klass = Object.class; 160 for (Object o : list) { 161 if (o != null) { 162 klass = scalarToSerializableClass(o.getClass()); 163 break; 164 } 165 } 166 if (Document.class.isAssignableFrom(klass)) { 167 List<Serializable> l = new ArrayList<>(list.size()); 168 for (Object el : list) { 169 l.add(bsonToState((Document) el)); 170 } 171 return (Serializable) l; 172 } else { 173 // turn the list into a properly-typed array 174 Object[] ar = (Object[]) Array.newInstance(klass, list.size()); 175 int i = 0; 176 for (Object el : list) { 177 ar[i++] = scalarToSerializable(el); 178 } 179 return ar; 180 } 181 } 182 } else if (value instanceof Document) { 183 return bsonToState((Document) value); 184 } else { 185 return scalarToSerializable(value); 186 } 187 } 188 189 public Object serializableToBson(Object value) { 190 if (value instanceof Calendar) { 191 return ((Calendar) value).getTime(); 192 } 193 return value; 194 } 195 196 public Serializable scalarToSerializable(Object val) { 197 if (val instanceof Date) { 198 Calendar cal = Calendar.getInstance(); 199 cal.setTime((Date) val); 200 return cal; 201 } 202 return (Serializable) val; 203 } 204 205 public Class<?> scalarToSerializableClass(Class<?> klass) { 206 if (Date.class.isAssignableFrom(klass)) { 207 return Calendar.class; 208 } 209 return klass; 210 } 211 212 213 /** 214 * Update list builder to prevent several updates of the same field. 215 * <p> 216 * This happens if two operations act on two fields where one is a prefix of the other. 217 * <p> 218 * Example: Cannot update 'mylist.0.string' and 'mylist' at the same time (error 16837) 219 * 220 * @since 5.9.5 221 */ 222 public class UpdateBuilder { 223 224 protected final Document set = new Document(); 225 226 protected final Document unset = new Document(); 227 228 protected final Document push = new Document(); 229 230 protected final Document inc = new Document(); 231 232 protected final List<Document> updates = new ArrayList<>(10); 233 234 protected Document update; 235 236 protected Set<String> prefixKeys; 237 238 protected Set<String> keys; 239 240 public List<Document> build(StateDiff diff) { 241 processStateDiff(diff, null); 242 newUpdate(); 243 for (Entry<String, Object> en : set.entrySet()) { 244 update(MONGODB_SET, en.getKey(), en.getValue()); 245 } 246 for (Entry<String, Object> en : unset.entrySet()) { 247 update(MONGODB_UNSET, en.getKey(), en.getValue()); 248 } 249 for (Entry<String, Object> en : push.entrySet()) { 250 update(MONGODB_PUSH, en.getKey(), en.getValue()); 251 } 252 for (Entry<String, Object> en : inc.entrySet()) { 253 update(MONGODB_INC, en.getKey(), en.getValue()); 254 } 255 return updates; 256 } 257 258 protected void processStateDiff(StateDiff diff, String prefix) { 259 String elemPrefix = prefix == null ? "" : prefix + '.'; 260 for (Entry<String, Serializable> en : diff.entrySet()) { 261 String name = elemPrefix + en.getKey(); 262 Serializable value = en.getValue(); 263 if (value instanceof StateDiff) { 264 processStateDiff((StateDiff) value, name); 265 } else if (value instanceof ListDiff) { 266 processListDiff((ListDiff) value, name); 267 } else if (value instanceof Delta) { 268 processDelta((Delta) value, name); 269 } else { 270 // not a diff 271 processValue(name, value); 272 } 273 } 274 } 275 276 protected void processListDiff(ListDiff listDiff, String prefix) { 277 if (listDiff.diff != null) { 278 String elemPrefix = prefix == null ? "" : prefix + '.'; 279 int i = 0; 280 for (Object value : listDiff.diff) { 281 String name = elemPrefix + i; 282 if (value instanceof StateDiff) { 283 processStateDiff((StateDiff) value, name); 284 } else if (value != NOP) { 285 // set value 286 set.put(name, valueToBson(value)); 287 } 288 i++; 289 } 290 } 291 if (listDiff.rpush != null) { 292 Object pushed; 293 if (listDiff.rpush.size() == 1) { 294 // no need to use $each for one element 295 pushed = valueToBson(listDiff.rpush.get(0)); 296 } else { 297 pushed = new Document(MONGODB_EACH, listToBson(listDiff.rpush)); 298 } 299 push.put(prefix, pushed); 300 } 301 } 302 303 protected void processDelta(Delta delta, String prefix) { 304 // MongoDB can $inc a field that doesn't exist, it's treated as 0 BUT it doesn't work on null 305 // so we ensure (in diffToUpdates) that we never store a null but remove the field instead 306 Object incValue = valueToBson(delta.getDeltaValue()); 307 inc.put(prefix, incValue); 308 } 309 310 protected void processValue(String name, Serializable value) { 311 if (value == null) { 312 // for null values, beyond the space saving, 313 // it's important to unset the field instead of setting the value to null 314 // because $inc does not work on nulls but works on non-existent fields 315 unset.put(name, ONE); 316 } else { 317 set.put(name, valueToBson(value)); 318 } 319 } 320 321 protected void newUpdate() { 322 updates.add(update = new Document()); 323 prefixKeys = new HashSet<>(); 324 keys = new HashSet<>(); 325 } 326 327 protected void update(String op, String key, Object value) { 328 checkForConflict(key); 329 Document map = (Document) update.get(op); 330 if (map == null) { 331 update.put(op, map = new Document()); 332 } 333 map.put(key, value); 334 } 335 336 /** 337 * Checks if the key conflicts with one of the previous keys. 338 * <p> 339 * A conflict occurs if one key is equals to or is a prefix of the other. 340 */ 341 protected void checkForConflict(String key) { 342 List<String> pKeys = getPrefixKeys(key); 343 if (conflictKeys(key, pKeys)) { 344 newUpdate(); 345 } 346 prefixKeys.addAll(pKeys); 347 keys.add(key); 348 } 349 350 protected boolean conflictKeys(String key, List<String> subkeys) { 351 if (prefixKeys.contains(key)) { 352 return true; 353 } 354 for (String sk: subkeys) { 355 if (keys.contains(sk)) { 356 return true; 357 } 358 } 359 return false; 360 } 361 362 /** 363 * return a list of parents key 364 * foo.0.bar -> [foo, foo.0, foo.0.bar] 365 */ 366 protected List<String> getPrefixKeys(String key) { 367 List<String> ret = new ArrayList<>(10); 368 int i=0; 369 while ((i = key.indexOf('.', i)) > 0) { 370 ret.add(key.substring(0, i++)); 371 } 372 ret.add(key); 373 return ret; 374 } 375 376 } 377 378}