001/* 002 * (C) Copyright 2012-2014 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Thomas Roger 018 * Florent Guillaume 019 * Julien Carsique 020 */ 021package org.nuxeo.ecm.csv.core; 022 023import static org.nuxeo.ecm.csv.core.CSVImportLog.Status.ERROR; 024import static org.nuxeo.ecm.csv.core.Constants.CSV_NAME_COL; 025import static org.nuxeo.ecm.csv.core.Constants.CSV_TYPE_COL; 026 027import java.io.BufferedReader; 028import java.io.File; 029import java.io.IOException; 030import java.io.InputStream; 031import java.io.InputStreamReader; 032import java.io.Reader; 033import java.io.Serializable; 034import java.text.DateFormat; 035import java.text.ParseException; 036import java.text.SimpleDateFormat; 037import java.util.ArrayList; 038import java.util.Arrays; 039import java.util.Collections; 040import java.util.Date; 041import java.util.HashMap; 042import java.util.List; 043import java.util.ListIterator; 044import java.util.Map; 045 046import org.apache.commons.csv.CSVFormat; 047import org.apache.commons.csv.CSVParser; 048import org.apache.commons.csv.CSVRecord; 049import org.apache.commons.io.Charsets; 050import org.apache.commons.io.FilenameUtils; 051import org.apache.commons.io.IOUtils; 052import org.apache.commons.io.input.BOMInputStream; 053import org.apache.commons.lang.StringUtils; 054import org.apache.commons.logging.Log; 055import org.apache.commons.logging.LogFactory; 056import org.nuxeo.common.utils.ExceptionUtils; 057import org.nuxeo.common.utils.Path; 058import org.nuxeo.ecm.automation.AutomationService; 059import org.nuxeo.ecm.automation.OperationChain; 060import org.nuxeo.ecm.automation.OperationContext; 061import org.nuxeo.ecm.automation.core.operations.notification.MailTemplateHelper; 062import org.nuxeo.ecm.automation.core.operations.notification.SendMail; 063import org.nuxeo.ecm.automation.core.scripting.Expression; 064import org.nuxeo.ecm.automation.core.scripting.Scripting; 065import org.nuxeo.ecm.automation.core.util.ComplexTypeJSONDecoder; 066import org.nuxeo.ecm.automation.core.util.StringList; 067import org.nuxeo.ecm.core.api.Blob; 068import org.nuxeo.ecm.core.api.Blobs; 069import org.nuxeo.ecm.core.api.DocumentModel; 070import org.nuxeo.ecm.core.api.DocumentRef; 071import org.nuxeo.ecm.core.api.NuxeoException; 072import org.nuxeo.ecm.core.api.NuxeoPrincipal; 073import org.nuxeo.ecm.core.api.PathRef; 074import org.nuxeo.ecm.core.query.sql.NXQL; 075import org.nuxeo.ecm.core.schema.DocumentType; 076import org.nuxeo.ecm.core.schema.SchemaManager; 077import org.nuxeo.ecm.core.schema.types.ComplexType; 078import org.nuxeo.ecm.core.schema.types.CompositeType; 079import org.nuxeo.ecm.core.schema.types.Field; 080import org.nuxeo.ecm.core.schema.types.ListType; 081import org.nuxeo.ecm.core.schema.types.SimpleTypeImpl; 082import org.nuxeo.ecm.core.schema.types.Type; 083import org.nuxeo.ecm.core.schema.types.primitives.BooleanType; 084import org.nuxeo.ecm.core.schema.types.primitives.DateType; 085import org.nuxeo.ecm.core.schema.types.primitives.DoubleType; 086import org.nuxeo.ecm.core.schema.types.primitives.IntegerType; 087import org.nuxeo.ecm.core.schema.types.primitives.LongType; 088import org.nuxeo.ecm.core.schema.types.primitives.StringType; 089import org.nuxeo.ecm.core.transientstore.api.TransientStore; 090import org.nuxeo.ecm.core.transientstore.work.TransientStoreWork; 091import org.nuxeo.ecm.core.work.api.WorkManager; 092import org.nuxeo.ecm.csv.core.CSVImportLog.Status; 093import org.nuxeo.ecm.platform.ec.notification.NotificationEventListener; 094import org.nuxeo.ecm.platform.ec.notification.service.NotificationServiceHelper; 095import org.nuxeo.ecm.platform.types.TypeManager; 096import org.nuxeo.ecm.platform.url.api.DocumentViewCodecManager; 097import org.nuxeo.ecm.platform.url.codec.api.DocumentViewCodec; 098import org.nuxeo.ecm.platform.usermanager.UserManager; 099import org.nuxeo.runtime.api.Framework; 100 101; 102 103/** 104 * Work task to import form a CSV file. Because the file is read from the local filesystem, this must be executed in a 105 * local queue. Since NXP-15252 the CSV reader manages "records", not "lines". 106 * 107 * @since 5.7 108 */ 109public class CSVImporterWork extends TransientStoreWork { 110 111 public static final String NUXEO_CSV_MAIL_TO = "nuxeo.csv.mail.to"; 112 113 public static final String LABEL_CSV_IMPORTER_NOT_EXISTING_FIELD = "label.csv.importer.notExistingField"; 114 115 public static final String LABEL_CSV_IMPORTER_CANNOT_CONVERT_FIELD_VALUE = "label.csv.importer.cannotConvertFieldValue"; 116 117 public static final String LABEL_CSV_IMPORTER_NOT_EXISTING_FILE = "label.csv.importer.notExistingFile"; 118 119 public static final String NUXEO_CSV_BLOBS_FOLDER = "nuxeo.csv.blobs.folder"; 120 121 public static final String LABEL_CSV_IMPORTER_DOCUMENT_ALREADY_EXISTS = "label.csv.importer.documentAlreadyExists"; 122 123 public static final String LABEL_CSV_IMPORTER_UNABLE_TO_UPDATE = "label.csv.importer.unableToUpdate"; 124 125 public static final String LABEL_CSV_IMPORTER_DOCUMENT_UPDATED = "label.csv.importer.documentUpdated"; 126 127 public static final String LABEL_CSV_IMPORTER_UNABLE_TO_CREATE = "label.csv.importer.unableToCreate"; 128 129 public static final String LABEL_CSV_IMPORTER_PARENT_DOES_NOT_EXIST = "label.csv.importer.parentDoesNotExist"; 130 131 public static final String LABEL_CSV_IMPORTER_DOCUMENT_CREATED = "label.csv.importer.documentCreated"; 132 133 public static final String LABEL_CSV_IMPORTER_NOT_ALLOWED_SUB_TYPE = "label.csv.importer.notAllowedSubType"; 134 135 public static final String LABEL_CSV_IMPORTER_UNABLE_TO_SAVE = "label.csv.importer.unableToSave"; 136 137 public static final String LABEL_CSV_IMPORTER_ERROR_IMPORTING_LINE = "label.csv.importer.errorImportingLine"; 138 139 public static final String LABEL_CSV_IMPORTER_NOT_EXISTING_TYPE = "label.csv.importer.notExistingType"; 140 141 public static final String LABEL_CSV_IMPORTER_MISSING_TYPE_VALUE = "label.csv.importer.missingTypeValue"; 142 143 public static final String LABEL_CSV_IMPORTER_MISSING_NAME_VALUE = "label.csv.importer.missingNameValue"; 144 145 public static final String LABEL_CSV_IMPORTER_MISSING_NAME_COLUMN = "label.csv.importer.missingNameColumn"; 146 147 public static final String LABEL_CSV_IMPORTER_EMPTY_FILE = "label.csv.importer.emptyFile"; 148 149 public static final String LABEL_CSV_IMPORTER_ERROR_DURING_IMPORT = "label.csv.importer.errorDuringImport"; 150 151 public static final String LABEL_CSV_IMPORTER_EMPTY_LINE = "label.csv.importer.emptyLine"; 152 153 private static final long serialVersionUID = 1L; 154 155 private static final Log log = LogFactory.getLog(CSVImporterWork.class); 156 157 private static final String TEMPLATE_IMPORT_RESULT = "templates/csvImportResult.ftl"; 158 159 public static final String CATEGORY_CSV_IMPORTER = "csvImporter"; 160 161 public static final String CONTENT_FILED_TYPE_NAME = "content"; 162 163 private static final long COMPUTE_TOTAL_THRESHOLD_KB = 1000; 164 165 /** 166 * CSV headers that won't be checked if the field exists on the document type. 167 * 168 * @since 7.3 169 */ 170 public static List<String> AUTHORIZED_HEADERS = Arrays.asList(NXQL.ECM_LIFECYCLESTATE, NXQL.ECM_UUID); 171 172 protected String parentPath; 173 174 protected String username; 175 176 protected CSVImporterOptions options; 177 178 protected transient DateFormat dateformat; 179 180 protected boolean hasTypeColumn; 181 182 protected Date startDate; 183 184 protected ArrayList<CSVImportLog> importLogs = new ArrayList<>(); 185 186 protected boolean computeTotal = false; 187 188 protected long total = -1L; 189 190 protected long docsCreatedCount; 191 192 public CSVImporterWork(String id) { 193 super(id); 194 } 195 196 public CSVImporterWork(String repositoryName, String parentPath, String username, Blob csvBlob, 197 CSVImporterOptions options) { 198 super(CSVImportId.create(repositoryName, parentPath, csvBlob)); 199 getStore().putBlobs(id, Collections.singletonList(csvBlob)); 200 setDocument(repositoryName, null); 201 setOriginatingUsername(username); 202 this.parentPath = parentPath; 203 this.username = username; 204 if (csvBlob.getLength() >= 0 && csvBlob.getLength() / 1024 < COMPUTE_TOTAL_THRESHOLD_KB) { 205 computeTotal = true; 206 } 207 this.options = options; 208 startDate = new Date(); 209 } 210 211 @Override 212 public String getCategory() { 213 return CATEGORY_CSV_IMPORTER; 214 } 215 216 @Override 217 public String getTitle() { 218 return String.format("CSV import in '%s'", parentPath); 219 } 220 221 public List<CSVImportLog> getImportLogs() { 222 return new ArrayList<>(importLogs); 223 } 224 225 @Override 226 public void work() { 227 TransientStore store = getStore(); 228 setStatus("Importing"); 229 openUserSession(); 230 CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader().withEscape(options.getEscapeCharacter()).withCommentMarker( 231 options.getCommentMarker()); 232 try (Reader in = newReader(getBlob()); CSVParser parser = csvFormat.parse(in)) { 233 doImport(parser); 234 } catch (IOException e) { 235 logError(0, "Error while doing the import: %s", LABEL_CSV_IMPORTER_ERROR_DURING_IMPORT, e.getMessage()); 236 log.debug(e, e); 237 } 238 store.putParameter(id, "logs", importLogs); 239 if (options.sendEmail()) { 240 setStatus("Sending email"); 241 sendMail(); 242 } 243 setStatus(null); 244 } 245 246 @Override 247 public void cleanUp(boolean ok, Exception e) { 248 try { 249 super.cleanUp(ok, e); 250 } finally { 251 getStore().putParameter(id, "status", new CSVImportStatus(CSVImportStatus.State.COMPLETED, total, total)); 252 } 253 } 254 255 static final Serializable EMPTY_LOGS = new ArrayList<CSVImportLog>(); 256 257 String launch() { 258 WorkManager works = Framework.getService(WorkManager.class); 259 260 TransientStore store = getStore(); 261 store.putParameter(id, "logs", EMPTY_LOGS); 262 store.putParameter(id, "status", new CSVImportStatus(CSVImportStatus.State.SCHEDULED)); 263 works.schedule(this); 264 return id; 265 } 266 267 static CSVImportStatus getStatus(String id) { 268 TransientStore store = getStore(); 269 if (!store.exists(id)) { 270 return null; 271 } 272 return (CSVImportStatus) store.getParameter(id, "status"); 273 } 274 275 @SuppressWarnings("unchecked") 276 static List<CSVImportLog> getLastImportLogs(String id) { 277 TransientStore store = getStore(); 278 if (!store.exists(id)) { 279 return Collections.emptyList(); 280 } 281 return (ArrayList<CSVImportLog>) store.getParameter(id, "logs"); 282 } 283 284 /** 285 * @throws IOException 286 * @since 7.3 287 */ 288 protected BufferedReader newReader(Blob blob) throws IOException { 289 return new BufferedReader(new InputStreamReader(new BOMInputStream(blob.getStream()))); 290 } 291 292 protected void doImport(CSVParser parser) { 293 log.info(String.format("Importing CSV file: %s", getBlob().getFilename())); 294 Map<String, Integer> header = parser.getHeaderMap(); 295 if (header == null) { 296 logError(0, "No header line, empty file?", LABEL_CSV_IMPORTER_EMPTY_FILE); 297 return; 298 } 299 if (!header.containsKey(CSV_NAME_COL)) { 300 logError(0, "Missing 'name' column", LABEL_CSV_IMPORTER_MISSING_NAME_COLUMN); 301 return; 302 } 303 hasTypeColumn = header.containsKey(CSV_TYPE_COL); 304 305 try { 306 int batchSize = options.getBatchSize(); 307 Iterable<CSVRecord> it = parser; 308 if (computeTotal) { 309 try { 310 List<CSVRecord> l = parser.getRecords(); 311 total = l.size(); 312 it = l; 313 } catch (IOException e) { 314 log.warn("Could not compute total number of document to be imported"); 315 } 316 } 317 for (CSVRecord record : it) { 318 if (record.size() == 0) { 319 // empty record 320 importLogs.add(new CSVImportLog(getLineNumber(record), Status.SKIPPED, "Empty record", 321 LABEL_CSV_IMPORTER_EMPTY_LINE)); 322 continue; 323 } 324 try { 325 if (importRecord(record, header)) { 326 docsCreatedCount++; 327 getStore().putParameter(id, "status", 328 new CSVImportStatus(CSVImportStatus.State.RUNNING, docsCreatedCount, total)); 329 if (docsCreatedCount % batchSize == 0) { 330 commitOrRollbackTransaction(); 331 startTransaction(); 332 } 333 } 334 } catch (NuxeoException e) { 335 // try next line 336 Throwable unwrappedException = unwrapException(e); 337 logError(getLineNumber(parser), "Error while importing line: %s", 338 LABEL_CSV_IMPORTER_ERROR_IMPORTING_LINE, unwrappedException.getMessage()); 339 log.debug(unwrappedException, unwrappedException); 340 } 341 } 342 343 try { 344 session.save(); 345 } catch (NuxeoException e) { 346 Throwable ue = unwrapException(e); 347 logError(getLineNumber(parser), "Unable to save: %s", LABEL_CSV_IMPORTER_UNABLE_TO_SAVE, 348 ue.getMessage()); 349 log.debug(ue, ue); 350 } 351 } finally { 352 commitOrRollbackTransaction(); 353 startTransaction(); 354 } 355 log.info(String.format("Done importing CSV file: %s", getBlob().getFilename())); 356 } 357 358 /** 359 * Import a line from the CSV file. 360 * 361 * @return {@code true} if a document has been created or updated, {@code false} otherwise. 362 * @since 6.0 363 */ 364 protected boolean importRecord(CSVRecord record, Map<String, Integer> header) { 365 String name = record.get(CSV_NAME_COL); 366 if (StringUtils.isBlank(name)) { 367 log.debug("record.isSet=" + record.isSet(CSV_NAME_COL)); 368 logError(getLineNumber(record), "Missing 'name' value", LABEL_CSV_IMPORTER_MISSING_NAME_VALUE); 369 return false; 370 } 371 372 Path targetPath = new Path(parentPath).append(name); 373 name = targetPath.lastSegment(); 374 String newParentPath = targetPath.removeLastSegments(1).toString(); 375 boolean exists = options.getCSVImporterDocumentFactory().exists(session, newParentPath, name, null); 376 377 DocumentRef docRef = null; 378 String type = null; 379 if (exists) { 380 docRef = new PathRef(targetPath.toString()); 381 type = session.getDocument(docRef).getType(); 382 } else { 383 if (hasTypeColumn) { 384 type = record.get(CSV_TYPE_COL); 385 } 386 if (StringUtils.isBlank(type)) { 387 log.debug("record.isSet=" + record.isSet(CSV_TYPE_COL)); 388 logError(getLineNumber(record), "Missing 'type' value", LABEL_CSV_IMPORTER_MISSING_TYPE_VALUE); 389 return false; 390 } 391 } 392 393 DocumentType docType = Framework.getService(SchemaManager.class).getDocumentType(type); 394 if (docType == null) { 395 logError(getLineNumber(record), "The type '%s' does not exist", LABEL_CSV_IMPORTER_NOT_EXISTING_TYPE, 396 type); 397 return false; 398 } 399 Map<String, Serializable> properties = computePropertiesMap(record, docType, header); 400 if (properties == null) { 401 // skip this line 402 return false; 403 } 404 405 long lineNumber = getLineNumber(record); 406 if (exists) { 407 return updateDocument(lineNumber, docRef, properties); 408 } else { 409 return createDocument(lineNumber, newParentPath, name, type, properties); 410 } 411 } 412 413 // our code expects line numbers to start at 1 for the header and 2 for the line after, 414 // but since commons-csv 1.5 record numbers restart at 1 on the line after the header 415 // thus we need to add 1 416 protected long getLineNumber(CSVRecord record) { 417 return record.getRecordNumber() + 1; 418 } 419 420 protected long getLineNumber(CSVParser parser) { 421 return parser.getRecordNumber() + 1; 422 } 423 424 /** 425 * @since 6.0 426 */ 427 protected Map<String, Serializable> computePropertiesMap(CSVRecord record, CompositeType compositeType, 428 Map<String, Integer> header) { 429 Map<String, Serializable> values = new HashMap<>(); 430 for (String headerValue : header.keySet()) { 431 String lineValue = record.get(headerValue); 432 lineValue = lineValue.trim(); 433 String fieldName = headerValue; 434 if (!CSV_NAME_COL.equals(headerValue) && !CSV_TYPE_COL.equals(headerValue)) { 435 if (AUTHORIZED_HEADERS.contains(headerValue) && !StringUtils.isBlank(lineValue)) { 436 values.put(headerValue, lineValue); 437 } else { 438 if (!compositeType.hasField(fieldName)) { 439 fieldName = fieldName.split(":")[1]; 440 } 441 if (compositeType.hasField(fieldName) && !StringUtils.isBlank(lineValue)) { 442 Serializable convertedValue = convertValue(compositeType, fieldName, headerValue, lineValue, 443 getLineNumber(record)); 444 if (convertedValue == null) { 445 return null; 446 } 447 values.put(headerValue, convertedValue); 448 } 449 } 450 } 451 } 452 return values; 453 } 454 455 @SuppressWarnings("unchecked") 456 protected Serializable convertValue(CompositeType compositeType, String fieldName, String headerValue, 457 String stringValue, long lineNumber) { 458 if (compositeType.hasField(fieldName)) { 459 Field field = compositeType.getField(fieldName); 460 if (field != null) { 461 try { 462 Serializable fieldValue = null; 463 Type fieldType = field.getType(); 464 if (fieldType.isComplexType()) { 465 if (fieldType.getName().equals(CONTENT_FILED_TYPE_NAME)) { 466 fieldValue = (Serializable) createBlobFromFilePath(stringValue); 467 if (fieldValue == null) { 468 logError(lineNumber, "The file '%s' does not exist", 469 LABEL_CSV_IMPORTER_NOT_EXISTING_FILE, stringValue); 470 return null; 471 } 472 } else { 473 fieldValue = (Serializable) ComplexTypeJSONDecoder.decode((ComplexType) fieldType, 474 stringValue); 475 replaceBlobs((Map<String, Object>) fieldValue); 476 } 477 } else { 478 if (fieldType.isListType()) { 479 Type listFieldType = ((ListType) fieldType).getFieldType(); 480 if (listFieldType.isSimpleType()) { 481 /* 482 * Array. 483 */ 484 fieldValue = stringValue.split(options.getListSeparatorRegex()); 485 } else { 486 /* 487 * Complex list. 488 */ 489 fieldValue = (Serializable) ComplexTypeJSONDecoder.decodeList((ListType) fieldType, 490 stringValue); 491 replaceBlobs((List<Object>) fieldValue); 492 } 493 } else { 494 /* 495 * Primitive type. 496 */ 497 Type type = field.getType(); 498 if (type instanceof SimpleTypeImpl) { 499 type = type.getSuperType(); 500 } 501 if (type.isSimpleType()) { 502 if (type instanceof StringType) { 503 fieldValue = stringValue; 504 } else if (type instanceof IntegerType) { 505 fieldValue = Integer.valueOf(stringValue); 506 } else if (type instanceof LongType) { 507 fieldValue = Long.valueOf(stringValue); 508 } else if (type instanceof DoubleType) { 509 fieldValue = Double.valueOf(stringValue); 510 } else if (type instanceof BooleanType) { 511 fieldValue = Boolean.valueOf(stringValue); 512 } else if (type instanceof DateType) { 513 fieldValue = getDateFormat().parse(stringValue); 514 } 515 } 516 } 517 } 518 return fieldValue; 519 } catch (ParseException | NumberFormatException | IOException e) { 520 logError(lineNumber, "Unable to convert field '%s' with value '%s'", 521 LABEL_CSV_IMPORTER_CANNOT_CONVERT_FIELD_VALUE, headerValue, stringValue); 522 log.debug(e, e); 523 } 524 } 525 } else { 526 logError(lineNumber, "Field '%s' does not exist on type '%s'", LABEL_CSV_IMPORTER_NOT_EXISTING_FIELD, 527 headerValue, compositeType.getName()); 528 } 529 return null; 530 } 531 532 /** 533 * Creates a {@code Blob} from a relative file path. The File will be looked up in the folder registered by the 534 * {@code nuxeo.csv.blobs.folder} property. 535 * 536 * @since 9.3 537 */ 538 protected Blob createBlobFromFilePath(String fileRelativePath) throws IOException { 539 String blobsFolderPath = Framework.getProperty(NUXEO_CSV_BLOBS_FOLDER); 540 String path = FilenameUtils.normalize(blobsFolderPath + "/" + fileRelativePath); 541 File file = new File(path); 542 if (file.exists()) { 543 return Blobs.createBlob(file, null, null, FilenameUtils.getName(fileRelativePath)); 544 } else { 545 return null; 546 } 547 } 548 549 /** 550 * Creates a {@code Blob} from a {@code StringBlob}. Assume that the {@code StringBlob} content is the relative file 551 * path. The File will be looked up in the folder registered by the {@code nuxeo.csv.blobs.folder} property. 552 * 553 * @since 9.3 554 */ 555 protected Blob createBlobFromStringBlob(Blob stringBlob) throws IOException { 556 String fileRelativePath = stringBlob.getString(); 557 Blob blob = createBlobFromFilePath(fileRelativePath); 558 if (blob == null) { 559 throw new IOException(String.format("File %s does not exist", fileRelativePath)); 560 } 561 562 blob.setMimeType(stringBlob.getMimeType()); 563 blob.setEncoding(stringBlob.getEncoding()); 564 String filename = stringBlob.getFilename(); 565 if (filename != null) { 566 blob.setFilename(filename); 567 } 568 return blob; 569 } 570 571 /** 572 * Recursively replaces all {@code Blob}s with {@code Blob}s created from Files stored in the folder registered by 573 * the {@code nuxeo.csv.blobs.folder} property. 574 * 575 * @since 9.3 576 */ 577 @SuppressWarnings("unchecked") 578 protected void replaceBlobs(Map<String, Object> map) throws IOException { 579 for (Map.Entry<String, Object> entry : map.entrySet()) { 580 Object value = entry.getValue(); 581 if (value instanceof Blob) { 582 Blob blob = (Blob) value; 583 entry.setValue(createBlobFromStringBlob(blob)); 584 } else if (value instanceof List) { 585 replaceBlobs((List<Object>) value); 586 } else if (value instanceof Map) { 587 replaceBlobs((Map<String, Object>) value); 588 } 589 } 590 } 591 592 /** 593 * Recursively replaces all {@code Blob}s with {@code Blob}s created from Files stored in the folder registered by 594 * the {@code nuxeo.csv.blobs.folder} property. 595 * 596 * @since 9.3 597 */ 598 @SuppressWarnings("unchecked") 599 protected void replaceBlobs(List<Object> list) throws IOException { 600 for (ListIterator<Object> it = list.listIterator(); it.hasNext();) { 601 Object value = it.next(); 602 if (value instanceof Blob) { 603 Blob blob = (Blob) value; 604 it.set(createBlobFromStringBlob(blob)); 605 } else if (value instanceof List) { 606 replaceBlobs((List<Object>) value); 607 } else if (value instanceof Map) { 608 replaceBlobs((Map<String, Object>) value); 609 } 610 } 611 } 612 613 protected DateFormat getDateFormat() { 614 // transient field so may become null 615 if (dateformat == null) { 616 dateformat = new SimpleDateFormat(options.getDateFormat()); 617 } 618 return dateformat; 619 } 620 621 protected boolean createDocument(long lineNumber, String newParentPath, String name, String type, 622 Map<String, Serializable> properties) { 623 try { 624 DocumentRef parentRef = new PathRef(newParentPath); 625 if (session.exists(parentRef)) { 626 DocumentModel parent = session.getDocument(parentRef); 627 628 TypeManager typeManager = Framework.getService(TypeManager.class); 629 if (options.checkAllowedSubTypes() && !typeManager.isAllowedSubType(type, parent.getType())) { 630 logError(lineNumber, "'%s' type is not allowed in '%s'", LABEL_CSV_IMPORTER_NOT_ALLOWED_SUB_TYPE, 631 type, parent.getType()); 632 } else { 633 options.getCSVImporterDocumentFactory().createDocument(session, newParentPath, name, type, 634 properties); 635 importLogs.add(new CSVImportLog(lineNumber, Status.SUCCESS, "Document created", 636 LABEL_CSV_IMPORTER_DOCUMENT_CREATED)); 637 return true; 638 } 639 } else { 640 logError(lineNumber, "Parent document '%s' does not exist", LABEL_CSV_IMPORTER_PARENT_DOES_NOT_EXIST, 641 newParentPath); 642 } 643 } catch (RuntimeException e) { 644 Throwable unwrappedException = unwrapException(e); 645 logError(lineNumber, "Unable to create document: %s", LABEL_CSV_IMPORTER_UNABLE_TO_CREATE, 646 unwrappedException.getMessage()); 647 log.debug(unwrappedException, unwrappedException); 648 } 649 return false; 650 } 651 652 protected boolean updateDocument(long lineNumber, DocumentRef docRef, Map<String, Serializable> properties) { 653 if (options.updateExisting()) { 654 try { 655 options.getCSVImporterDocumentFactory().updateDocument(session, docRef, properties); 656 importLogs.add(new CSVImportLog(lineNumber, Status.SUCCESS, "Document updated", 657 LABEL_CSV_IMPORTER_DOCUMENT_UPDATED)); 658 return true; 659 } catch (RuntimeException e) { 660 Throwable unwrappedException = unwrapException(e); 661 logError(lineNumber, "Unable to update document: %s", LABEL_CSV_IMPORTER_UNABLE_TO_UPDATE, 662 unwrappedException.getMessage()); 663 log.debug(unwrappedException, unwrappedException); 664 } 665 } else { 666 importLogs.add(new CSVImportLog(lineNumber, Status.SKIPPED, "Document already exists", 667 LABEL_CSV_IMPORTER_DOCUMENT_ALREADY_EXISTS)); 668 } 669 return false; 670 } 671 672 protected void logError(long lineNumber, String message, String localizedMessage, String... params) { 673 importLogs.add(new CSVImportLog(lineNumber, ERROR, String.format(message, (Object[]) params), localizedMessage, 674 params)); 675 String lineMessage = String.format("Line %d", lineNumber); 676 String errorMessage = String.format(message, (Object[]) params); 677 log.error(String.format("%s: %s", lineMessage, errorMessage)); 678 getStore().putParameter(id, "status", 679 new CSVImportStatus(CSVImportStatus.State.ERROR, docsCreatedCount, docsCreatedCount)); 680 } 681 682 protected void sendMail() { 683 UserManager userManager = Framework.getService(UserManager.class); 684 NuxeoPrincipal principal = userManager.getPrincipal(username); 685 String email = principal.getEmail(); 686 if (email == null) { 687 log.info(String.format("Not sending import result email to '%s', no email configured", username)); 688 return; 689 } 690 691 try (OperationContext ctx = new OperationContext(session)) { 692 ctx.setInput(session.getRootDocument()); 693 694 CSVImporter csvImporter = Framework.getService(CSVImporter.class); 695 List<CSVImportLog> importerLogs = csvImporter.getImportLogs(getId()); 696 CSVImportResult importResult = CSVImportResult.fromImportLogs(importerLogs); 697 List<CSVImportLog> skippedAndErrorImportLogs = csvImporter.getImportLogs(getId(), Status.SKIPPED, 698 Status.ERROR); 699 ctx.put("importResult", importResult); 700 ctx.put("skippedAndErrorImportLogs", skippedAndErrorImportLogs); 701 ctx.put("csvFilename", getBlob().getFilename()); 702 ctx.put("startDate", DateFormat.getInstance().format(startDate)); 703 ctx.put("username", username); 704 705 DocumentModel importFolder = session.getDocument(new PathRef(parentPath)); 706 String importFolderUrl = getDocumentUrl(importFolder); 707 ctx.put("importFolderTitle", importFolder.getTitle()); 708 ctx.put("importFolderUrl", importFolderUrl); 709 ctx.put("userUrl", getUserUrl()); 710 711 StringList to = buildRecipientsList(email); 712 Expression from = Scripting.newExpression("Env[\"mail.from\"]"); 713 String subject = "CSV Import result of " + getBlob().getFilename(); 714 String message = loadTemplate(TEMPLATE_IMPORT_RESULT); 715 716 OperationChain chain = new OperationChain("SendMail"); 717 chain.add(SendMail.ID) 718 .set("from", from) 719 .set("to", to) 720 .set("HTML", true) 721 .set("subject", subject) 722 .set("message", message); 723 Framework.getService(AutomationService.class).run(ctx, chain); 724 } catch (Exception e) { 725 ExceptionUtils.checkInterrupt(e); 726 log.error(String.format("Unable to notify user '%s' for import result of '%s': %s", username, 727 getBlob().getFilename(), e.getMessage())); 728 log.debug(e, e); 729 throw ExceptionUtils.runtimeException(e); 730 } 731 } 732 733 /** 734 * @since 9.1 735 */ 736 private Blob getBlob() { 737 return getStore().getBlobs(id).get(0); 738 } 739 740 protected String getDocumentUrl(DocumentModel doc) { 741 return MailTemplateHelper.getDocumentUrl(doc, null); 742 } 743 744 protected String getUserUrl() { 745 DocumentViewCodecManager codecService = Framework.getService(DocumentViewCodecManager.class); 746 DocumentViewCodec codec = codecService.getCodec(NotificationEventListener.NOTIFICATION_DOCUMENT_ID_CODEC_NAME); 747 boolean isNotificationCodec = codec != null; 748 boolean isJSFUI = isNotificationCodec 749 && NotificationEventListener.JSF_NOTIFICATION_DOCUMENT_ID_CODEC_PREFIX.equals(codec.getPrefix()); 750 StringBuilder userUrl = new StringBuilder(); 751 if (isNotificationCodec) { 752 753 userUrl.append(NotificationServiceHelper.getNotificationService().getServerUrlPrefix()); 754 if (!isJSFUI) { 755 userUrl.append("ui/"); 756 userUrl.append("#!/"); 757 } 758 userUrl.append("user/").append(username); 759 } 760 return userUrl.toString(); 761 } 762 763 protected StringList buildRecipientsList(String userEmail) { 764 String csvMailTo = Framework.getProperty(NUXEO_CSV_MAIL_TO); 765 if (StringUtils.isBlank(csvMailTo)) { 766 return new StringList(new String[] { userEmail }); 767 } else { 768 return new StringList(new String[] { userEmail, csvMailTo }); 769 } 770 } 771 772 private static String loadTemplate(String key) { 773 InputStream io = CSVImporterWork.class.getClassLoader().getResourceAsStream(key); 774 if (io != null) { 775 try { 776 return IOUtils.toString(io, Charsets.UTF_8); 777 } catch (IOException e) { 778 // cannot happen 779 throw new NuxeoException(e); 780 } finally { 781 try { 782 io.close(); 783 } catch (IOException e) { 784 // nothing to do 785 } 786 } 787 } 788 return null; 789 } 790 791 public static Throwable unwrapException(Throwable t) { 792 Throwable cause = null; 793 if (t != null) { 794 cause = t.getCause(); 795 } 796 if (cause == null) { 797 return t; 798 } else { 799 return unwrapException(cause); 800 } 801 } 802 803}