001/* 002 * (C) Copyright 2012-2019 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Thomas Roger 018 * Florent Guillaume 019 * Julien Carsique 020 */ 021package org.nuxeo.ecm.csv.core; 022 023import static java.nio.charset.StandardCharsets.UTF_8; 024import static org.nuxeo.ecm.csv.core.CSVImportLog.Status.ERROR; 025import static org.nuxeo.ecm.csv.core.Constants.CSV_NAME_COL; 026import static org.nuxeo.ecm.csv.core.Constants.CSV_TYPE_COL; 027 028import java.io.BufferedReader; 029import java.io.File; 030import java.io.IOException; 031import java.io.InputStream; 032import java.io.InputStreamReader; 033import java.io.Reader; 034import java.io.Serializable; 035import java.text.DateFormat; 036import java.text.ParseException; 037import java.util.ArrayList; 038import java.util.Arrays; 039import java.util.Collections; 040import java.util.Date; 041import java.util.HashMap; 042import java.util.List; 043import java.util.ListIterator; 044import java.util.Map; 045 046import org.apache.commons.csv.CSVFormat; 047import org.apache.commons.csv.CSVParser; 048import org.apache.commons.csv.CSVRecord; 049import org.apache.commons.io.FilenameUtils; 050import org.apache.commons.io.IOUtils; 051import org.apache.commons.io.input.BOMInputStream; 052import org.apache.commons.lang3.StringUtils; 053import org.apache.logging.log4j.LogManager; 054import org.apache.logging.log4j.Logger; 055import org.nuxeo.common.utils.ExceptionUtils; 056import org.nuxeo.common.utils.Path; 057import org.nuxeo.ecm.automation.AutomationService; 058import org.nuxeo.ecm.automation.OperationChain; 059import org.nuxeo.ecm.automation.OperationContext; 060import org.nuxeo.ecm.automation.core.operations.notification.MailTemplateHelper; 061import org.nuxeo.ecm.automation.core.operations.notification.SendMail; 062import org.nuxeo.ecm.automation.core.scripting.Expression; 063import org.nuxeo.ecm.automation.core.scripting.Scripting; 064import org.nuxeo.ecm.automation.core.util.ComplexTypeJSONDecoder; 065import org.nuxeo.ecm.automation.core.util.StringList; 066import org.nuxeo.ecm.core.api.Blob; 067import org.nuxeo.ecm.core.api.Blobs; 068import org.nuxeo.ecm.core.api.DocumentModel; 069import org.nuxeo.ecm.core.api.DocumentRef; 070import org.nuxeo.ecm.core.api.NuxeoException; 071import org.nuxeo.ecm.core.api.NuxeoPrincipal; 072import org.nuxeo.ecm.core.api.PathRef; 073import org.nuxeo.ecm.core.query.sql.NXQL; 074import org.nuxeo.ecm.core.schema.DocumentType; 075import org.nuxeo.ecm.core.schema.SchemaManager; 076import org.nuxeo.ecm.core.schema.types.ComplexType; 077import org.nuxeo.ecm.core.schema.types.CompositeType; 078import org.nuxeo.ecm.core.schema.types.Field; 079import org.nuxeo.ecm.core.schema.types.ListType; 080import org.nuxeo.ecm.core.schema.types.SimpleTypeImpl; 081import org.nuxeo.ecm.core.schema.types.Type; 082import org.nuxeo.ecm.core.schema.types.primitives.BooleanType; 083import org.nuxeo.ecm.core.schema.types.primitives.DateType; 084import org.nuxeo.ecm.core.schema.types.primitives.DoubleType; 085import org.nuxeo.ecm.core.schema.types.primitives.IntegerType; 086import org.nuxeo.ecm.core.schema.types.primitives.LongType; 087import org.nuxeo.ecm.core.schema.types.primitives.StringType; 088import org.nuxeo.ecm.core.transientstore.api.TransientStore; 089import org.nuxeo.ecm.core.transientstore.work.TransientStoreWork; 090import org.nuxeo.ecm.core.work.api.WorkManager; 091import org.nuxeo.ecm.csv.core.CSVImportLog.Status; 092import org.nuxeo.ecm.platform.ec.notification.NotificationEventListener; 093import org.nuxeo.ecm.platform.ec.notification.service.NotificationServiceHelper; 094import org.nuxeo.ecm.platform.types.TypeManager; 095import org.nuxeo.ecm.platform.url.api.DocumentViewCodecManager; 096import org.nuxeo.ecm.platform.url.codec.api.DocumentViewCodec; 097import org.nuxeo.ecm.platform.usermanager.UserManager; 098import org.nuxeo.runtime.api.Framework; 099 100/** 101 * Work task to import form a CSV file. Because the file is read from the local filesystem, this must be executed in a 102 * local queue. Since NXP-15252 the CSV reader manages "records", not "lines". 103 * 104 * @since 5.7 105 */ 106public class CSVImporterWork extends TransientStoreWork { 107 108 public static final String NUXEO_CSV_MAIL_TO = "nuxeo.csv.mail.to"; 109 110 public static final String LABEL_CSV_IMPORTER_NOT_EXISTING_FIELD = "label.csv.importer.notExistingField"; 111 112 public static final String LABEL_CSV_IMPORTER_CANNOT_CONVERT_FIELD_VALUE = "label.csv.importer.cannotConvertFieldValue"; 113 114 public static final String LABEL_CSV_IMPORTER_NOT_EXISTING_FILE = "label.csv.importer.notExistingFile"; 115 116 public static final String NUXEO_CSV_BLOBS_FOLDER = "nuxeo.csv.blobs.folder"; 117 118 public static final String LABEL_CSV_IMPORTER_DOCUMENT_ALREADY_EXISTS = "label.csv.importer.documentAlreadyExists"; 119 120 public static final String LABEL_CSV_IMPORTER_UNABLE_TO_UPDATE = "label.csv.importer.unableToUpdate"; 121 122 public static final String LABEL_CSV_IMPORTER_DOCUMENT_UPDATED = "label.csv.importer.documentUpdated"; 123 124 public static final String LABEL_CSV_IMPORTER_UNABLE_TO_CREATE = "label.csv.importer.unableToCreate"; 125 126 public static final String LABEL_CSV_IMPORTER_PARENT_DOES_NOT_EXIST = "label.csv.importer.parentDoesNotExist"; 127 128 public static final String LABEL_CSV_IMPORTER_DOCUMENT_CREATED = "label.csv.importer.documentCreated"; 129 130 public static final String LABEL_CSV_IMPORTER_NOT_ALLOWED_SUB_TYPE = "label.csv.importer.notAllowedSubType"; 131 132 public static final String LABEL_CSV_IMPORTER_UNABLE_TO_SAVE = "label.csv.importer.unableToSave"; 133 134 public static final String LABEL_CSV_IMPORTER_ERROR_IMPORTING_LINE = "label.csv.importer.errorImportingLine"; 135 136 public static final String LABEL_CSV_IMPORTER_NOT_EXISTING_TYPE = "label.csv.importer.notExistingType"; 137 138 public static final String LABEL_CSV_IMPORTER_MISSING_TYPE_VALUE = "label.csv.importer.missingTypeValue"; 139 140 public static final String LABEL_CSV_IMPORTER_MISSING_NAME_VALUE = "label.csv.importer.missingNameValue"; 141 142 public static final String LABEL_CSV_IMPORTER_MISSING_NAME_COLUMN = "label.csv.importer.missingNameColumn"; 143 144 public static final String LABEL_CSV_IMPORTER_EMPTY_FILE = "label.csv.importer.emptyFile"; 145 146 public static final String LABEL_CSV_IMPORTER_ERROR_DURING_IMPORT = "label.csv.importer.errorDuringImport"; 147 148 public static final String LABEL_CSV_IMPORTER_EMPTY_LINE = "label.csv.importer.emptyLine"; 149 150 private static final long serialVersionUID = 1L; 151 152 private static final Logger log = LogManager.getLogger(CSVImporterWork.class); 153 154 private static final String TEMPLATE_IMPORT_RESULT = "templates/csvImportResult.ftl"; 155 156 public static final String CATEGORY_CSV_IMPORTER = "csvImporter"; 157 158 public static final String CONTENT_FILED_TYPE_NAME = "content"; 159 160 private static final long COMPUTE_TOTAL_THRESHOLD_KB = 1000; 161 162 /** 163 * CSV headers that won't be checked if the field exists on the document type. 164 * 165 * @since 7.3 166 */ 167 protected static final List<String> AUTHORIZED_HEADERS = Arrays.asList(NXQL.ECM_LIFECYCLESTATE, NXQL.ECM_UUID); 168 169 protected String parentPath; 170 171 protected String username; 172 173 protected CSVImporterOptions options; 174 175 protected boolean hasTypeColumn; 176 177 protected Date startDate; 178 179 protected ArrayList<CSVImportLog> importLogs = new ArrayList<>(); 180 181 protected boolean computeTotal = false; 182 183 protected long total = -1L; 184 185 /** @since 11.1 */ 186 protected long docsProcessedCount; 187 188 protected long docsCreatedCount; 189 190 public CSVImporterWork(String id) { 191 super(id); 192 } 193 194 public CSVImporterWork(String repositoryName, String parentPath, String username, Blob csvBlob, 195 CSVImporterOptions options) { 196 super(CSVImportId.create(repositoryName, parentPath, csvBlob)); 197 getStore().putBlobs(id, Collections.singletonList(csvBlob)); 198 setDocument(repositoryName, null); 199 setOriginatingUsername(username); 200 this.parentPath = parentPath; 201 this.username = username; 202 if (csvBlob.getLength() >= 0 && csvBlob.getLength() / 1024 < COMPUTE_TOTAL_THRESHOLD_KB) { 203 computeTotal = true; 204 } 205 this.options = options; 206 startDate = new Date(); 207 } 208 209 @Override 210 public String getCategory() { 211 return CATEGORY_CSV_IMPORTER; 212 } 213 214 @Override 215 public String getTitle() { 216 return String.format("CSV import in '%s'", parentPath); 217 } 218 219 public List<CSVImportLog> getImportLogs() { 220 return new ArrayList<>(importLogs); 221 } 222 223 @Override 224 public void work() { 225 TransientStore store = getStore(); 226 setStatus("Importing"); 227 openUserSession(); 228 CSVFormat csvFormat = CSVFormat.DEFAULT.withHeader() 229 .withEscape(options.getEscapeCharacter()) 230 .withCommentMarker(options.getCommentMarker()) 231 .withIgnoreSurroundingSpaces(); 232 try (Reader in = newReader(getBlob()); CSVParser parser = csvFormat.parse(in)) { 233 doImport(parser); 234 } catch (IOException e) { 235 logError(0, "Error while doing the import: %s", LABEL_CSV_IMPORTER_ERROR_DURING_IMPORT, e.getMessage()); 236 log.debug(e, e); 237 } 238 store.putParameter(id, "logs", importLogs); 239 if (options.sendEmail()) { 240 setStatus("Sending email"); 241 sendMail(); 242 } 243 setStatus(null); 244 } 245 246 @Override 247 public void cleanUp(boolean ok, Exception e) { 248 try { 249 super.cleanUp(ok, e); 250 } finally { 251 getStore().putParameter(id, "status", 252 new CSVImportStatus(CSVImportStatus.State.COMPLETED, docsCreatedCount, docsProcessedCount)); 253 } 254 } 255 256 static final Serializable EMPTY_LOGS = new ArrayList<CSVImportLog>(); 257 258 String launch() { 259 WorkManager works = Framework.getService(WorkManager.class); 260 261 TransientStore store = getStore(); 262 store.putParameter(id, "logs", EMPTY_LOGS); 263 store.putParameter(id, "status", new CSVImportStatus(CSVImportStatus.State.SCHEDULED)); 264 works.schedule(this); 265 return id; 266 } 267 268 static CSVImportStatus getStatus(String id) { 269 TransientStore store = getStore(); 270 if (!store.exists(id)) { 271 return null; 272 } 273 return (CSVImportStatus) store.getParameter(id, "status"); 274 } 275 276 @SuppressWarnings("unchecked") 277 static List<CSVImportLog> getLastImportLogs(String id) { 278 TransientStore store = getStore(); 279 if (!store.exists(id)) { 280 return Collections.emptyList(); 281 } 282 return (ArrayList<CSVImportLog>) store.getParameter(id, "logs"); 283 } 284 285 /** 286 * @since 7.3 287 */ 288 protected BufferedReader newReader(Blob blob) throws IOException { 289 return new BufferedReader(new InputStreamReader(new BOMInputStream(blob.getStream()))); 290 } 291 292 protected void doImport(CSVParser parser) { 293 log.info("Importing CSV file: {}", () -> getBlob().getFilename()); 294 Map<String, Integer> header = parser.getHeaderMap(); 295 if (header == null) { 296 logError(0, "No header line, empty file?", LABEL_CSV_IMPORTER_EMPTY_FILE); 297 return; 298 } 299 if (!header.containsKey(CSV_NAME_COL)) { 300 logError(0, "Missing 'name' column", LABEL_CSV_IMPORTER_MISSING_NAME_COLUMN); 301 return; 302 } 303 hasTypeColumn = header.containsKey(CSV_TYPE_COL); 304 305 try { 306 int batchSize = options.getBatchSize(); 307 Iterable<CSVRecord> it = parser; 308 if (computeTotal) { 309 try { 310 List<CSVRecord> l = parser.getRecords(); 311 total = l.size(); 312 it = l; 313 } catch (IOException e) { 314 log.warn("Could not compute total number of document to be imported"); 315 } 316 } 317 for (CSVRecord record : it) { 318 docsProcessedCount++; 319 if (record.size() == 0) { 320 // empty record 321 importLogs.add(new CSVImportLog(getLineNumber(record), Status.SKIPPED, "Empty record", 322 LABEL_CSV_IMPORTER_EMPTY_LINE)); 323 continue; 324 } 325 try { 326 if (importRecord(record, header)) { 327 docsCreatedCount++; 328 getStore().putParameter(id, "status", 329 new CSVImportStatus(CSVImportStatus.State.RUNNING, docsCreatedCount, total)); 330 if (docsCreatedCount % batchSize == 0) { 331 commitOrRollbackTransaction(); 332 startTransaction(); 333 } 334 } 335 } catch (NuxeoException e) { 336 // try next line 337 Throwable unwrappedException = unwrapException(e); 338 logError(getLineNumber(parser), "Error while importing line: %s", 339 LABEL_CSV_IMPORTER_ERROR_IMPORTING_LINE, unwrappedException.getMessage()); 340 log.debug(unwrappedException, unwrappedException); 341 } 342 } 343 344 try { 345 session.save(); 346 } catch (NuxeoException e) { 347 Throwable ue = unwrapException(e); 348 logError(getLineNumber(parser), "Unable to save: %s", LABEL_CSV_IMPORTER_UNABLE_TO_SAVE, 349 ue.getMessage()); 350 log.debug(ue, ue); 351 } 352 } finally { 353 commitOrRollbackTransaction(); 354 startTransaction(); 355 } 356 log.info("Done importing CSV file: {}", () -> getBlob().getFilename()); 357 } 358 359 /** 360 * Import a line from the CSV file. 361 * 362 * @return {@code true} if a document has been created or updated, {@code false} otherwise. 363 * @since 6.0 364 */ 365 protected boolean importRecord(CSVRecord record, Map<String, Integer> header) { 366 String name = record.get(CSV_NAME_COL); 367 if (StringUtils.isBlank(name)) { 368 log.debug("record.isSet={}", () -> record.isSet(CSV_NAME_COL)); 369 logError(getLineNumber(record), "Missing 'name' value", LABEL_CSV_IMPORTER_MISSING_NAME_VALUE); 370 return false; 371 } 372 373 Path targetPath = new Path(parentPath).append(name); 374 name = targetPath.lastSegment(); 375 String newParentPath = targetPath.removeLastSegments(1).toString(); 376 boolean exists = options.getCSVImporterDocumentFactory().exists(session, newParentPath, name, null); 377 378 DocumentRef docRef = null; 379 String type = null; 380 if (exists) { 381 docRef = new PathRef(targetPath.toString()); 382 type = session.getDocument(docRef).getType(); 383 } else { 384 if (hasTypeColumn) { 385 type = record.get(CSV_TYPE_COL); 386 } 387 if (StringUtils.isBlank(type)) { 388 log.debug("record.isSet={}", () -> record.isSet(CSV_TYPE_COL)); 389 logError(getLineNumber(record), "Missing 'type' value", LABEL_CSV_IMPORTER_MISSING_TYPE_VALUE); 390 return false; 391 } 392 } 393 394 DocumentType docType = Framework.getService(SchemaManager.class).getDocumentType(type); 395 if (docType == null) { 396 logError(getLineNumber(record), "The type '%s' does not exist", LABEL_CSV_IMPORTER_NOT_EXISTING_TYPE, type); 397 return false; 398 } 399 Map<String, Serializable> properties = computePropertiesMap(record, docType, header); 400 if (properties == null) { 401 // skip this line 402 return false; 403 } 404 405 long lineNumber = getLineNumber(record); 406 if (exists) { 407 return updateDocument(lineNumber, docRef, properties); 408 } else { 409 return createDocument(lineNumber, newParentPath, name, type, properties); 410 } 411 } 412 413 // our code expects line numbers to start at 1 for the header and 2 for the line after, 414 // but since commons-csv 1.5 record numbers restart at 1 on the line after the header 415 // thus we need to add 1 416 protected long getLineNumber(CSVRecord record) { 417 return record.getRecordNumber() + 1; 418 } 419 420 protected long getLineNumber(CSVParser parser) { 421 return parser.getRecordNumber() + 1; 422 } 423 424 /** 425 * @since 6.0 426 */ 427 protected Map<String, Serializable> computePropertiesMap(CSVRecord record, CompositeType compositeType, 428 Map<String, Integer> header) { 429 Map<String, Serializable> values = new HashMap<>(); 430 for (String headerValue : header.keySet()) { 431 String lineValue = record.get(headerValue); 432 lineValue = lineValue.trim(); 433 String fieldName = headerValue; 434 if (!CSV_NAME_COL.equals(headerValue) && !CSV_TYPE_COL.equals(headerValue)) { 435 if (AUTHORIZED_HEADERS.contains(headerValue) && !StringUtils.isBlank(lineValue)) { 436 values.put(headerValue, lineValue); 437 } else { 438 if (!compositeType.hasField(fieldName)) { 439 fieldName = fieldName.split(":")[1]; 440 } 441 if (compositeType.hasField(fieldName) && !StringUtils.isBlank(lineValue)) { 442 Serializable convertedValue = convertValue(compositeType, fieldName, headerValue, lineValue, 443 getLineNumber(record)); 444 if (convertedValue == null) { 445 return null; 446 } 447 values.put(headerValue, convertedValue); 448 } 449 } 450 } 451 } 452 return values; 453 } 454 455 @SuppressWarnings("unchecked") 456 protected Serializable convertValue(CompositeType compositeType, String fieldName, String headerValue, 457 String stringValue, long lineNumber) { 458 if (compositeType.hasField(fieldName)) { 459 Field field = compositeType.getField(fieldName); 460 if (field != null) { 461 try { 462 Serializable fieldValue = null; 463 Type fieldType = field.getType(); 464 if (fieldType.isComplexType()) { 465 if (fieldType.getName().equals(CONTENT_FILED_TYPE_NAME)) { 466 fieldValue = (Serializable) createBlobFromFilePath(stringValue); 467 if (fieldValue == null) { 468 logError(lineNumber, "The file '%s' does not exist", 469 LABEL_CSV_IMPORTER_NOT_EXISTING_FILE, stringValue); 470 return null; 471 } 472 } else { 473 fieldValue = (Serializable) ComplexTypeJSONDecoder.decode((ComplexType) fieldType, 474 stringValue); 475 replaceBlobs((Map<String, Object>) fieldValue); 476 } 477 } else { 478 if (fieldType.isListType()) { 479 Type listFieldType = ((ListType) fieldType).getFieldType(); 480 if (listFieldType.isSimpleType()) { 481 /* 482 * Array. 483 */ 484 fieldValue = stringValue.split(options.getListSeparatorRegex()); 485 } else { 486 /* 487 * Complex list. 488 */ 489 fieldValue = (Serializable) ComplexTypeJSONDecoder.decodeList((ListType) fieldType, 490 stringValue); 491 replaceBlobs((List<Object>) fieldValue); 492 } 493 } else { 494 /* 495 * Primitive type. 496 */ 497 Type type = field.getType(); 498 if (type instanceof SimpleTypeImpl) { 499 type = type.getSuperType(); 500 } 501 if (type.isSimpleType()) { 502 if (type instanceof StringType) { 503 fieldValue = stringValue; 504 } else if (type instanceof IntegerType) { 505 fieldValue = Integer.valueOf(stringValue); 506 } else if (type instanceof LongType) { 507 fieldValue = Long.valueOf(stringValue); 508 } else if (type instanceof DoubleType) { 509 fieldValue = Double.valueOf(stringValue); 510 } else if (type instanceof BooleanType) { 511 fieldValue = Boolean.valueOf(stringValue); 512 } else if (type instanceof DateType) { 513 DateFormat dateFormat = options.getDateFormat(); 514 fieldValue = dateFormat != null ? dateFormat.parse(stringValue) : stringValue; 515 } 516 } 517 } 518 } 519 return fieldValue; 520 } catch (ParseException | NumberFormatException | IOException e) { 521 logError(lineNumber, "Unable to convert field '%s' with value '%s'", 522 LABEL_CSV_IMPORTER_CANNOT_CONVERT_FIELD_VALUE, headerValue, stringValue); 523 log.debug(e, e); 524 } 525 } 526 } else { 527 logError(lineNumber, "Field '%s' does not exist on type '%s'", LABEL_CSV_IMPORTER_NOT_EXISTING_FIELD, 528 headerValue, compositeType.getName()); 529 } 530 return null; 531 } 532 533 /** 534 * Creates a {@code Blob} from a relative file path. The File will be looked up in the folder registered by the 535 * {@code nuxeo.csv.blobs.folder} property. 536 * 537 * @since 9.3 538 */ 539 protected Blob createBlobFromFilePath(String fileRelativePath) throws IOException { 540 String blobsFolderPath = Framework.getProperty(NUXEO_CSV_BLOBS_FOLDER); 541 String path = FilenameUtils.normalize(blobsFolderPath + "/" + fileRelativePath); 542 File file = new File(path); 543 if (file.exists()) { 544 return Blobs.createBlob(file, null, null, FilenameUtils.getName(fileRelativePath)); 545 } else { 546 return null; 547 } 548 } 549 550 /** 551 * Creates a {@code Blob} from a {@code StringBlob}. Assume that the {@code StringBlob} content is the relative file 552 * path. The File will be looked up in the folder registered by the {@code nuxeo.csv.blobs.folder} property. 553 * 554 * @since 9.3 555 */ 556 protected Blob createBlobFromStringBlob(Blob stringBlob) throws IOException { 557 String fileRelativePath = stringBlob.getString(); 558 Blob blob = createBlobFromFilePath(fileRelativePath); 559 if (blob == null) { 560 throw new IOException(String.format("File %s does not exist", fileRelativePath)); 561 } 562 563 blob.setMimeType(stringBlob.getMimeType()); 564 blob.setEncoding(stringBlob.getEncoding()); 565 String filename = stringBlob.getFilename(); 566 if (filename != null) { 567 blob.setFilename(filename); 568 } 569 return blob; 570 } 571 572 /** 573 * Recursively replaces all {@code Blob}s with {@code Blob}s created from Files stored in the folder registered by 574 * the {@code nuxeo.csv.blobs.folder} property. 575 * 576 * @since 9.3 577 */ 578 @SuppressWarnings("unchecked") 579 protected void replaceBlobs(Map<String, Object> map) throws IOException { 580 for (Map.Entry<String, Object> entry : map.entrySet()) { 581 Object value = entry.getValue(); 582 if (value instanceof Blob) { 583 Blob blob = (Blob) value; 584 entry.setValue(createBlobFromStringBlob(blob)); 585 } else if (value instanceof List) { 586 replaceBlobs((List<Object>) value); 587 } else if (value instanceof Map) { 588 replaceBlobs((Map<String, Object>) value); 589 } 590 } 591 } 592 593 /** 594 * Recursively replaces all {@code Blob}s with {@code Blob}s created from Files stored in the folder registered by 595 * the {@code nuxeo.csv.blobs.folder} property. 596 * 597 * @since 9.3 598 */ 599 @SuppressWarnings("unchecked") 600 protected void replaceBlobs(List<Object> list) throws IOException { 601 for (ListIterator<Object> it = list.listIterator(); it.hasNext();) { 602 Object value = it.next(); 603 if (value instanceof Blob) { 604 Blob blob = (Blob) value; 605 it.set(createBlobFromStringBlob(blob)); 606 } else if (value instanceof List) { 607 replaceBlobs((List<Object>) value); 608 } else if (value instanceof Map) { 609 replaceBlobs((Map<String, Object>) value); 610 } 611 } 612 } 613 614 protected boolean createDocument(long lineNumber, String newParentPath, String name, String type, 615 Map<String, Serializable> properties) { 616 try { 617 DocumentRef parentRef = new PathRef(newParentPath); 618 if (session.exists(parentRef)) { 619 DocumentModel parent = session.getDocument(parentRef); 620 621 TypeManager typeManager = Framework.getService(TypeManager.class); 622 if (options.checkAllowedSubTypes() && !typeManager.isAllowedSubType(type, parent.getType())) { 623 logError(lineNumber, "'%s' type is not allowed in '%s'", LABEL_CSV_IMPORTER_NOT_ALLOWED_SUB_TYPE, 624 type, parent.getType()); 625 } else { 626 options.getCSVImporterDocumentFactory() 627 .createDocument(session, newParentPath, name, type, properties); 628 importLogs.add(new CSVImportLog(lineNumber, Status.SUCCESS, "Document created", 629 LABEL_CSV_IMPORTER_DOCUMENT_CREATED)); 630 return true; 631 } 632 } else { 633 logError(lineNumber, "Parent document '%s' does not exist", LABEL_CSV_IMPORTER_PARENT_DOES_NOT_EXIST, 634 newParentPath); 635 } 636 } catch (RuntimeException e) { 637 Throwable unwrappedException = unwrapException(e); 638 logError(lineNumber, "Unable to create document: %s", LABEL_CSV_IMPORTER_UNABLE_TO_CREATE, 639 unwrappedException.getMessage()); 640 log.debug(unwrappedException, unwrappedException); 641 } 642 return false; 643 } 644 645 protected boolean updateDocument(long lineNumber, DocumentRef docRef, Map<String, Serializable> properties) { 646 if (options.updateExisting()) { 647 try { 648 options.getCSVImporterDocumentFactory().updateDocument(session, docRef, properties); 649 importLogs.add(new CSVImportLog(lineNumber, Status.SUCCESS, "Document updated", 650 LABEL_CSV_IMPORTER_DOCUMENT_UPDATED)); 651 return true; 652 } catch (RuntimeException e) { 653 Throwable unwrappedException = unwrapException(e); 654 logError(lineNumber, "Unable to update document: %s", LABEL_CSV_IMPORTER_UNABLE_TO_UPDATE, 655 unwrappedException.getMessage()); 656 log.debug(unwrappedException, unwrappedException); 657 } 658 } else { 659 importLogs.add(new CSVImportLog(lineNumber, Status.SKIPPED, "Document already exists", 660 LABEL_CSV_IMPORTER_DOCUMENT_ALREADY_EXISTS)); 661 } 662 return false; 663 } 664 665 protected void logError(long lineNumber, String message, String localizedMessage, String... params) { 666 importLogs.add(new CSVImportLog(lineNumber, ERROR, String.format(message, (Object[]) params), localizedMessage, 667 params)); 668 String lineMessage = String.format("Line %d", lineNumber); 669 String errorMessage = String.format(message, (Object[]) params); 670 log.error("{}: {}", lineMessage, errorMessage); 671 getStore().putParameter(id, "status", 672 new CSVImportStatus(CSVImportStatus.State.ERROR, docsCreatedCount, docsCreatedCount)); 673 } 674 675 protected void sendMail() { 676 UserManager userManager = Framework.getService(UserManager.class); 677 NuxeoPrincipal principal = userManager.getPrincipal(username); 678 String email = principal.getEmail(); 679 if (email == null) { 680 log.info("Not sending import result email to '{}', no email configured", username); 681 return; 682 } 683 684 try (OperationContext ctx = new OperationContext(session)) { 685 ctx.setInput(session.getRootDocument()); 686 687 CSVImporter csvImporter = Framework.getService(CSVImporter.class); 688 List<CSVImportLog> importerLogs = csvImporter.getImportLogs(getId()); 689 CSVImportResult importResult = CSVImportResult.fromImportLogs(importerLogs); 690 List<CSVImportLog> skippedAndErrorImportLogs = csvImporter.getImportLogs(getId(), Status.SKIPPED, 691 Status.ERROR); 692 ctx.put("importResult", importResult); 693 ctx.put("skippedAndErrorImportLogs", skippedAndErrorImportLogs); 694 ctx.put("csvFilename", getBlob().getFilename()); 695 ctx.put("startDate", DateFormat.getInstance().format(startDate)); 696 ctx.put("username", username); 697 698 DocumentModel importFolder = session.getDocument(new PathRef(parentPath)); 699 String importFolderUrl = getDocumentUrl(importFolder); 700 ctx.put("importFolderTitle", importFolder.getTitle()); 701 ctx.put("importFolderUrl", importFolderUrl); 702 ctx.put("userUrl", getUserUrl()); 703 704 StringList to = buildRecipientsList(email); 705 Expression from = Scripting.newExpression("Env[\"mail.from\"]"); 706 String subject = "CSV Import result of " + getBlob().getFilename(); 707 String message = loadTemplate(TEMPLATE_IMPORT_RESULT); 708 709 OperationChain chain = new OperationChain("SendMail"); 710 chain.add(SendMail.ID) 711 .set("from", from) 712 .set("to", to) 713 .set("HTML", true) 714 .set("subject", subject) 715 .set("message", message); 716 Framework.getService(AutomationService.class).run(ctx, chain); 717 } catch (Exception e) { 718 ExceptionUtils.checkInterrupt(e); 719 log.error("Unable to notify user '{}' for import result of '{}': {}", () -> username, 720 () -> getBlob().getFilename(), e::getMessage); 721 log.debug(e, e); 722 throw ExceptionUtils.runtimeException(e); 723 } 724 } 725 726 /** 727 * @since 9.1 728 */ 729 private Blob getBlob() { 730 return getStore().getBlobs(id).get(0); 731 } 732 733 protected String getDocumentUrl(DocumentModel doc) { 734 return MailTemplateHelper.getDocumentUrl(doc, null); 735 } 736 737 protected String getUserUrl() { 738 DocumentViewCodecManager codecService = Framework.getService(DocumentViewCodecManager.class); 739 DocumentViewCodec codec = codecService.getCodec(NotificationEventListener.NOTIFICATION_DOCUMENT_ID_CODEC_NAME); 740 boolean isNotificationCodec = codec != null; 741 boolean isJSFUI = isNotificationCodec 742 && NotificationEventListener.JSF_NOTIFICATION_DOCUMENT_ID_CODEC_PREFIX.equals(codec.getPrefix()); 743 StringBuilder userUrl = new StringBuilder(); 744 if (isNotificationCodec) { 745 746 userUrl.append(NotificationServiceHelper.getNotificationService().getServerUrlPrefix()); 747 if (!isJSFUI) { 748 userUrl.append("ui/"); 749 userUrl.append("#!/"); 750 } 751 userUrl.append("user/").append(username); 752 } 753 return userUrl.toString(); 754 } 755 756 protected StringList buildRecipientsList(String userEmail) { 757 String csvMailTo = Framework.getProperty(NUXEO_CSV_MAIL_TO); 758 if (StringUtils.isBlank(csvMailTo)) { 759 return new StringList(new String[] { userEmail }); 760 } else { 761 return new StringList(new String[] { userEmail, csvMailTo }); 762 } 763 } 764 765 private static String loadTemplate(String key) { 766 InputStream io = CSVImporterWork.class.getClassLoader().getResourceAsStream(key); 767 if (io != null) { 768 try { 769 return IOUtils.toString(io, UTF_8); 770 } catch (IOException e) { 771 // cannot happen 772 throw new NuxeoException(e); 773 } finally { 774 try { 775 io.close(); 776 } catch (IOException e) { 777 // nothing to do 778 } 779 } 780 } 781 return null; 782 } 783 784 public static Throwable unwrapException(Throwable t) { 785 Throwable cause = null; 786 if (t != null) { 787 cause = t.getCause(); 788 } 789 if (cause == null) { 790 return t; 791 } else { 792 return unwrapException(cause); 793 } 794 } 795 796}