001/* 002 * (C) Copyright 2015-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Antoine Taillefer <ataillefer@nuxeo.com> 018 * Luís Duarte 019 * Florent Guillaume 020 */ 021package org.nuxeo.ecm.restapi.server.jaxrs; 022 023import static org.apache.commons.lang3.StringUtils.isEmpty; 024 025import java.io.IOException; 026import java.io.InputStream; 027import java.io.UnsupportedEncodingException; 028import java.net.URLDecoder; 029import java.util.ArrayList; 030import java.util.Collections; 031import java.util.HashMap; 032import java.util.List; 033import java.util.Map; 034import java.util.Set; 035import java.util.stream.Collectors; 036 037import javax.mail.MessagingException; 038import javax.servlet.http.HttpServletRequest; 039import javax.servlet.http.HttpServletResponse; 040import javax.ws.rs.DELETE; 041import javax.ws.rs.GET; 042import javax.ws.rs.POST; 043import javax.ws.rs.Path; 044import javax.ws.rs.PathParam; 045import javax.ws.rs.Produces; 046import javax.ws.rs.core.Context; 047import javax.ws.rs.core.MediaType; 048import javax.ws.rs.core.Response; 049import javax.ws.rs.core.Response.Status; 050import javax.ws.rs.core.Response.Status.Family; 051import javax.ws.rs.core.Response.StatusType; 052import javax.ws.rs.core.UriInfo; 053 054import org.apache.commons.collections.CollectionUtils; 055import org.apache.commons.lang3.StringUtils; 056import org.apache.commons.lang3.math.NumberUtils; 057import org.apache.commons.logging.Log; 058import org.apache.commons.logging.LogFactory; 059import org.nuxeo.ecm.automation.OperationContext; 060import org.nuxeo.ecm.automation.jaxrs.io.operations.ExecutionRequest; 061import org.nuxeo.ecm.automation.server.jaxrs.ResponseHelper; 062import org.nuxeo.ecm.automation.server.jaxrs.batch.Batch; 063import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchFileEntry; 064import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchHandler; 065import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchManager; 066import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchManagerConstants; 067import org.nuxeo.ecm.automation.server.jaxrs.batch.handler.BatchFileInfo; 068import org.nuxeo.ecm.core.api.Blob; 069import org.nuxeo.ecm.core.api.Blobs; 070import org.nuxeo.ecm.core.api.CoreSession; 071import org.nuxeo.ecm.core.api.NuxeoException; 072import org.nuxeo.ecm.webengine.forms.FormData; 073import org.nuxeo.ecm.webengine.jaxrs.context.RequestContext; 074import org.nuxeo.ecm.webengine.model.WebObject; 075import org.nuxeo.ecm.webengine.model.exceptions.IllegalParameterException; 076import org.nuxeo.ecm.webengine.model.impl.AbstractResource; 077import org.nuxeo.ecm.webengine.model.impl.ResourceTypeImpl; 078import org.nuxeo.runtime.api.Framework; 079import org.nuxeo.runtime.transaction.TransactionHelper; 080 081import com.fasterxml.jackson.databind.JsonNode; 082import com.fasterxml.jackson.databind.ObjectMapper; 083 084/** 085 * Batch upload endpoint. 086 * <p> 087 * Provides the APIs listed below: 088 * <ul> 089 * <li>POST /upload, see {@link #initBatch()}</li> 090 * <li>POST /upload/{batchId}/{fileIdx}, see {@link #upload(HttpServletRequest, String, String)}</li> 091 * <li>GET /upload/{batchId}, see {@link #getBatchInfo(String)}</li> 092 * <li>GET /upload/{batchId}/{fileIdx}, see {@link #getFileInfo(String, String)}</li> 093 * <li>POST /upload/{batchId}/execute/{operationId}, see {@link #execute(String, String, ExecutionRequest)}</li> 094 * <li>POST /upload/{batchId}/{fileIdx}/execute/{operationId}, see 095 * {@link #execute(String, String, String, ExecutionRequest)}</li> 096 * <li>DELETE /upload/{batchId}, see {@link #cancel(String)}</li> 097 * <li>DELETE /upload/{batchId}/{fileIdx}, see {@link #removeFile(String, String)}</li> 098 * </ul> 099 * Largely inspired by the excellent Google Drive REST API documentation about 100 * <a href="https://developers.google.com/drive/web/manage-uploads#resumable">resumable upload</a>. 101 * 102 * @since 7.4 103 */ 104@WebObject(type = "upload") 105public class BatchUploadObject extends AbstractResource<ResourceTypeImpl> { 106 107 protected static final Log log = LogFactory.getLog(BatchUploadObject.class); 108 109 protected static final String REQUEST_BATCH_ID = "batchId"; 110 111 protected static final String REQUEST_FILE_IDX = "fileIdx"; 112 113 protected static final String OPERATION_ID = "operationId"; 114 115 protected static final String REQUEST_HANDLER_NAME = "handlerName"; 116 117 public static final String UPLOAD_TYPE_NORMAL = "normal"; 118 119 public static final String UPLOAD_TYPE_CHUNKED = "chunked"; 120 121 public static final String KEY = "key"; 122 123 public static final String NAME = "name"; 124 125 public static final String MIMETYPE = "mimeType"; 126 127 public static final String FILE_SIZE = "fileSize"; 128 129 public static final String MD5 = "md5"; 130 131 protected Map<String, String> mapWithName(String name) { 132 return Collections.singletonMap("name", name); 133 } 134 135 @GET 136 @Path("handlers") 137 public Response handlers() throws IOException { 138 BatchManager bm = Framework.getService(BatchManager.class); 139 Set<String> supportedHandlers = bm.getSupportedHandlers(); 140 List<Map<String, String>> handlers = supportedHandlers.stream().map(this::mapWithName).collect( 141 Collectors.toList()); 142 Map<String, Object> result = Collections.singletonMap("handlers", handlers); 143 return buildResponse(Status.OK, result); 144 } 145 146 @GET 147 @Path("handlers/{handlerName}") 148 public Response getHandlerInfo(@PathParam(REQUEST_HANDLER_NAME) String handlerName) throws IOException { 149 BatchManager bm = Framework.getService(BatchManager.class); 150 BatchHandler handler = bm.getHandler(handlerName); 151 if (handler == null) { 152 return Response.status(Status.NOT_FOUND).build(); 153 } 154 Map<String, String> result = mapWithName(handler.getName()); 155 return buildResponse(Status.OK, result); 156 } 157 158 @POST 159 @Path("new/{handlerName}") 160 public Response createNewBatch(@PathParam(REQUEST_HANDLER_NAME) String handlerName) throws IOException { 161 BatchManager bm = Framework.getService(BatchManager.class); 162 Batch batch = bm.initBatch(handlerName); 163 return getBatchExtraInfo(batch.getKey()); 164 } 165 166 @POST 167 public Response initBatch() throws IOException { 168 BatchManager bm = Framework.getService(BatchManager.class); 169 String batchId = bm.initBatch(); 170 Map<String, String> result = new HashMap<>(); 171 result.put("batchId", batchId); 172 return buildResponse(Status.CREATED, result); 173 } 174 175 @POST 176 @Path("{batchId}/{fileIdx}") 177 public Response upload(@Context HttpServletRequest request, @PathParam(REQUEST_BATCH_ID) String batchId, 178 @PathParam(REQUEST_FILE_IDX) String fileIdx) throws IOException { 179 TransactionHelper.commitOrRollbackTransaction(); 180 try { 181 return uploadNoTransaction(request, batchId, fileIdx); 182 } finally { 183 TransactionHelper.startTransaction(); 184 } 185 } 186 187 protected Response uploadNoTransaction(@Context HttpServletRequest request, 188 @PathParam(REQUEST_BATCH_ID) String batchId, @PathParam(REQUEST_FILE_IDX) String fileIdx) 189 throws IOException { 190 BatchManager bm = Framework.getService(BatchManager.class); 191 192 if (!bm.hasBatch(batchId)) { 193 return buildEmptyResponse(Status.NOT_FOUND); 194 } 195 196 // Check file index parameter 197 if (!NumberUtils.isDigits(fileIdx)) { 198 return buildTextResponse(Status.BAD_REQUEST, "fileIdx request path parameter must be a number"); 199 } 200 201 // Parameters are passed as request header, the request body is the stream 202 String contentType = request.getHeader("Content-Type"); 203 String uploadType = request.getHeader("X-Upload-Type"); 204 // Use non chunked mode by default if X-Upload-Type header is not provided 205 if (!UPLOAD_TYPE_CHUNKED.equals(uploadType)) { 206 uploadType = UPLOAD_TYPE_NORMAL; 207 } 208 String uploadChunkIndexHeader = request.getHeader("X-Upload-Chunk-Index"); 209 String chunkCountHeader = request.getHeader("X-Upload-Chunk-Count"); 210 String fileName = request.getHeader("X-File-Name"); 211 String fileSizeHeader = request.getHeader("X-File-Size"); 212 String mimeType = request.getHeader("X-File-Type"); 213 214 int chunkCount = -1; 215 int uploadChunkIndex = -1; 216 long fileSize = -1; 217 if (UPLOAD_TYPE_CHUNKED.equals(uploadType)) { 218 try { 219 chunkCount = Integer.parseInt(chunkCountHeader); 220 uploadChunkIndex = Integer.parseInt(uploadChunkIndexHeader); 221 fileSize = Long.parseLong(fileSizeHeader); 222 } catch (NumberFormatException e) { 223 throw new IllegalParameterException( 224 "X-Upload-Chunk-Index, X-Upload-Chunk-Count and X-File-Size headers must be numbers"); 225 } 226 } 227 228 // TODO NXP-18247: should be set to the actual number of bytes uploaded instead of relying on the Content-Length 229 // header which is not necessarily set 230 long uploadedSize = getUploadedSize(request); 231 boolean isMultipart = contentType != null && contentType.contains("multipart"); 232 233 // Handle multipart case: mainly MSIE with jQueryFileupload 234 if (isMultipart) { 235 FormData formData = new FormData(request); 236 Blob blob = formData.getFirstBlob(); 237 if (blob == null) { 238 throw new NuxeoException("Cannot upload in multipart with no blobs"); 239 } 240 if (!UPLOAD_TYPE_CHUNKED.equals(uploadType)) { 241 fileName = blob.getFilename(); 242 } 243 // Don't change the mime-type if it was forced via the X-File-Type header 244 if (StringUtils.isBlank(mimeType)) { 245 mimeType = blob.getMimeType(); 246 } 247 uploadedSize = blob.getLength(); 248 addBlob(uploadType, batchId, fileIdx, blob, fileName, mimeType, uploadedSize, chunkCount, uploadChunkIndex, 249 fileSize); 250 } else { 251 if (fileName != null) { 252 fileName = URLDecoder.decode(fileName, "UTF-8"); 253 } 254 try (InputStream is = request.getInputStream()) { 255 Blob blob = Blobs.createBlob(is); 256 addBlob(uploadType, batchId, fileIdx, blob, fileName, mimeType, uploadedSize, chunkCount, 257 uploadChunkIndex, fileSize); 258 } 259 } 260 261 StatusType status = Status.CREATED; 262 Map<String, Object> result = new HashMap<>(); 263 result.put("uploaded", "true"); 264 result.put("batchId", batchId); 265 result.put("fileIdx", fileIdx); 266 result.put("uploadType", uploadType); 267 result.put("uploadedSize", String.valueOf(uploadedSize)); 268 if (UPLOAD_TYPE_CHUNKED.equals(uploadType)) { 269 BatchFileEntry fileEntry = bm.getFileEntry(batchId, fileIdx); 270 if (fileEntry != null) { 271 result.put("uploadedChunkIds", fileEntry.getOrderedChunkIndexes()); 272 result.put("chunkCount", fileEntry.getChunkCount()); 273 if (!fileEntry.isChunksCompleted()) { 274 status = new ResumeIncompleteStatusType(); 275 } 276 } 277 } 278 return buildResponse(status, result, isMultipart); 279 } 280 281 protected long getUploadedSize(HttpServletRequest request) { 282 String contentLength = request.getHeader("Content-Length"); 283 if (contentLength == null) { 284 return -1; 285 } 286 return Long.parseLong(contentLength); 287 } 288 289 protected void addBlob(String uploadType, String batchId, String fileIdx, Blob blob, String fileName, 290 String mimeType, long uploadedSize, int chunkCount, int uploadChunkIndex, long fileSize) 291 throws IOException { 292 BatchManager bm = Framework.getService(BatchManager.class); 293 String uploadedSizeDisplay = uploadedSize > -1 ? uploadedSize + "b" : "unknown size"; 294 Batch batch = bm.getBatch(batchId); 295 if (UPLOAD_TYPE_CHUNKED.equals(uploadType)) { 296 if (log.isDebugEnabled()) { 297 log.debug(String.format("Uploading chunk [index=%d / total=%d] (%s) for file %s", uploadChunkIndex, 298 chunkCount, uploadedSizeDisplay, fileName)); 299 } 300 batch.addChunk(fileIdx, blob, chunkCount, uploadChunkIndex, fileName, mimeType, fileSize); 301 } else { 302 if (log.isDebugEnabled()) { 303 log.debug(String.format("Uploading file %s (%s)", fileName, uploadedSizeDisplay)); 304 } 305 batch.addFile(fileIdx, blob, fileName, mimeType); 306 } 307 } 308 309 @GET 310 @Path("{batchId}") 311 public Response getBatchInfo(@PathParam(REQUEST_BATCH_ID) String batchId) throws IOException { 312 BatchManager bm = Framework.getService(BatchManager.class); 313 if (!bm.hasBatch(batchId)) { 314 return buildEmptyResponse(Status.NOT_FOUND); 315 } 316 List<BatchFileEntry> fileEntries = bm.getFileEntries(batchId); 317 if (CollectionUtils.isEmpty(fileEntries)) { 318 return buildEmptyResponse(Status.NO_CONTENT); 319 } 320 List<Map<String, Object>> result = new ArrayList<>(); 321 for (BatchFileEntry fileEntry : fileEntries) { 322 result.add(getFileInfo(fileEntry)); 323 } 324 return buildResponse(Status.OK, result); 325 } 326 327 @GET 328 @Path("{batchId}/{fileIdx}") 329 public Response getFileInfo(@PathParam(REQUEST_BATCH_ID) String batchId, 330 @PathParam(REQUEST_FILE_IDX) String fileIdx) throws IOException { 331 BatchManager bm = Framework.getService(BatchManager.class); 332 if (!bm.hasBatch(batchId)) { 333 return buildEmptyResponse(Status.NOT_FOUND); 334 } 335 BatchFileEntry fileEntry = bm.getFileEntry(batchId, fileIdx); 336 if (fileEntry == null) { 337 return buildEmptyResponse(Status.NOT_FOUND); 338 } 339 StatusType status = Status.OK; 340 if (fileEntry.isChunked() && !fileEntry.isChunksCompleted()) { 341 status = new ResumeIncompleteStatusType(); 342 } 343 Map<String, Object> result = getFileInfo(fileEntry); 344 return buildResponse(status, result); 345 } 346 347 @DELETE 348 @Path("{batchId}") 349 public Response cancel(@PathParam(REQUEST_BATCH_ID) String batchId) { 350 BatchManager bm = Framework.getService(BatchManager.class); 351 if (!bm.hasBatch(batchId)) { 352 return buildEmptyResponse(Status.NOT_FOUND); 353 } 354 bm.clean(batchId); 355 return buildEmptyResponse(Status.NO_CONTENT); 356 } 357 358 /** 359 * @since 8.4 360 */ 361 @DELETE 362 @Path("{batchId}/{fileIdx}") 363 public Response removeFile(@PathParam(REQUEST_BATCH_ID) String batchId, 364 @PathParam(REQUEST_FILE_IDX) String fileIdx) { 365 BatchManager bm = Framework.getService(BatchManager.class); 366 if (!bm.removeFileEntry(batchId, fileIdx)) { 367 return buildEmptyResponse(Status.NOT_FOUND); 368 } 369 return buildEmptyResponse(Status.NO_CONTENT); 370 } 371 372 @Context 373 protected HttpServletRequest request; 374 375 @Context 376 protected HttpServletResponse response; 377 378 @POST 379 @Produces("application/json") 380 @Path("{batchId}/execute/{operationId}") 381 public Object execute(@PathParam(REQUEST_BATCH_ID) String batchId, @PathParam(OPERATION_ID) String operationId, 382 ExecutionRequest xreq) { 383 return executeBatch(batchId, null, operationId, request, xreq); 384 } 385 386 @POST 387 @Produces("application/json") 388 @Path("{batchId}/{fileIdx}/execute/{operationId}") 389 public Object execute(@PathParam(REQUEST_BATCH_ID) String batchId, @PathParam(REQUEST_FILE_IDX) String fileIdx, 390 @PathParam(OPERATION_ID) String operationId, ExecutionRequest xreq) { 391 return executeBatch(batchId, fileIdx, operationId, request, xreq); 392 } 393 394 @GET 395 @Path("{batchId}/info") 396 public Response getBatchExtraInfo(@PathParam(REQUEST_BATCH_ID) String batchId) throws IOException { 397 BatchManager bm = Framework.getService(BatchManager.class); 398 if (!bm.hasBatch(batchId)) { 399 return buildEmptyResponse(Status.NOT_FOUND); 400 } 401 Batch batch = bm.getBatch(batchId); 402 Map<String, Object> properties = batch.getProperties(); 403 List<BatchFileEntry> fileEntries = batch.getFileEntries(); 404 405 List<Map<String, Object>> fileInfos = new ArrayList<>(); 406 if (!CollectionUtils.isEmpty(fileEntries)) { 407 fileEntries.stream().map(this::getFileInfo).forEach(fileInfos::add); 408 } 409 410 Map<String, Object> result = new HashMap<>(); 411 result.put("provider", batch.getHandlerName()); 412 if (properties != null && !properties.isEmpty()) { 413 result.put("extraInfo", properties); 414 } 415 416 result.put("fileEntries", fileInfos); 417 result.put("batchId", batch.getKey()); 418 return buildResponse(Status.OK, result); 419 } 420 421 @POST 422 @Path("{batchId}/{fileIdx}/complete") 423 public Response uploadCompleted(@PathParam(REQUEST_BATCH_ID) String batchId, 424 @PathParam(REQUEST_FILE_IDX) String fileIdx, String body) throws IOException { 425 BatchManager bm = Framework.getService(BatchManager.class); 426 JsonNode jsonNode = new ObjectMapper().readTree(body); 427 428 Batch batch = bm.getBatch(batchId); 429 if (batch == null) { 430 return buildEmptyResponse(Status.NOT_FOUND); 431 } 432 433 String key = jsonNode.hasNonNull(KEY) ? jsonNode.get(KEY).asText(null) : null; 434 String filename = jsonNode.hasNonNull(NAME) ? jsonNode.get(NAME).asText() : null; 435 String mimeType = jsonNode.hasNonNull(MIMETYPE) ? jsonNode.get(MIMETYPE).asText(null) : null; 436 Long length = jsonNode.hasNonNull(FILE_SIZE) ? jsonNode.get(FILE_SIZE).asLong() : -1L; 437 String md5 = jsonNode.hasNonNull(MD5) ? jsonNode.get(MD5).asText() : null; 438 439 BatchFileInfo batchFileInfo = new BatchFileInfo(key, filename, mimeType, length, md5); 440 441 BatchHandler handler = bm.getHandler(batch.getHandlerName()); 442 if (!handler.completeUpload(batchId, fileIdx, batchFileInfo)) { 443 return Response.status(Status.CONFLICT).build(); 444 } 445 446 Map<String, Object> result = new HashMap<>(); 447 result.put("uploaded", "true"); 448 result.put("batchId", batchId); 449 result.put("fileIdx", fileIdx); 450 return buildResponse(Status.OK, result); 451 } 452 453 protected Object executeBatch(String batchId, String fileIdx, String operationId, HttpServletRequest request, 454 ExecutionRequest xreq) { 455 BatchManager bm = Framework.getService(BatchManager.class); 456 457 if (!bm.hasBatch(batchId)) { 458 return buildEmptyResponse(Status.NOT_FOUND); 459 } 460 461 if (!Boolean.parseBoolean( 462 RequestContext.getActiveContext(request).getRequest().getHeader(BatchManagerConstants.NO_DROP_FLAG))) { 463 RequestContext.getActiveContext(request).addRequestCleanupHandler(req -> { 464 bm.clean(batchId); 465 }); 466 } 467 468 try { 469 CoreSession session = ctx.getCoreSession(); 470 Object result; 471 try (OperationContext ctx = xreq.createContext(request, response, session)) { 472 Map<String, Object> params = xreq.getParams(); 473 if (StringUtils.isBlank(fileIdx)) { 474 result = bm.execute(batchId, operationId, session, ctx, params); 475 } else { 476 result = bm.execute(batchId, fileIdx, operationId, session, ctx, params); 477 } 478 } 479 return ResponseHelper.getResponse(result, request); 480 } catch (MessagingException | IOException e) { 481 log.error("Error while executing automation batch ", e); 482 throw new NuxeoException(e); 483 } 484 } 485 486 protected Response buildResponse(StatusType status, Object object) throws IOException { 487 return buildResponse(status, object, false); 488 } 489 490 protected Response buildResponse(StatusType status, Object object, boolean html) throws IOException { 491 ObjectMapper mapper = new ObjectMapper(); 492 String result = mapper.writeValueAsString(object); 493 if (html) { 494 // For MSIE with iframe transport: we need to return HTML! 495 return buildHTMLResponse(status, result); 496 } else { 497 return buildJSONResponse(status, result); 498 } 499 } 500 501 protected Response buildJSONResponse(StatusType status, String message) throws UnsupportedEncodingException { 502 return buildResponse(status, MediaType.APPLICATION_JSON, message); 503 } 504 505 protected Response buildHTMLResponse(StatusType status, String message) throws UnsupportedEncodingException { 506 message = "<html>" + message + "</html>"; 507 return buildResponse(status, MediaType.TEXT_HTML, message); 508 } 509 510 protected Response buildTextResponse(StatusType status, String message) throws UnsupportedEncodingException { 511 return buildResponse(status, MediaType.TEXT_PLAIN, message); 512 } 513 514 protected Response buildEmptyResponse(StatusType status) { 515 return Response.status(status).build(); 516 } 517 518 protected Response buildResponse(StatusType status, String type, String message) 519 throws UnsupportedEncodingException { 520 return Response.status(status) 521 .header("Content-Length", message.getBytes("UTF-8").length) 522 .type(type + "; charset=UTF-8") 523 .entity(message) 524 .build(); 525 } 526 527 protected Map<String, Object> getFileInfo(BatchFileEntry fileEntry) { 528 Map<String, Object> info = new HashMap<>(); 529 boolean chunked = fileEntry.isChunked(); 530 String uploadType; 531 if (chunked) { 532 uploadType = UPLOAD_TYPE_CHUNKED; 533 } else { 534 uploadType = UPLOAD_TYPE_NORMAL; 535 } 536 info.put("name", fileEntry.getFileName()); 537 info.put("size", fileEntry.getFileSize()); 538 info.put("uploadType", uploadType); 539 if (chunked) { 540 info.put("uploadedChunkIds", fileEntry.getOrderedChunkIndexes()); 541 info.put("chunkCount", fileEntry.getChunkCount()); 542 } 543 return info; 544 } 545 546 public final class ResumeIncompleteStatusType implements StatusType { 547 548 @Override 549 public int getStatusCode() { 550 return 308; 551 } 552 553 @Override 554 public String getReasonPhrase() { 555 return "Resume Incomplete"; 556 } 557 558 @Override 559 public Family getFamily() { 560 // Technically we don't use 308 Resume Incomplete as a redirection but it is the default family for 3xx 561 // status codes defined by Response$Status 562 return Family.REDIRECTION; 563 } 564 } 565 566}