001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Antoine Taillefer <ataillefer@nuxeo.com>
018 */
019package org.nuxeo.ecm.restapi.server.jaxrs;
020
021import java.io.IOException;
022import java.io.InputStream;
023import java.io.UnsupportedEncodingException;
024import java.net.URLDecoder;
025import java.util.ArrayList;
026import java.util.HashMap;
027import java.util.List;
028import java.util.Map;
029
030import javax.mail.MessagingException;
031import javax.servlet.http.HttpServletRequest;
032import javax.servlet.http.HttpServletResponse;
033import javax.ws.rs.DELETE;
034import javax.ws.rs.GET;
035import javax.ws.rs.POST;
036import javax.ws.rs.Path;
037import javax.ws.rs.PathParam;
038import javax.ws.rs.Produces;
039import javax.ws.rs.core.Context;
040import javax.ws.rs.core.MediaType;
041import javax.ws.rs.core.Response;
042import javax.ws.rs.core.Response.Status;
043import javax.ws.rs.core.Response.Status.Family;
044import javax.ws.rs.core.Response.StatusType;
045
046import org.apache.commons.collections.CollectionUtils;
047import org.apache.commons.lang.math.NumberUtils;
048import org.apache.commons.lang3.StringUtils;
049import org.apache.commons.logging.Log;
050import org.apache.commons.logging.LogFactory;
051import org.codehaus.jackson.map.ObjectMapper;
052import org.nuxeo.ecm.automation.OperationContext;
053import org.nuxeo.ecm.automation.jaxrs.io.operations.ExecutionRequest;
054import org.nuxeo.ecm.automation.server.jaxrs.ResponseHelper;
055import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchFileEntry;
056import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchManager;
057import org.nuxeo.ecm.automation.server.jaxrs.batch.BatchManagerConstants;
058import org.nuxeo.ecm.core.api.Blob;
059import org.nuxeo.ecm.core.api.CoreSession;
060import org.nuxeo.ecm.core.api.NuxeoException;
061import org.nuxeo.ecm.webengine.forms.FormData;
062import org.nuxeo.ecm.webengine.jaxrs.context.RequestContext;
063import org.nuxeo.ecm.webengine.model.WebObject;
064import org.nuxeo.ecm.webengine.model.exceptions.IllegalParameterException;
065import org.nuxeo.ecm.webengine.model.impl.AbstractResource;
066import org.nuxeo.ecm.webengine.model.impl.ResourceTypeImpl;
067import org.nuxeo.runtime.api.Framework;
068import org.nuxeo.runtime.transaction.TransactionHelper;
069
070/**
071 * Batch upload endpoint.
072 * <p>
073 * Provides the APIs listed below:
074 * <ul>
075 * <li>POST /upload, see {@link #initBatch()}</li>
076 * <li>POST /upload/{batchId}/{fileIdx}, see {@link #upload(HttpServletRequest, String, String)}</li>
077 * <li>GET /upload/{batchId}, see {@link #getBatchInfo(String)}</li>
078 * <li>GET /upload/{batchId}/{fileIdx}, see {@link #getFileInfo(String, String)}</li>
079 * <li>POST /upload/{batchId}/execute/{operationId}, see {@link #execute(String, String, ExecutionRequest)}</li>
080 * <li>POST /upload/{batchId}/{fileIdx}/execute/{operationId}, see
081 * {@link #execute(String, String, String, ExecutionRequest)}</li>
082 * <li>DELETE /upload/{batchId}, see {@link #cancel(String)}</li>
083 * <li>DELETE /upload/{batchId}/{fileIdx}, see {@link #removeFile(String, String)}</li>
084 * </ul>
085 * Largely inspired by the excellent Google Drive REST API documentation about
086 * <a href="https://developers.google.com/drive/web/manage-uploads#resumable">resumable upload</a>.
087 *
088 * @since 7.4
089 */
090@WebObject(type = "upload")
091public class BatchUploadObject extends AbstractResource<ResourceTypeImpl> {
092
093    protected static final Log log = LogFactory.getLog(BatchUploadObject.class);
094
095    protected static final String REQUEST_BATCH_ID = "batchId";
096
097    protected static final String REQUEST_FILE_IDX = "fileIdx";
098
099    protected static final String OPERATION_ID = "operationId";
100
101    public static final String UPLOAD_TYPE_NORMAL = "normal";
102
103    public static final String UPLOAD_TYPE_CHUNKED = "chunked";
104
105    @POST
106    public Response initBatch() throws IOException {
107        BatchManager bm = Framework.getService(BatchManager.class);
108        String batchId = bm.initBatch();
109        Map<String, String> result = new HashMap<String, String>();
110        result.put("batchId", batchId);
111        return buildResponse(Status.CREATED, result);
112    }
113
114    @POST
115    @Path("{batchId}/{fileIdx}")
116    public Response upload(@Context HttpServletRequest request, @PathParam(REQUEST_BATCH_ID) String batchId,
117            @PathParam(REQUEST_FILE_IDX) String fileIdx) throws IOException {
118        TransactionHelper.commitOrRollbackTransaction();
119        try {
120            return uploadNoTransaction(request, batchId, fileIdx);
121        } finally {
122            TransactionHelper.startTransaction();
123        }
124    }
125
126    protected Response uploadNoTransaction(@Context HttpServletRequest request,
127            @PathParam(REQUEST_BATCH_ID) String batchId, @PathParam(REQUEST_FILE_IDX) String fileIdx)
128            throws IOException {
129
130        if (!Framework.getService(BatchManager.class).hasBatch(batchId)) {
131            return buildEmptyResponse(Status.NOT_FOUND);
132        }
133
134        // Check file index parameter
135        if (!NumberUtils.isDigits(fileIdx)) {
136            return buildTextResponse(Status.BAD_REQUEST, "fileIdx request path parameter must be a number");
137        }
138
139        // Parameters are passed as request header, the request body is the stream
140        String contentType = request.getHeader("Content-Type");
141        String uploadType = request.getHeader("X-Upload-Type");
142        // Use non chunked mode by default if X-Upload-Type header is not provided
143        if (!UPLOAD_TYPE_CHUNKED.equals(uploadType)) {
144            uploadType = UPLOAD_TYPE_NORMAL;
145        }
146        String uploadChunkIndexHeader = request.getHeader("X-Upload-Chunk-Index");
147        String chunkCountHeader = request.getHeader("X-Upload-Chunk-Count");
148        String fileName = request.getHeader("X-File-Name");
149        String fileSizeHeader = request.getHeader("X-File-Size");
150        String mimeType = request.getHeader("X-File-Type");
151
152        int chunkCount = -1;
153        int uploadChunkIndex = -1;
154        long fileSize = -1;
155        if (UPLOAD_TYPE_CHUNKED.equals(uploadType)) {
156            try {
157                chunkCount = Integer.parseInt(chunkCountHeader);
158                uploadChunkIndex = Integer.parseInt(uploadChunkIndexHeader);
159                fileSize = Long.parseLong(fileSizeHeader);
160            } catch (NumberFormatException e) {
161                throw new IllegalParameterException(
162                        "X-Upload-Chunk-Index, X-Upload-Chunk-Count and X-File-Size headers must be numbers");
163            }
164        }
165
166        // TODO NXP-18247: should be set to the actual number of bytes uploaded instead of relying on the Content-Length
167        // header which is not necessarily set
168        long uploadedSize = getUploadedSize(request);
169        boolean isMultipart = contentType != null && contentType.contains("multipart");
170
171        // Handle multipart case: mainly MSIE with jQueryFileupload
172        if (isMultipart) {
173            FormData formData = new FormData(request);
174            Blob blob = formData.getFirstBlob();
175            if (blob == null) {
176                throw new NuxeoException("Cannot upload in multipart with no blobs");
177            }
178            if (!UPLOAD_TYPE_CHUNKED.equals(uploadType)) {
179                fileName = blob.getFilename();
180            }
181            // Don't change the mime-type if it was forced via the X-File-Type header
182            if (StringUtils.isBlank(mimeType)) {
183                mimeType = blob.getMimeType();
184            }
185            uploadedSize = blob.getLength();
186            try (InputStream is = blob.getStream()) {
187                addStream(uploadType, batchId, fileIdx, is, fileName, mimeType, uploadedSize, chunkCount,
188                        uploadChunkIndex, fileSize);
189            }
190        } else {
191            if (fileName != null) {
192                fileName = URLDecoder.decode(fileName, "UTF-8");
193            }
194            try (InputStream is = request.getInputStream()) {
195                addStream(uploadType, batchId, fileIdx, is, fileName, mimeType, uploadedSize, chunkCount,
196                        uploadChunkIndex, fileSize);
197            }
198        }
199
200        StatusType status = Status.CREATED;
201        Map<String, Object> result = new HashMap<>();
202        result.put("uploaded", "true");
203        result.put("batchId", batchId);
204        result.put("fileIdx", fileIdx);
205        result.put("uploadType", uploadType);
206        result.put("uploadedSize", String.valueOf(uploadedSize));
207        if (UPLOAD_TYPE_CHUNKED.equals(uploadType)) {
208            BatchFileEntry fileEntry = Framework.getService(BatchManager.class).getFileEntry(batchId, fileIdx);
209            if (fileEntry != null) {
210                result.put("uploadedChunkIds", fileEntry.getOrderedChunkIndexes());
211                result.put("chunkCount", fileEntry.getChunkCount());
212                if (!fileEntry.isChunksCompleted()) {
213                    status = new ResumeIncompleteStatusType();
214                }
215            }
216        }
217        return buildResponse(status, result, isMultipart);
218    }
219
220    protected long getUploadedSize(HttpServletRequest request) {
221        String contentLength = request.getHeader("Content-Length");
222        if (contentLength == null) {
223            return -1;
224        }
225        return Long.parseLong(contentLength);
226    }
227
228    protected void addStream(String uploadType, String batchId, String fileIdx, InputStream is, String fileName,
229            String mimeType, long uploadedSize, int chunkCount, int uploadChunkIndex, long fileSize)
230            throws IOException {
231        String uploadedSizeDisplay = uploadedSize > -1 ? uploadedSize + "b" : "unknown size";
232        if (UPLOAD_TYPE_CHUNKED.equals(uploadType)) {
233            if (log.isDebugEnabled()) {
234                log.debug(String.format("Uploading chunk [index=%d / total=%d] (%s) for file %s", uploadChunkIndex,
235                        chunkCount, uploadedSizeDisplay, fileName));
236            }
237            Framework.getService(BatchManager.class).addStream(batchId, fileIdx, is, chunkCount, uploadChunkIndex,
238                    fileName, mimeType, fileSize);
239        } else {
240            if (log.isDebugEnabled()) {
241                log.debug(String.format("Uploading file %s (%s)", fileName, uploadedSizeDisplay));
242            }
243            Framework.getService(BatchManager.class).addStream(batchId, fileIdx, is, fileName, mimeType);
244        }
245    }
246
247    @GET
248    @Path("{batchId}")
249    public Response getBatchInfo(@PathParam(REQUEST_BATCH_ID) String batchId) throws IOException {
250        BatchManager bm = Framework.getService(BatchManager.class);
251        if (!bm.hasBatch(batchId)) {
252            return buildEmptyResponse(Status.NOT_FOUND);
253        }
254        List<BatchFileEntry> fileEntries = bm.getFileEntries(batchId);
255        if (CollectionUtils.isEmpty(fileEntries)) {
256            return buildEmptyResponse(Status.NO_CONTENT);
257        }
258        List<Map<String, Object>> result = new ArrayList<>();
259        for (BatchFileEntry fileEntry : fileEntries) {
260            result.add(getFileInfo(fileEntry));
261        }
262        return buildResponse(Status.OK, result);
263    }
264
265    @GET
266    @Path("{batchId}/{fileIdx}")
267    public Response getFileInfo(@PathParam(REQUEST_BATCH_ID) String batchId,
268            @PathParam(REQUEST_FILE_IDX) String fileIdx) throws IOException {
269        BatchManager bm = Framework.getService(BatchManager.class);
270        if (!bm.hasBatch(batchId)) {
271            return buildEmptyResponse(Status.NOT_FOUND);
272        }
273        BatchFileEntry fileEntry = bm.getFileEntry(batchId, fileIdx);
274        if (fileEntry == null) {
275            return buildEmptyResponse(Status.NOT_FOUND);
276        }
277        StatusType status = Status.OK;
278        if (fileEntry.isChunked() && !fileEntry.isChunksCompleted()) {
279            status = new ResumeIncompleteStatusType();
280        }
281        Map<String, Object> result = getFileInfo(fileEntry);
282        return buildResponse(status, result);
283    }
284
285    @DELETE
286    @Path("{batchId}")
287    public Response cancel(@PathParam(REQUEST_BATCH_ID) String batchId) {
288        BatchManager bm = Framework.getService(BatchManager.class);
289        if (!bm.hasBatch(batchId)) {
290            return buildEmptyResponse(Status.NOT_FOUND);
291        }
292        bm.clean(batchId);
293        return buildEmptyResponse(Status.NO_CONTENT);
294    }
295
296    /**
297     * @since 8.4
298     */
299    @DELETE
300    @Path("{batchId}/{fileIdx}")
301    public Response removeFile(@PathParam(REQUEST_BATCH_ID) String batchId,
302            @PathParam(REQUEST_FILE_IDX) String fileIdx) {
303        BatchManager bm = Framework.getService(BatchManager.class);
304        if (!bm.removeFileEntry(batchId, fileIdx)) {
305            return buildEmptyResponse(Status.NOT_FOUND);
306        }
307        return buildEmptyResponse(Status.NO_CONTENT);
308    }
309
310    @Context
311    protected HttpServletRequest request;
312
313    @Context
314    protected HttpServletResponse response;
315
316    @POST
317    @Produces("application/json")
318    @Path("{batchId}/execute/{operationId}")
319    public Object execute(@PathParam(REQUEST_BATCH_ID) String batchId, @PathParam(OPERATION_ID) String operationId,
320            ExecutionRequest xreq) {
321        return executeBatch(batchId, null, operationId, request, xreq);
322    }
323
324    @POST
325    @Produces("application/json")
326    @Path("{batchId}/{fileIdx}/execute/{operationId}")
327    public Object execute(@PathParam(REQUEST_BATCH_ID) String batchId, @PathParam(REQUEST_FILE_IDX) String fileIdx,
328            @PathParam(OPERATION_ID) String operationId, ExecutionRequest xreq) {
329        return executeBatch(batchId, fileIdx, operationId, request, xreq);
330    }
331
332    protected Object executeBatch(String batchId, String fileIdx, String operationId, HttpServletRequest request,
333            ExecutionRequest xreq) {
334
335        if (!Framework.getService(BatchManager.class).hasBatch(batchId)) {
336            return buildEmptyResponse(Status.NOT_FOUND);
337        }
338
339        if (!Boolean.parseBoolean(
340                RequestContext.getActiveContext(request).getRequest().getHeader(BatchManagerConstants.NO_DROP_FLAG))) {
341            RequestContext.getActiveContext(request).addRequestCleanupHandler(req -> {
342                BatchManager bm = Framework.getService(BatchManager.class);
343                bm.clean(batchId);
344            });
345        }
346
347        try {
348            CoreSession session = ctx.getCoreSession();
349            OperationContext ctx = xreq.createContext(request, response, session);
350            Map<String, Object> params = xreq.getParams();
351            BatchManager bm = Framework.getService(BatchManager.class);
352            Object result;
353            if (StringUtils.isBlank(fileIdx)) {
354                result = bm.execute(batchId, operationId, session, ctx, params);
355            } else {
356                result = bm.execute(batchId, fileIdx, operationId, session, ctx, params);
357            }
358            return ResponseHelper.getResponse(result, request);
359        } catch (MessagingException | IOException e) {
360            log.error("Error while executing automation batch ", e);
361            throw new NuxeoException(e);
362        }
363    }
364
365    protected Response buildResponse(StatusType status, Object object) throws IOException {
366        return buildResponse(status, object, false);
367    }
368
369    protected Response buildResponse(StatusType status, Object object, boolean html) throws IOException {
370        ObjectMapper mapper = new ObjectMapper();
371        String result = mapper.writeValueAsString(object);
372        if (html) {
373            // For MSIE with iframe transport: we need to return HTML!
374            return buildHTMLResponse(status, result);
375        } else {
376            return buildJSONResponse(status, result);
377        }
378    }
379
380    protected Response buildJSONResponse(StatusType status, String message) throws UnsupportedEncodingException {
381        return buildResponse(status, MediaType.APPLICATION_JSON, message);
382    }
383
384    protected Response buildHTMLResponse(StatusType status, String message) throws UnsupportedEncodingException {
385        message = "<html>" + message + "</html>";
386        return buildResponse(status, MediaType.TEXT_HTML, message);
387    }
388
389    protected Response buildTextResponse(StatusType status, String message) throws UnsupportedEncodingException {
390        return buildResponse(status, MediaType.TEXT_PLAIN, message);
391    }
392
393    protected Response buildEmptyResponse(StatusType status) {
394        return Response.status(status).build();
395    }
396
397    protected Response buildResponse(StatusType status, String type, String message)
398            throws UnsupportedEncodingException {
399        return Response.status(status)
400                       .header("Content-Length", message.getBytes("UTF-8").length)
401                       .type(type + "; charset=UTF-8")
402                       .entity(message)
403                       .build();
404    }
405
406    protected Map<String, Object> getFileInfo(BatchFileEntry fileEntry) {
407        Map<String, Object> info = new HashMap<>();
408        boolean chunked = fileEntry.isChunked();
409        String uploadType;
410        if (chunked) {
411            uploadType = UPLOAD_TYPE_CHUNKED;
412        } else {
413            uploadType = UPLOAD_TYPE_NORMAL;
414        }
415        info.put("name", fileEntry.getFileName());
416        info.put("size", fileEntry.getFileSize());
417        info.put("uploadType", uploadType);
418        if (chunked) {
419            info.put("uploadedChunkIds", fileEntry.getOrderedChunkIndexes());
420            info.put("chunkCount", fileEntry.getChunkCount());
421        }
422        return info;
423    }
424
425    public final class ResumeIncompleteStatusType implements StatusType {
426
427        @Override
428        public int getStatusCode() {
429            return 308;
430        }
431
432        @Override
433        public String getReasonPhrase() {
434            return "Resume Incomplete";
435        }
436
437        @Override
438        public Family getFamily() {
439            // Technically we don't use 308 Resume Incomplete as a redirection but it is the default family for 3xx
440            // status codes defined by Response$Status
441            return Family.REDIRECTION;
442        }
443    }
444
445}