001/*
002 * (C) Copyright 2011-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Luís Duarte
018 *     Florent Guillaume
019 */
020package org.nuxeo.ecm.core.storage.sql;
021
022import static java.lang.Math.min;
023import static org.apache.commons.lang3.StringUtils.isNotBlank;
024
025import java.util.ArrayList;
026import java.util.List;
027import java.util.stream.Collectors;
028
029import org.nuxeo.runtime.aws.NuxeoAWSCredentialsProvider;
030
031import com.amazonaws.auth.AWSCredentialsProvider;
032import com.amazonaws.auth.AWSStaticCredentialsProvider;
033import com.amazonaws.auth.BasicAWSCredentials;
034import com.amazonaws.auth.BasicSessionCredentials;
035import com.amazonaws.services.s3.AmazonS3;
036import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
037import com.amazonaws.services.s3.model.CopyObjectRequest;
038import com.amazonaws.services.s3.model.CopyPartRequest;
039import com.amazonaws.services.s3.model.CopyPartResult;
040import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
041import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
042import com.amazonaws.services.s3.model.ObjectMetadata;
043import com.amazonaws.services.s3.model.PartETag;
044
045/**
046 * AWS S3 utilities.
047 *
048 * @since 10.1
049 */
050public class S3Utils {
051
052    /** The maximum size of a file that can be copied without using multipart: 5 GB */
053    public static final long NON_MULTIPART_COPY_MAX_SIZE = 5L * 1024 * 1024 * 1024;
054
055    /** The size of the parts that we use for multipart copy. */
056    public static final long PART_SIZE = 5L * 1024 * 1024; // 5 MB
057
058    private S3Utils() {
059        // utility class
060    }
061
062    /**
063     * Represents an operation that accepts a slice number and a slice begin and end position.
064     */
065    @FunctionalInterface
066    public static interface SliceConsumer {
067        /**
068         * Performs this operation on the arguments.
069         *
070         * @param num the slice number, starting at 0
071         * @param begin the begin position
072         * @param end the end position + 1
073         */
074        public void accept(int num, long begin, long end);
075    }
076
077    /**
078     * Calls the consumer on all slices.
079     *
080     * @param slice the slice size
081     * @param length the total length
082     * @param consumer the slice consumer
083     */
084    public static void processSlices(long slice, long length, SliceConsumer consumer) {
085        if (slice <= 0) {
086            throw new IllegalArgumentException("Invalid slice length: " + slice);
087        }
088        long begin = 0;
089        for (int num = 0; begin < length; num++) {
090            long end = min(begin + slice, length);
091            consumer.accept(num, begin, end);
092            begin += slice;
093        }
094    }
095
096    /**
097     * Copies a file using multipart upload.
098     *
099     * @param amazonS3 the S3 client
100     * @param objectMetadata the metadata of the object being copied
101     * @param sourceBucket the source bucket
102     * @param sourceKey the source key
103     * @param targetBucket the target bucket
104     * @param targetKey the target key
105     * @param deleteSource whether to delete the source object if the copy is successful
106     */
107    public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata,
108            String sourceBucket, String sourceKey, String targetBucket, String targetKey, boolean deleteSource) {
109        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(sourceBucket,
110                targetKey);
111        InitiateMultipartUploadResult initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(
112                initiateMultipartUploadRequest);
113
114        String uploadId = initiateMultipartUploadResult.getUploadId();
115        long objectSize = objectMetadata.getContentLength();
116        List<CopyPartResult> copyResponses = new ArrayList<>();
117
118        SliceConsumer partCopy = (num, begin, end) -> {
119            CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucket)
120                                                               .withSourceKey(sourceKey)
121                                                               .withDestinationBucketName(targetBucket)
122                                                               .withDestinationKey(targetKey)
123                                                               .withFirstByte(begin)
124                                                               .withLastByte(end - 1)
125                                                               .withUploadId(uploadId)
126                                                               .withPartNumber(num + 1);
127            copyResponses.add(amazonS3.copyPart(copyRequest));
128        };
129        processSlices(PART_SIZE, objectSize, partCopy);
130
131        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey,
132                uploadId, responsesToETags(copyResponses));
133        amazonS3.completeMultipartUpload(completeRequest);
134        if (deleteSource) {
135            amazonS3.deleteObject(sourceBucket, sourceKey);
136        }
137        return amazonS3.getObjectMetadata(targetBucket, targetKey);
138    }
139
140    protected static List<PartETag> responsesToETags(List<CopyPartResult> responses) {
141        return responses.stream().map(response -> new PartETag(response.getPartNumber(), response.getETag())).collect(
142                Collectors.toList());
143    }
144
145    /**
146     * Copies a file without using multipart upload.
147     *
148     * @param amazonS3 the S3 client
149     * @param objectMetadata the metadata of the object being copied
150     * @param sourceBucket the source bucket
151     * @param sourceKey the source key
152     * @param targetBucket the target bucket
153     * @param targetKey the target key
154     * @param deleteSource whether to delete the source object if the copy is successful
155     */
156    public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket,
157            String sourceKey, String targetBucket, String targetKey, boolean deleteSource) {
158        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, targetBucket, targetKey);
159        amazonS3.copyObject(copyObjectRequest);
160        if (deleteSource) {
161            amazonS3.deleteObject(sourceBucket, sourceKey);
162        }
163        return amazonS3.getObjectMetadata(targetBucket, targetKey);
164    }
165
166    /**
167     * Gets the credentials providers for the given AWS key and secret.
168     *
169     * @param accessKeyId the AWS access key id
170     * @param secretKey the secret key
171     * @param sessionToken the session token (optional)
172     * @since 10.10
173     */
174    public static AWSCredentialsProvider getAWSCredentialsProvider(String accessKeyId, String secretKey,
175            String sessionToken) {
176        if (isNotBlank(accessKeyId) && isNotBlank(secretKey)) {
177            // explicit values from service-specific Nuxeo configuration
178            if (isNotBlank(sessionToken)) {
179                return new AWSStaticCredentialsProvider(
180                        new BasicSessionCredentials(accessKeyId, secretKey, sessionToken));
181            } else {
182                return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, secretKey));
183            }
184        }
185        return NuxeoAWSCredentialsProvider.getInstance();
186    }
187
188}