001/*
002 * (C) Copyright 2011-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Luís Duarte
018 *     Florent Guillaume
019 */
020package org.nuxeo.ecm.core.storage.sql;
021
022import static java.lang.Math.min;
023import static org.apache.commons.lang3.StringUtils.isBlank;
024
025import java.util.ArrayList;
026import java.util.List;
027import java.util.stream.Collectors;
028
029import org.nuxeo.ecm.core.api.NuxeoException;
030
031import com.amazonaws.AmazonClientException;
032import com.amazonaws.auth.AWSCredentialsProvider;
033import com.amazonaws.auth.AWSStaticCredentialsProvider;
034import com.amazonaws.auth.BasicAWSCredentials;
035import com.amazonaws.auth.InstanceProfileCredentialsProvider;
036import com.amazonaws.services.s3.AmazonS3;
037import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
038import com.amazonaws.services.s3.model.CopyObjectRequest;
039import com.amazonaws.services.s3.model.CopyPartRequest;
040import com.amazonaws.services.s3.model.CopyPartResult;
041import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
042import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
043import com.amazonaws.services.s3.model.ObjectMetadata;
044import com.amazonaws.services.s3.model.PartETag;
045
046/**
047 * AWS S3 utilities.
048 *
049 * @since 10.1
050 */
051public class S3Utils {
052
053    /** The maximum size of a file that can be copied without using multipart: 5 GB */
054    public static final long NON_MULTIPART_COPY_MAX_SIZE = 5L * 1024 * 1024 * 1024;
055
056    /** The size of the parts that we use for multipart copy. */
057    public static final long PART_SIZE = 5L * 1024 * 1024; // 5 MB
058
059    private S3Utils() {
060        // utility class
061    }
062
063    /**
064     * Represents an operation that accepts a slice number and a slice begin and end position.
065     */
066    @FunctionalInterface
067    public static interface SliceConsumer {
068        /**
069         * Performs this operation on the arguments.
070         *
071         * @param num the slice number, starting at 0
072         * @param begin the begin position
073         * @param end the end position + 1
074         */
075        public void accept(int num, long begin, long end);
076    }
077
078    /**
079     * Calls the consumer on all slices.
080     *
081     * @param slice the slice size
082     * @param length the total length
083     * @param consumer the slice consumer
084     */
085    public static void processSlices(long slice, long length, SliceConsumer consumer) {
086        if (slice <= 0) {
087            throw new IllegalArgumentException("Invalid slice length: " + slice);
088        }
089        long begin = 0;
090        for (int num = 0; begin < length; num++) {
091            long end = min(begin + slice, length);
092            consumer.accept(num, begin, end);
093            begin += slice;
094        }
095    }
096
097    /**
098     * Copies a file using multipart upload.
099     *
100     * @param amazonS3 the S3 client
101     * @param objectMetadata the metadata of the object being copied
102     * @param sourceBucket the source bucket
103     * @param sourceKey the source key
104     * @param targetBucket the target bucket
105     * @param targetKey the target key
106     * @param deleteSource whether to delete the source object if the copy is successful
107     */
108    public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata,
109            String sourceBucket, String sourceKey, String targetBucket, String targetKey, boolean deleteSource) {
110        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(sourceBucket,
111                targetKey);
112        InitiateMultipartUploadResult initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(
113                initiateMultipartUploadRequest);
114
115        String uploadId = initiateMultipartUploadResult.getUploadId();
116        long objectSize = objectMetadata.getContentLength();
117        List<CopyPartResult> copyResponses = new ArrayList<>();
118
119        SliceConsumer partCopy = (num, begin, end) -> {
120            CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucket)
121                                                               .withSourceKey(sourceKey)
122                                                               .withDestinationBucketName(targetBucket)
123                                                               .withDestinationKey(targetKey)
124                                                               .withFirstByte(begin)
125                                                               .withLastByte(end - 1)
126                                                               .withUploadId(uploadId)
127                                                               .withPartNumber(num + 1);
128            copyResponses.add(amazonS3.copyPart(copyRequest));
129        };
130        processSlices(PART_SIZE, objectSize, partCopy);
131
132        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey,
133                uploadId, responsesToETags(copyResponses));
134        amazonS3.completeMultipartUpload(completeRequest);
135        if (deleteSource) {
136            amazonS3.deleteObject(sourceBucket, sourceKey);
137        }
138        return amazonS3.getObjectMetadata(targetBucket, targetKey);
139    }
140
141    protected static List<PartETag> responsesToETags(List<CopyPartResult> responses) {
142        return responses.stream().map(response -> new PartETag(response.getPartNumber(), response.getETag())).collect(
143                Collectors.toList());
144    }
145
146    /**
147     * Copies a file without using multipart upload.
148     *
149     * @param amazonS3 the S3 client
150     * @param objectMetadata the metadata of the object being copied
151     * @param sourceBucket the source bucket
152     * @param sourceKey the source key
153     * @param targetBucket the target bucket
154     * @param targetKey the target key
155     * @param deleteSource whether to delete the source object if the copy is successful
156     */
157    public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket,
158            String sourceKey, String targetBucket, String targetKey, boolean deleteSource) {
159        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, targetBucket, targetKey);
160        amazonS3.copyObject(copyObjectRequest);
161        if (deleteSource) {
162            amazonS3.deleteObject(sourceBucket, sourceKey);
163        }
164        return amazonS3.getObjectMetadata(targetBucket, targetKey);
165    }
166
167    /**
168     * Gets the credentials providers for the given AWS key and secret.
169     *
170     * @param awsSecretKeyId the AWS key id
171     * @param awsSecretAccessKey the secret
172     */
173    public static AWSCredentialsProvider getAWSCredentialsProvider(String awsSecretKeyId, String awsSecretAccessKey) {
174        AWSCredentialsProvider awsCredentialsProvider;
175        if (isBlank(awsSecretKeyId) || isBlank(awsSecretAccessKey)) {
176            awsCredentialsProvider = InstanceProfileCredentialsProvider.getInstance();
177            try {
178                awsCredentialsProvider.getCredentials();
179            } catch (AmazonClientException e) {
180                throw new NuxeoException("Missing AWS credentials and no instance role found", e);
181            }
182        } else {
183            awsCredentialsProvider = new AWSStaticCredentialsProvider(
184                    new BasicAWSCredentials(awsSecretKeyId, awsSecretAccessKey));
185        }
186        return awsCredentialsProvider;
187    }
188
189}