Source code

001/*
002 * (C) Copyright 2011-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Luís Duarte
018 *     Florent Guillaume
019 */
020package org.nuxeo.ecm.core.storage.sql;
021
022import static java.lang.Math.min;
023import static org.apache.commons.lang3.StringUtils.isNotBlank;
024
025import java.util.ArrayList;
026import java.util.List;
027import java.util.stream.Collectors;
028
029import org.nuxeo.runtime.api.Framework;
030import org.nuxeo.runtime.aws.NuxeoAWSCredentialsProvider;
031import org.nuxeo.runtime.services.config.ConfigurationService;
032
033import com.amazonaws.auth.AWSCredentialsProvider;
034import com.amazonaws.auth.AWSStaticCredentialsProvider;
035import com.amazonaws.auth.BasicAWSCredentials;
036import com.amazonaws.auth.BasicSessionCredentials;
037import com.amazonaws.services.s3.AmazonS3;
038import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
039import com.amazonaws.services.s3.model.CopyObjectRequest;
040import com.amazonaws.services.s3.model.CopyPartRequest;
041import com.amazonaws.services.s3.model.CopyPartResult;
042import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
043import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
044import com.amazonaws.services.s3.model.ObjectMetadata;
045import com.amazonaws.services.s3.model.PartETag;
046
047/**
048 * AWS S3 utilities.
049 *
050 * @since 10.1
051 */
052public class S3Utils {
053
054    /** The maximum size of a file that can be copied without using multipart: 5 GB */
055    public static final long NON_MULTIPART_COPY_MAX_SIZE = 5L * 1024 * 1024 * 1024;
056
057    /**
058     * The default multipart copy part size. This default is used only if the configuration service is not available or
059     * if the configuration property {@value #MULTIPART_COPY_PART_SIZE_PROPERTY} is not defined.
060     *
061     * @since 11.1
062     */
063    public static final long MULTIPART_COPY_PART_SIZE_DEFAULT = 5L * 1024 * 1024; // 5 MB
064
065    /**
066     * @deprecated since 11.1, use {@link #MULTIPART_COPY_PART_SIZE_DEFAULT} instead
067     */
068    @Deprecated
069    public static final long PART_SIZE = MULTIPART_COPY_PART_SIZE_DEFAULT;
070
071    /**
072     * The configuration property to define the multipart copy part size.
073     *
074     * @since 11.1
075     */
076    public static final String MULTIPART_COPY_PART_SIZE_PROPERTY = "nuxeo.s3.multipart.copy.part.size";
077
078    private S3Utils() {
079        // utility class
080    }
081
082    /**
083     * Represents an operation that accepts a slice number and a slice begin and end position.
084     */
085    @FunctionalInterface
086    public interface SliceConsumer {
087        /**
088         * Performs this operation on the arguments.
089         *
090         * @param num the slice number, starting at 0
091         * @param begin the begin position
092         * @param end the end position + 1
093         */
094        void accept(int num, long begin, long end);
095    }
096
097    /**
098     * Calls the consumer on all slices.
099     *
100     * @param slice the slice size
101     * @param length the total length
102     * @param consumer the slice consumer
103     */
104    public static void processSlices(long slice, long length, SliceConsumer consumer) {
105        if (slice <= 0) {
106            throw new IllegalArgumentException("Invalid slice length: " + slice);
107        }
108        long begin = 0;
109        for (int num = 0; begin < length; num++) {
110            long end = min(begin + slice, length);
111            consumer.accept(num, begin, end);
112            begin += slice;
113        }
114    }
115
116    /**
117     * Copies a file, using multipart upload if needed.
118     *
119     * @param amazonS3 the S3 client
120     * @param objectMetadata the metadata of the object being copied
121     * @param sourceBucket the source bucket
122     * @param sourceKey the source key
123     * @param targetBucket the target bucket
124     * @param targetKey the target key
125     * @param targetSSEAlgorithm the target SSE Algorithm to use, or {@code null}
126     * @param deleteSource whether to delete the source object if the copy is successful
127     *
128     * @since 11.1
129     * @deprecated since 11.2, use {@link com.amazonaws.services.s3.transfer.TransferManager#copy} instead
130     */
131    @Deprecated
132    public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket,
133            String sourceKey, String targetBucket, String targetKey, String targetSSEAlgorithm, boolean deleteSource) {
134        if (objectMetadata.getContentLength() > NON_MULTIPART_COPY_MAX_SIZE) {
135            return copyFileMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, targetSSEAlgorithm, deleteSource);
136        } else {
137            return copyFileNonMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, targetSSEAlgorithm, deleteSource);
138        }
139    }
140
141    /**
142     * Copies a file using multipart upload.
143     *
144     * @param amazonS3 the S3 client
145     * @param objectMetadata the metadata of the object being copied
146     * @param sourceBucket the source bucket
147     * @param sourceKey the source key
148     * @param targetBucket the target bucket
149     * @param targetKey the target key
150     * @param deleteSource whether to delete the source object if the copy is successful
151     * @deprecated since 11.1, use
152     *             {@link #copyFileMultipart(AmazonS3, ObjectMetadata, String, String, String, String, String, boolean)}
153     *             instead
154     */
155    @Deprecated
156    public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata,
157            String sourceBucket, String sourceKey, String targetBucket, String targetKey, boolean deleteSource) {
158        return copyFileMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, null,
159                deleteSource);
160    }
161
162    /**
163     * Copies a file using multipart upload.
164     *
165     * @param amazonS3 the S3 client
166     * @param objectMetadata the metadata of the object being copied
167     * @param sourceBucket the source bucket
168     * @param sourceKey the source key
169     * @param targetBucket the target bucket
170     * @param targetKey the target key
171     * @param targetSSEAlgorithm the target SSE Algorithm to use, or {@code null}
172     * @param deleteSource whether to delete the source object if the copy is successful
173     * @since 11.1
174     * @deprecated since 11.2, use {@link com.amazonaws.services.s3.transfer.TransferManager#copy} instead
175     */
176    @Deprecated
177    public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata,
178            String sourceBucket, String sourceKey, String targetBucket, String targetKey, String targetSSEAlgorithm,
179            boolean deleteSource) {
180        InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(sourceBucket,
181                targetKey);
182
183        // server-side encryption
184        if (targetSSEAlgorithm != null) {
185            ObjectMetadata newObjectMetadata = new ObjectMetadata();
186            newObjectMetadata.setSSEAlgorithm(targetSSEAlgorithm);
187            initiateMultipartUploadRequest.setObjectMetadata(newObjectMetadata);
188        }
189
190        InitiateMultipartUploadResult initiateMultipartUploadResult = amazonS3.initiateMultipartUpload(
191                initiateMultipartUploadRequest);
192
193        String uploadId = initiateMultipartUploadResult.getUploadId();
194        long objectSize = objectMetadata.getContentLength();
195        List<CopyPartResult> copyResponses = new ArrayList<>();
196
197        SliceConsumer partCopy = (num, begin, end) -> {
198            CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucket)
199                                                               .withSourceKey(sourceKey)
200                                                               .withDestinationBucketName(targetBucket)
201                                                               .withDestinationKey(targetKey)
202                                                               .withFirstByte(begin)
203                                                               .withLastByte(end - 1)
204                                                               .withUploadId(uploadId)
205                                                               .withPartNumber(num + 1);
206            copyResponses.add(amazonS3.copyPart(copyRequest));
207        };
208        ConfigurationService configurationService = Framework.getService(ConfigurationService.class);
209        long partSize = configurationService == null ? MULTIPART_COPY_PART_SIZE_DEFAULT
210                : configurationService.getLong(MULTIPART_COPY_PART_SIZE_PROPERTY, MULTIPART_COPY_PART_SIZE_DEFAULT);
211        processSlices(partSize, objectSize, partCopy);
212
213        CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey,
214                uploadId, responsesToETags(copyResponses));
215        amazonS3.completeMultipartUpload(completeRequest);
216        if (deleteSource) {
217            amazonS3.deleteObject(sourceBucket, sourceKey);
218        }
219        return amazonS3.getObjectMetadata(targetBucket, targetKey);
220    }
221
222    protected static List<PartETag> responsesToETags(List<CopyPartResult> responses) {
223        return responses.stream().map(response -> new PartETag(response.getPartNumber(), response.getETag())).collect(
224                Collectors.toList());
225    }
226
227    /**
228     * Copies a file without using multipart upload.
229     *
230     * @param amazonS3 the S3 client
231     * @param objectMetadata the metadata of the object being copied
232     * @param sourceBucket the source bucket
233     * @param sourceKey the source key
234     * @param targetBucket the target bucket
235     * @param targetKey the target key
236     * @param deleteSource whether to delete the source object if the copy is successful
237     * @deprecated since 11.1, use {@link #copyFileNonMultipart} instead
238     */
239    @Deprecated
240    public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket,
241            String sourceKey, String targetBucket, String targetKey, boolean deleteSource) {
242        return copyFileNonMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, null, deleteSource);
243    }
244
245    /**
246     * Copies a file without using multipart upload.
247     *
248     * @param amazonS3 the S3 client
249     * @param objectMetadata the metadata of the object being copied
250     * @param sourceBucket the source bucket
251     * @param sourceKey the source key
252     * @param targetBucket the target bucket
253     * @param targetKey the target key
254     * @param targetSSEAlgorithm the target SSE Algorithm to use, or {@code null}
255     * @param deleteSource whether to delete the source object if the copy is successful
256     *
257     * @since 11.1
258     * @deprecated since 11.2, use {@link com.amazonaws.services.s3.transfer.TransferManager#copy} instead
259     */
260    @Deprecated
261    public static ObjectMetadata copyFileNonMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket,
262            String sourceKey, String targetBucket, String targetKey, String targetSSEAlgorithm, boolean deleteSource) {
263        CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, targetBucket, targetKey);
264        // server-side encryption
265        if (targetSSEAlgorithm != null) {
266            ObjectMetadata newObjectMetadata = new ObjectMetadata();
267            newObjectMetadata.setSSEAlgorithm(targetSSEAlgorithm);
268            copyObjectRequest.setNewObjectMetadata(newObjectMetadata);
269        }
270        amazonS3.copyObject(copyObjectRequest);
271        if (deleteSource) {
272            amazonS3.deleteObject(sourceBucket, sourceKey);
273        }
274        return amazonS3.getObjectMetadata(targetBucket, targetKey);
275    }
276
277    /**
278     * Gets the credentials providers for the given AWS key and secret.
279     *
280     * @param accessKeyId the AWS access key id
281     * @param secretKey the secret key
282     * @param sessionToken the session token (optional)
283     *
284     * @since 10.10
285     */
286    public static AWSCredentialsProvider getAWSCredentialsProvider(String accessKeyId, String secretKey,
287            String sessionToken) {
288        if (isNotBlank(accessKeyId) && isNotBlank(secretKey)) {
289            // explicit values from service-specific Nuxeo configuration
290            if (isNotBlank(sessionToken)) {
291                return new AWSStaticCredentialsProvider(
292                        new BasicSessionCredentials(accessKeyId, secretKey, sessionToken));
293            } else {
294                return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, secretKey));
295            }
296        }
297        return NuxeoAWSCredentialsProvider.getInstance();
298    }
299
300}