001/* 002 * (C) Copyright 2011-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Luís Duarte 018 * Florent Guillaume 019 */ 020package org.nuxeo.ecm.core.storage.sql; 021 022import static java.lang.Math.min; 023import static org.apache.commons.lang3.StringUtils.isBlank; 024 025import java.util.ArrayList; 026import java.util.List; 027import java.util.stream.Collectors; 028 029import org.nuxeo.ecm.core.api.NuxeoException; 030 031import com.amazonaws.AmazonClientException; 032import com.amazonaws.auth.AWSCredentialsProvider; 033import com.amazonaws.auth.AWSStaticCredentialsProvider; 034import com.amazonaws.auth.BasicAWSCredentials; 035import com.amazonaws.auth.InstanceProfileCredentialsProvider; 036import com.amazonaws.services.s3.AmazonS3; 037import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; 038import com.amazonaws.services.s3.model.CopyObjectRequest; 039import com.amazonaws.services.s3.model.CopyPartRequest; 040import com.amazonaws.services.s3.model.CopyPartResult; 041import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; 042import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; 043import com.amazonaws.services.s3.model.ObjectMetadata; 044import com.amazonaws.services.s3.model.PartETag; 045 046/** 047 * AWS S3 utilities. 048 * 049 * @since 10.1 050 */ 051public class S3Utils { 052 053 /** The maximum size of a file that can be copied without using multipart: 5 GB */ 054 public static final long NON_MULTIPART_COPY_MAX_SIZE = 5L * 1024 * 1024 * 1024; 055 056 /** The size of the parts that we use for multipart copy. */ 057 public static final long PART_SIZE = 5L * 1024 * 1024; // 5 MB 058 059 private S3Utils() { 060 // utility class 061 } 062 063 /** 064 * Represents an operation that accepts a slice number and a slice begin and end position. 065 */ 066 @FunctionalInterface 067 public static interface SliceConsumer { 068 /** 069 * Performs this operation on the arguments. 070 * 071 * @param num the slice number, starting at 0 072 * @param begin the begin position 073 * @param end the end position + 1 074 */ 075 public void accept(int num, long begin, long end); 076 } 077 078 /** 079 * Calls the consumer on all slices. 080 * 081 * @param slice the slice size 082 * @param length the total length 083 * @param consumer the slice consumer 084 */ 085 public static void processSlices(long slice, long length, SliceConsumer consumer) { 086 if (slice <= 0) { 087 throw new IllegalArgumentException("Invalid slice length: " + slice); 088 } 089 long begin = 0; 090 for (int num = 0; begin < length; num++) { 091 long end = min(begin + slice, length); 092 consumer.accept(num, begin, end); 093 begin += slice; 094 } 095 } 096 097 /** 098 * Copies a file using multipart upload. 099 * 100 * @param amazonS3 the S3 client 101 * @param objectMetadata the metadata of the object being copied 102 * @param sourceBucket the source bucket 103 * @param sourceKey the source key 104 * @param targetBucket the target bucket 105 * @param targetKey the target key 106 * @param deleteSource whether to delete the source object if the copy is successful 107 */ 108 public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata, 109 String sourceBucket, String sourceKey, String targetBucket, String targetKey, boolean deleteSource) { 110 InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(sourceBucket, 111 targetKey); 112 InitiateMultipartUploadResult initiateMultipartUploadResult = amazonS3.initiateMultipartUpload( 113 initiateMultipartUploadRequest); 114 115 String uploadId = initiateMultipartUploadResult.getUploadId(); 116 long objectSize = objectMetadata.getContentLength(); 117 List<CopyPartResult> copyResponses = new ArrayList<>(); 118 119 SliceConsumer partCopy = (num, begin, end) -> { 120 CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucket) 121 .withSourceKey(sourceKey) 122 .withDestinationBucketName(targetBucket) 123 .withDestinationKey(targetKey) 124 .withFirstByte(begin) 125 .withLastByte(end - 1) 126 .withUploadId(uploadId) 127 .withPartNumber(num + 1); 128 copyResponses.add(amazonS3.copyPart(copyRequest)); 129 }; 130 processSlices(PART_SIZE, objectSize, partCopy); 131 132 CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey, 133 uploadId, responsesToETags(copyResponses)); 134 amazonS3.completeMultipartUpload(completeRequest); 135 if (deleteSource) { 136 amazonS3.deleteObject(sourceBucket, sourceKey); 137 } 138 return amazonS3.getObjectMetadata(targetBucket, targetKey); 139 } 140 141 protected static List<PartETag> responsesToETags(List<CopyPartResult> responses) { 142 return responses.stream().map(response -> new PartETag(response.getPartNumber(), response.getETag())).collect( 143 Collectors.toList()); 144 } 145 146 /** 147 * Copies a file without using multipart upload. 148 * 149 * @param amazonS3 the S3 client 150 * @param objectMetadata the metadata of the object being copied 151 * @param sourceBucket the source bucket 152 * @param sourceKey the source key 153 * @param targetBucket the target bucket 154 * @param targetKey the target key 155 * @param deleteSource whether to delete the source object if the copy is successful 156 */ 157 public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket, 158 String sourceKey, String targetBucket, String targetKey, boolean deleteSource) { 159 CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, targetBucket, targetKey); 160 amazonS3.copyObject(copyObjectRequest); 161 if (deleteSource) { 162 amazonS3.deleteObject(sourceBucket, sourceKey); 163 } 164 return amazonS3.getObjectMetadata(targetBucket, targetKey); 165 } 166 167 /** 168 * Gets the credentials providers for the given AWS key and secret. 169 * 170 * @param awsSecretKeyId the AWS key id 171 * @param awsSecretAccessKey the secret 172 */ 173 public static AWSCredentialsProvider getAWSCredentialsProvider(String awsSecretKeyId, String awsSecretAccessKey) { 174 AWSCredentialsProvider awsCredentialsProvider; 175 if (isBlank(awsSecretKeyId) || isBlank(awsSecretAccessKey)) { 176 awsCredentialsProvider = InstanceProfileCredentialsProvider.getInstance(); 177 try { 178 awsCredentialsProvider.getCredentials(); 179 } catch (AmazonClientException e) { 180 throw new NuxeoException("Missing AWS credentials and no instance role found", e); 181 } 182 } else { 183 awsCredentialsProvider = new AWSStaticCredentialsProvider( 184 new BasicAWSCredentials(awsSecretKeyId, awsSecretAccessKey)); 185 } 186 return awsCredentialsProvider; 187 } 188 189}