001/* 002 * (C) Copyright 2011-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Luís Duarte 018 * Florent Guillaume 019 */ 020package org.nuxeo.ecm.core.storage.sql; 021 022import static java.lang.Math.min; 023import static org.apache.commons.lang3.StringUtils.isNotBlank; 024 025import java.util.ArrayList; 026import java.util.List; 027import java.util.stream.Collectors; 028 029import org.nuxeo.runtime.aws.NuxeoAWSCredentialsProvider; 030 031import com.amazonaws.auth.AWSCredentialsProvider; 032import com.amazonaws.auth.AWSStaticCredentialsProvider; 033import com.amazonaws.auth.BasicAWSCredentials; 034import com.amazonaws.auth.BasicSessionCredentials; 035import com.amazonaws.services.s3.AmazonS3; 036import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; 037import com.amazonaws.services.s3.model.CopyObjectRequest; 038import com.amazonaws.services.s3.model.CopyPartRequest; 039import com.amazonaws.services.s3.model.CopyPartResult; 040import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; 041import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; 042import com.amazonaws.services.s3.model.ObjectMetadata; 043import com.amazonaws.services.s3.model.PartETag; 044 045/** 046 * AWS S3 utilities. 047 * 048 * @since 10.1 049 */ 050public class S3Utils { 051 052 /** The maximum size of a file that can be copied without using multipart: 5 GB */ 053 public static final long NON_MULTIPART_COPY_MAX_SIZE = 5L * 1024 * 1024 * 1024; 054 055 /** The size of the parts that we use for multipart copy. */ 056 public static final long PART_SIZE = 5L * 1024 * 1024; // 5 MB 057 058 private S3Utils() { 059 // utility class 060 } 061 062 /** 063 * Represents an operation that accepts a slice number and a slice begin and end position. 064 */ 065 @FunctionalInterface 066 public static interface SliceConsumer { 067 /** 068 * Performs this operation on the arguments. 069 * 070 * @param num the slice number, starting at 0 071 * @param begin the begin position 072 * @param end the end position + 1 073 */ 074 public void accept(int num, long begin, long end); 075 } 076 077 /** 078 * Calls the consumer on all slices. 079 * 080 * @param slice the slice size 081 * @param length the total length 082 * @param consumer the slice consumer 083 */ 084 public static void processSlices(long slice, long length, SliceConsumer consumer) { 085 if (slice <= 0) { 086 throw new IllegalArgumentException("Invalid slice length: " + slice); 087 } 088 long begin = 0; 089 for (int num = 0; begin < length; num++) { 090 long end = min(begin + slice, length); 091 consumer.accept(num, begin, end); 092 begin += slice; 093 } 094 } 095 096 /** 097 * Copies a file using multipart upload. 098 * 099 * @param amazonS3 the S3 client 100 * @param objectMetadata the metadata of the object being copied 101 * @param sourceBucket the source bucket 102 * @param sourceKey the source key 103 * @param targetBucket the target bucket 104 * @param targetKey the target key 105 * @param deleteSource whether to delete the source object if the copy is successful 106 */ 107 public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata, 108 String sourceBucket, String sourceKey, String targetBucket, String targetKey, boolean deleteSource) { 109 InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(sourceBucket, 110 targetKey); 111 InitiateMultipartUploadResult initiateMultipartUploadResult = amazonS3.initiateMultipartUpload( 112 initiateMultipartUploadRequest); 113 114 String uploadId = initiateMultipartUploadResult.getUploadId(); 115 long objectSize = objectMetadata.getContentLength(); 116 List<CopyPartResult> copyResponses = new ArrayList<>(); 117 118 SliceConsumer partCopy = (num, begin, end) -> { 119 CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucket) 120 .withSourceKey(sourceKey) 121 .withDestinationBucketName(targetBucket) 122 .withDestinationKey(targetKey) 123 .withFirstByte(begin) 124 .withLastByte(end - 1) 125 .withUploadId(uploadId) 126 .withPartNumber(num + 1); 127 copyResponses.add(amazonS3.copyPart(copyRequest)); 128 }; 129 processSlices(PART_SIZE, objectSize, partCopy); 130 131 CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey, 132 uploadId, responsesToETags(copyResponses)); 133 amazonS3.completeMultipartUpload(completeRequest); 134 if (deleteSource) { 135 amazonS3.deleteObject(sourceBucket, sourceKey); 136 } 137 return amazonS3.getObjectMetadata(targetBucket, targetKey); 138 } 139 140 protected static List<PartETag> responsesToETags(List<CopyPartResult> responses) { 141 return responses.stream().map(response -> new PartETag(response.getPartNumber(), response.getETag())).collect( 142 Collectors.toList()); 143 } 144 145 /** 146 * Copies a file without using multipart upload. 147 * 148 * @param amazonS3 the S3 client 149 * @param objectMetadata the metadata of the object being copied 150 * @param sourceBucket the source bucket 151 * @param sourceKey the source key 152 * @param targetBucket the target bucket 153 * @param targetKey the target key 154 * @param deleteSource whether to delete the source object if the copy is successful 155 */ 156 public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket, 157 String sourceKey, String targetBucket, String targetKey, boolean deleteSource) { 158 CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, targetBucket, targetKey); 159 amazonS3.copyObject(copyObjectRequest); 160 if (deleteSource) { 161 amazonS3.deleteObject(sourceBucket, sourceKey); 162 } 163 return amazonS3.getObjectMetadata(targetBucket, targetKey); 164 } 165 166 /** 167 * Gets the credentials providers for the given AWS key and secret. 168 * 169 * @param accessKeyId the AWS access key id 170 * @param secretKey the secret key 171 * @param sessionToken the session token (optional) 172 * @since 10.10 173 */ 174 public static AWSCredentialsProvider getAWSCredentialsProvider(String accessKeyId, String secretKey, 175 String sessionToken) { 176 if (isNotBlank(accessKeyId) && isNotBlank(secretKey)) { 177 // explicit values from service-specific Nuxeo configuration 178 if (isNotBlank(sessionToken)) { 179 return new AWSStaticCredentialsProvider( 180 new BasicSessionCredentials(accessKeyId, secretKey, sessionToken)); 181 } else { 182 return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, secretKey)); 183 } 184 } 185 return NuxeoAWSCredentialsProvider.getInstance(); 186 } 187 188}