001/* 002 * (C) Copyright 2011-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Luís Duarte 018 * Florent Guillaume 019 */ 020package org.nuxeo.ecm.core.storage.sql; 021 022import static java.lang.Math.min; 023import static org.apache.commons.lang3.StringUtils.isNotBlank; 024 025import java.util.ArrayList; 026import java.util.List; 027import java.util.stream.Collectors; 028 029import org.nuxeo.runtime.api.Framework; 030import org.nuxeo.runtime.aws.NuxeoAWSCredentialsProvider; 031import org.nuxeo.runtime.services.config.ConfigurationService; 032 033import com.amazonaws.auth.AWSCredentialsProvider; 034import com.amazonaws.auth.AWSStaticCredentialsProvider; 035import com.amazonaws.auth.BasicAWSCredentials; 036import com.amazonaws.auth.BasicSessionCredentials; 037import com.amazonaws.services.s3.AmazonS3; 038import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; 039import com.amazonaws.services.s3.model.CopyObjectRequest; 040import com.amazonaws.services.s3.model.CopyPartRequest; 041import com.amazonaws.services.s3.model.CopyPartResult; 042import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; 043import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; 044import com.amazonaws.services.s3.model.ObjectMetadata; 045import com.amazonaws.services.s3.model.PartETag; 046 047/** 048 * AWS S3 utilities. 049 * 050 * @since 10.1 051 */ 052public class S3Utils { 053 054 /** The maximum size of a file that can be copied without using multipart: 5 GB */ 055 public static final long NON_MULTIPART_COPY_MAX_SIZE = 5L * 1024 * 1024 * 1024; 056 057 /** 058 * The default multipart copy part size. This default is used only if the configuration service is not available or 059 * if the configuration property {@value #MULTIPART_COPY_PART_SIZE_PROPERTY} is not defined. 060 * 061 * @since 11.1 062 */ 063 public static final long MULTIPART_COPY_PART_SIZE_DEFAULT = 5L * 1024 * 1024; // 5 MB 064 065 /** 066 * @deprecated since 11.1, use {@link #MULTIPART_COPY_PART_SIZE_DEFAULT} instead 067 */ 068 @Deprecated 069 public static final long PART_SIZE = MULTIPART_COPY_PART_SIZE_DEFAULT; 070 071 /** 072 * The configuration property to define the multipart copy part size. 073 * 074 * @since 11.1 075 */ 076 public static final String MULTIPART_COPY_PART_SIZE_PROPERTY = "nuxeo.s3.multipart.copy.part.size"; 077 078 private S3Utils() { 079 // utility class 080 } 081 082 /** 083 * Represents an operation that accepts a slice number and a slice begin and end position. 084 */ 085 @FunctionalInterface 086 public interface SliceConsumer { 087 /** 088 * Performs this operation on the arguments. 089 * 090 * @param num the slice number, starting at 0 091 * @param begin the begin position 092 * @param end the end position + 1 093 */ 094 void accept(int num, long begin, long end); 095 } 096 097 /** 098 * Calls the consumer on all slices. 099 * 100 * @param slice the slice size 101 * @param length the total length 102 * @param consumer the slice consumer 103 */ 104 public static void processSlices(long slice, long length, SliceConsumer consumer) { 105 if (slice <= 0) { 106 throw new IllegalArgumentException("Invalid slice length: " + slice); 107 } 108 long begin = 0; 109 for (int num = 0; begin < length; num++) { 110 long end = min(begin + slice, length); 111 consumer.accept(num, begin, end); 112 begin += slice; 113 } 114 } 115 116 /** 117 * Copies a file, using multipart upload if needed. 118 * 119 * @param amazonS3 the S3 client 120 * @param objectMetadata the metadata of the object being copied 121 * @param sourceBucket the source bucket 122 * @param sourceKey the source key 123 * @param targetBucket the target bucket 124 * @param targetKey the target key 125 * @param targetSSEAlgorithm the target SSE Algorithm to use, or {@code null} 126 * @param deleteSource whether to delete the source object if the copy is successful 127 * 128 * @since 11.1 129 * @deprecated since 11.2, use {@link com.amazonaws.services.s3.transfer.TransferManager#copy} instead 130 */ 131 @Deprecated 132 public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket, 133 String sourceKey, String targetBucket, String targetKey, String targetSSEAlgorithm, boolean deleteSource) { 134 if (objectMetadata.getContentLength() > NON_MULTIPART_COPY_MAX_SIZE) { 135 return copyFileMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, targetSSEAlgorithm, deleteSource); 136 } else { 137 return copyFileNonMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, targetSSEAlgorithm, deleteSource); 138 } 139 } 140 141 /** 142 * Copies a file using multipart upload. 143 * 144 * @param amazonS3 the S3 client 145 * @param objectMetadata the metadata of the object being copied 146 * @param sourceBucket the source bucket 147 * @param sourceKey the source key 148 * @param targetBucket the target bucket 149 * @param targetKey the target key 150 * @param deleteSource whether to delete the source object if the copy is successful 151 * @deprecated since 11.1, use 152 * {@link #copyFileMultipart(AmazonS3, ObjectMetadata, String, String, String, String, String, boolean)} 153 * instead 154 */ 155 @Deprecated 156 public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata, 157 String sourceBucket, String sourceKey, String targetBucket, String targetKey, boolean deleteSource) { 158 return copyFileMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, null, 159 deleteSource); 160 } 161 162 /** 163 * Copies a file using multipart upload. 164 * 165 * @param amazonS3 the S3 client 166 * @param objectMetadata the metadata of the object being copied 167 * @param sourceBucket the source bucket 168 * @param sourceKey the source key 169 * @param targetBucket the target bucket 170 * @param targetKey the target key 171 * @param targetSSEAlgorithm the target SSE Algorithm to use, or {@code null} 172 * @param deleteSource whether to delete the source object if the copy is successful 173 * @since 11.1 174 * @deprecated since 11.2, use {@link com.amazonaws.services.s3.transfer.TransferManager#copy} instead 175 */ 176 @Deprecated 177 public static ObjectMetadata copyFileMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata, 178 String sourceBucket, String sourceKey, String targetBucket, String targetKey, String targetSSEAlgorithm, 179 boolean deleteSource) { 180 InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(sourceBucket, 181 targetKey); 182 183 // server-side encryption 184 if (targetSSEAlgorithm != null) { 185 ObjectMetadata newObjectMetadata = new ObjectMetadata(); 186 newObjectMetadata.setSSEAlgorithm(targetSSEAlgorithm); 187 initiateMultipartUploadRequest.setObjectMetadata(newObjectMetadata); 188 } 189 190 InitiateMultipartUploadResult initiateMultipartUploadResult = amazonS3.initiateMultipartUpload( 191 initiateMultipartUploadRequest); 192 193 String uploadId = initiateMultipartUploadResult.getUploadId(); 194 long objectSize = objectMetadata.getContentLength(); 195 List<CopyPartResult> copyResponses = new ArrayList<>(); 196 197 SliceConsumer partCopy = (num, begin, end) -> { 198 CopyPartRequest copyRequest = new CopyPartRequest().withSourceBucketName(sourceBucket) 199 .withSourceKey(sourceKey) 200 .withDestinationBucketName(targetBucket) 201 .withDestinationKey(targetKey) 202 .withFirstByte(begin) 203 .withLastByte(end - 1) 204 .withUploadId(uploadId) 205 .withPartNumber(num + 1); 206 copyResponses.add(amazonS3.copyPart(copyRequest)); 207 }; 208 ConfigurationService configurationService = Framework.getService(ConfigurationService.class); 209 long partSize = configurationService == null ? MULTIPART_COPY_PART_SIZE_DEFAULT 210 : configurationService.getLong(MULTIPART_COPY_PART_SIZE_PROPERTY, MULTIPART_COPY_PART_SIZE_DEFAULT); 211 processSlices(partSize, objectSize, partCopy); 212 213 CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(targetBucket, targetKey, 214 uploadId, responsesToETags(copyResponses)); 215 amazonS3.completeMultipartUpload(completeRequest); 216 if (deleteSource) { 217 amazonS3.deleteObject(sourceBucket, sourceKey); 218 } 219 return amazonS3.getObjectMetadata(targetBucket, targetKey); 220 } 221 222 protected static List<PartETag> responsesToETags(List<CopyPartResult> responses) { 223 return responses.stream().map(response -> new PartETag(response.getPartNumber(), response.getETag())).collect( 224 Collectors.toList()); 225 } 226 227 /** 228 * Copies a file without using multipart upload. 229 * 230 * @param amazonS3 the S3 client 231 * @param objectMetadata the metadata of the object being copied 232 * @param sourceBucket the source bucket 233 * @param sourceKey the source key 234 * @param targetBucket the target bucket 235 * @param targetKey the target key 236 * @param deleteSource whether to delete the source object if the copy is successful 237 * @deprecated since 11.1, use {@link #copyFileNonMultipart} instead 238 */ 239 @Deprecated 240 public static ObjectMetadata copyFile(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket, 241 String sourceKey, String targetBucket, String targetKey, boolean deleteSource) { 242 return copyFileNonMultipart(amazonS3, objectMetadata, sourceBucket, sourceKey, targetBucket, targetKey, null, deleteSource); 243 } 244 245 /** 246 * Copies a file without using multipart upload. 247 * 248 * @param amazonS3 the S3 client 249 * @param objectMetadata the metadata of the object being copied 250 * @param sourceBucket the source bucket 251 * @param sourceKey the source key 252 * @param targetBucket the target bucket 253 * @param targetKey the target key 254 * @param targetSSEAlgorithm the target SSE Algorithm to use, or {@code null} 255 * @param deleteSource whether to delete the source object if the copy is successful 256 * 257 * @since 11.1 258 * @deprecated since 11.2, use {@link com.amazonaws.services.s3.transfer.TransferManager#copy} instead 259 */ 260 @Deprecated 261 public static ObjectMetadata copyFileNonMultipart(AmazonS3 amazonS3, ObjectMetadata objectMetadata, String sourceBucket, 262 String sourceKey, String targetBucket, String targetKey, String targetSSEAlgorithm, boolean deleteSource) { 263 CopyObjectRequest copyObjectRequest = new CopyObjectRequest(sourceBucket, sourceKey, targetBucket, targetKey); 264 // server-side encryption 265 if (targetSSEAlgorithm != null) { 266 ObjectMetadata newObjectMetadata = new ObjectMetadata(); 267 newObjectMetadata.setSSEAlgorithm(targetSSEAlgorithm); 268 copyObjectRequest.setNewObjectMetadata(newObjectMetadata); 269 } 270 amazonS3.copyObject(copyObjectRequest); 271 if (deleteSource) { 272 amazonS3.deleteObject(sourceBucket, sourceKey); 273 } 274 return amazonS3.getObjectMetadata(targetBucket, targetKey); 275 } 276 277 /** 278 * Gets the credentials providers for the given AWS key and secret. 279 * 280 * @param accessKeyId the AWS access key id 281 * @param secretKey the secret key 282 * @param sessionToken the session token (optional) 283 * 284 * @since 10.10 285 */ 286 public static AWSCredentialsProvider getAWSCredentialsProvider(String accessKeyId, String secretKey, 287 String sessionToken) { 288 if (isNotBlank(accessKeyId) && isNotBlank(secretKey)) { 289 // explicit values from service-specific Nuxeo configuration 290 if (isNotBlank(sessionToken)) { 291 return new AWSStaticCredentialsProvider( 292 new BasicSessionCredentials(accessKeyId, secretKey, sessionToken)); 293 } else { 294 return new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, secretKey)); 295 } 296 } 297 return NuxeoAWSCredentialsProvider.getInstance(); 298 } 299 300}