001/* 002 * (C) Copyright 2019 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.blob.s3; 020 021import static org.apache.commons.lang3.StringUtils.isBlank; 022import static org.apache.commons.lang3.StringUtils.isNotBlank; 023import static org.nuxeo.ecm.blob.s3.S3BlobStoreConfiguration.DISABLE_PROXY_PROPERTY; 024 025import java.io.File; 026import java.io.FileInputStream; 027import java.io.IOException; 028import java.security.GeneralSecurityException; 029import java.security.KeyPair; 030import java.security.KeyStore; 031import java.security.PrivateKey; 032import java.security.PublicKey; 033import java.security.cert.Certificate; 034import java.time.Duration; 035import java.util.Date; 036import java.util.Map; 037import java.util.Optional; 038 039import org.apache.logging.log4j.LogManager; 040import org.apache.logging.log4j.Logger; 041import org.nuxeo.common.Environment; 042import org.nuxeo.ecm.blob.CloudBlobStoreConfiguration; 043import org.nuxeo.ecm.core.api.NuxeoException; 044import org.nuxeo.ecm.core.storage.sql.S3Utils; 045import org.nuxeo.runtime.api.Framework; 046import org.nuxeo.runtime.aws.NuxeoAWSRegionProvider; 047import org.nuxeo.runtime.services.config.ConfigurationService; 048 049import com.amazonaws.AmazonServiceException; 050import com.amazonaws.ClientConfiguration; 051import com.amazonaws.auth.AWSCredentialsProvider; 052import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; 053import com.amazonaws.services.s3.AmazonS3; 054import com.amazonaws.services.s3.AmazonS3Builder; 055import com.amazonaws.services.s3.AmazonS3ClientBuilder; 056import com.amazonaws.services.s3.AmazonS3EncryptionClientBuilder; 057import com.amazonaws.services.s3.model.CryptoConfiguration; 058import com.amazonaws.services.s3.model.DefaultRetention; 059import com.amazonaws.services.s3.model.EncryptionMaterials; 060import com.amazonaws.services.s3.model.GetObjectLockConfigurationRequest; 061import com.amazonaws.services.s3.model.GetObjectLockConfigurationResult; 062import com.amazonaws.services.s3.model.ObjectLockConfiguration; 063import com.amazonaws.services.s3.model.ObjectLockRetentionMode; 064import com.amazonaws.services.s3.model.ObjectLockRule; 065import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider; 066import com.amazonaws.services.s3.transfer.TransferManager; 067import com.amazonaws.services.s3.transfer.TransferManagerBuilder; 068 069/** 070 * Blob storage configuration in S3. 071 * 072 * @since 11.1 073 */ 074public class S3BlobStoreConfiguration extends CloudBlobStoreConfiguration { 075 076 private static final Logger log = LogManager.getLogger(S3BlobStoreConfiguration.class); 077 078 public static final String SYSTEM_PROPERTY_PREFIX = "nuxeo.s3storage"; 079 080 public static final String BUCKET_NAME_PROPERTY = "bucket"; 081 082 public static final String BUCKET_PREFIX_PROPERTY = "bucket_prefix"; 083 084 public static final String BUCKET_REGION_PROPERTY = "region"; 085 086 public static final String AWS_ID_PROPERTY = "awsid"; 087 088 public static final String AWS_SECRET_PROPERTY = "awssecret"; 089 090 public static final String AWS_SESSION_TOKEN_PROPERTY = "awstoken"; 091 092 /** AWS ClientConfiguration default 50 */ 093 public static final String CONNECTION_MAX_PROPERTY = "connection.max"; 094 095 /** AWS ClientConfiguration default 3 (with exponential backoff) */ 096 public static final String CONNECTION_RETRY_PROPERTY = "connection.retry"; 097 098 /** AWS ClientConfiguration default 50*1000 = 50s */ 099 public static final String CONNECTION_TIMEOUT_PROPERTY = "connection.timeout"; 100 101 /** AWS ClientConfiguration default 50*1000 = 50s */ 102 public static final String SOCKET_TIMEOUT_PROPERTY = "socket.timeout"; 103 104 public static final String KEYSTORE_FILE_PROPERTY = "crypt.keystore.file"; 105 106 public static final String KEYSTORE_PASS_PROPERTY = "crypt.keystore.password"; 107 108 public static final String SERVERSIDE_ENCRYPTION_PROPERTY = "crypt.serverside"; 109 110 public static final String SERVERSIDE_ENCRYPTION_KMS_KEY_PROPERTY = "crypt.kms.key"; 111 112 public static final String PRIVKEY_ALIAS_PROPERTY = "crypt.key.alias"; 113 114 public static final String PRIVKEY_PASS_PROPERTY = "crypt.key.password"; 115 116 public static final String ENDPOINT_PROPERTY = "endpoint"; 117 118 public static final String PATHSTYLEACCESS_PROPERTY = "pathstyleaccess"; 119 120 public static final String ACCELERATE_MODE_PROPERTY = "accelerateMode"; 121 122 public static final String DIRECTDOWNLOAD_PROPERTY_COMPAT = "downloadfroms3"; 123 124 public static final String DIRECTDOWNLOAD_EXPIRE_PROPERTY_COMPAT = "downloadfroms3.expire"; 125 126 public static final String METADATA_ADD_USERNAME_PROPERTY = "metadata.addusername"; 127 128 /** 129 * Disable automatic abort of old multipart uploads at startup time. 130 * 131 * @since 11.1 132 */ 133 public static final String MULTIPART_CLEANUP_DISABLED_PROPERTY = "multipart.cleanup.disabled"; 134 135 public static final String DELIMITER = "/"; 136 137 /** 138 * The configuration property to define the multipart copy part size. 139 */ 140 public static final String MULTIPART_COPY_PART_SIZE_PROPERTY = "nuxeo.s3.multipart.copy.part.size"; 141 142 /** 143 * Framework property to disable usage of the proxy environment variables ({@code nuxeo.http.proxy.*}) for the 144 * connection to the S3 endpoint. 145 * 146 * @since 11.1 147 */ 148 public static final String DISABLE_PROXY_PROPERTY = "nuxeo.s3.proxy.disabled"; 149 150 public static final ObjectLockRetentionMode DEFAULT_RETENTION_MODE = ObjectLockRetentionMode.GOVERNANCE; 151 152 public final CloudFrontConfiguration cloudFront; 153 154 public final AmazonS3 amazonS3; 155 156 public final TransferManager transferManager; 157 158 public final String bucketName; 159 160 public final String bucketPrefix; 161 162 public final boolean useServerSideEncryption; 163 164 public final String serverSideKMSKeyID; 165 166 public final boolean useClientSideEncryption; 167 168 public final boolean metadataAddUsername; 169 170 /** 171 * The retention mode with which the bucket is configured. 172 * 173 * @since 11.4 174 */ 175 public final ObjectLockRetentionMode bucketRetentionMode; 176 177 /** 178 * The retention mode to use when setting the retention on an object. 179 */ 180 public final ObjectLockRetentionMode retentionMode; 181 182 public S3BlobStoreConfiguration(Map<String, String> properties) throws IOException { 183 super(SYSTEM_PROPERTY_PREFIX, properties); 184 cloudFront = new CloudFrontConfiguration(SYSTEM_PROPERTY_PREFIX, properties); 185 186 bucketName = getBucketName(); 187 bucketPrefix = getBucketPrefix(); 188 189 String sseprop = getProperty(SERVERSIDE_ENCRYPTION_PROPERTY); 190 if (isNotBlank(sseprop)) { 191 useServerSideEncryption = Boolean.parseBoolean(sseprop); 192 serverSideKMSKeyID = getProperty(SERVERSIDE_ENCRYPTION_KMS_KEY_PROPERTY); 193 } else { 194 useServerSideEncryption = false; 195 serverSideKMSKeyID = null; 196 } 197 198 AWSCredentialsProvider awsCredentialsProvider = getAWSCredentialsProvider(); 199 ClientConfiguration clientConfiguration = getClientConfiguration(); 200 EncryptionMaterials encryptionMaterials = getEncryptionMaterials(); 201 useClientSideEncryption = encryptionMaterials != null; 202 203 AmazonS3Builder<?, ?> s3Builder; 204 if (useClientSideEncryption) { 205 CryptoConfiguration cryptoConfiguration = new CryptoConfiguration(); 206 s3Builder = AmazonS3EncryptionClientBuilder.standard() 207 .withCredentials(awsCredentialsProvider) 208 .withClientConfiguration(clientConfiguration) 209 .withCryptoConfiguration(cryptoConfiguration) 210 .withEncryptionMaterials(new StaticEncryptionMaterialsProvider( 211 encryptionMaterials)); 212 } else { 213 s3Builder = AmazonS3ClientBuilder.standard() 214 .withCredentials(awsCredentialsProvider) 215 .withClientConfiguration(clientConfiguration); 216 } 217 218 configurePathStyleAccess(s3Builder); 219 configureRegionOrEndpoint(s3Builder); 220 configureAccelerateMode(s3Builder); 221 222 amazonS3 = getAmazonS3(s3Builder); 223 224 metadataAddUsername = getBooleanProperty(METADATA_ADD_USERNAME_PROPERTY); 225 bucketRetentionMode = computeBucketRetentionMode(); 226 retentionMode = bucketRetentionMode == null ? DEFAULT_RETENTION_MODE : bucketRetentionMode; 227 228 transferManager = createTransferManager(); 229 230 abortOldUploads(); 231 } 232 233 /** 234 * Returns a copy of the S3BlobStoreConfiguration with a different namespace. 235 */ 236 public S3BlobStoreConfiguration withNamespace(String ns) throws IOException { 237 return new S3BlobStoreConfiguration(propertiesWithNamespace(ns)); 238 } 239 240 public void close() { 241 transferManager.shutdownNow(); 242 } 243 244 @Override 245 protected boolean parseDirectDownload() { 246 String directDownloadCompat = getProperty(DIRECTDOWNLOAD_PROPERTY_COMPAT); 247 if (directDownloadCompat != null) { 248 return Boolean.parseBoolean(directDownloadCompat); 249 } else { 250 return super.parseDirectDownload(); 251 } 252 } 253 254 @Override 255 protected long parseDirectDownloadExpire() { 256 int directDownloadExpireCompat = getIntProperty(DIRECTDOWNLOAD_EXPIRE_PROPERTY_COMPAT); 257 if (directDownloadExpireCompat >= 0) { 258 return directDownloadExpireCompat; 259 } else { 260 return super.parseDirectDownloadExpire(); 261 } 262 } 263 264 protected String getBucketName() { 265 String bn = getProperty(BUCKET_NAME_PROPERTY); 266 if (isBlank(bn)) { 267 throw new NuxeoException("Missing configuration: " + BUCKET_NAME_PROPERTY); 268 } 269 return bn; 270 } 271 272 protected String getBucketPrefix() { 273 // bucket prefix is optional so we don't want to use the fallback mechanism to system properties, 274 // as there may be a globally defined bucket prefix for another blob provider 275 String value = properties.get(BUCKET_PREFIX_PROPERTY); 276 if (isBlank(value)) { 277 value = ""; 278 } else if (!value.endsWith(DELIMITER)) { 279 log.debug(String.format("%s %s S3 bucket prefix should end with '/': added automatically.", 280 BUCKET_PREFIX_PROPERTY, value)); 281 value += DELIMITER; 282 } 283 if (isNotBlank(namespace)) { 284 // use namespace as an additional prefix 285 value += namespace; 286 if (!value.endsWith(DELIMITER)) { 287 value += DELIMITER; 288 } 289 } 290 return value; 291 } 292 293 protected AWSCredentialsProvider getAWSCredentialsProvider() { 294 String awsID = getProperty(AWS_ID_PROPERTY); 295 String awsSecret = getProperty(AWS_SECRET_PROPERTY); 296 String awsToken = getProperty(AWS_SESSION_TOKEN_PROPERTY); 297 return S3Utils.getAWSCredentialsProvider(awsID, awsSecret, awsToken); 298 } 299 300 protected ClientConfiguration getClientConfiguration() { 301 boolean proxyDisabled = Framework.isBooleanPropertyTrue(DISABLE_PROXY_PROPERTY); 302 String proxyHost = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_HOST); 303 String proxyPort = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_PORT); 304 String proxyLogin = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_LOGIN); 305 String proxyPassword = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_PASSWORD); 306 int maxConnections = getIntProperty(CONNECTION_MAX_PROPERTY); 307 int maxErrorRetry = getIntProperty(CONNECTION_RETRY_PROPERTY); 308 int connectionTimeout = getIntProperty(CONNECTION_TIMEOUT_PROPERTY); 309 int socketTimeout = getIntProperty(SOCKET_TIMEOUT_PROPERTY); 310 ClientConfiguration clientConfiguration = new ClientConfiguration(); 311 if (!proxyDisabled) { 312 if (isNotBlank(proxyHost)) { 313 clientConfiguration.setProxyHost(proxyHost); 314 } 315 if (isNotBlank(proxyPort)) { 316 clientConfiguration.setProxyPort(Integer.parseInt(proxyPort)); 317 } 318 if (isNotBlank(proxyLogin)) { 319 clientConfiguration.setProxyUsername(proxyLogin); 320 } 321 if (proxyPassword != null) { // could be blank 322 clientConfiguration.setProxyPassword(proxyPassword); 323 } 324 } 325 if (maxConnections > 0) { 326 clientConfiguration.setMaxConnections(maxConnections); 327 } 328 if (maxErrorRetry >= 0) { // 0 is allowed 329 clientConfiguration.setMaxErrorRetry(maxErrorRetry); 330 } 331 if (connectionTimeout >= 0) { // 0 is allowed 332 clientConfiguration.setConnectionTimeout(connectionTimeout); 333 } 334 if (socketTimeout >= 0) { // 0 is allowed 335 clientConfiguration.setSocketTimeout(socketTimeout); 336 } 337 return clientConfiguration; 338 } 339 340 protected EncryptionMaterials getEncryptionMaterials() { 341 String keystoreFile = getProperty(KEYSTORE_FILE_PROPERTY); 342 String keystorePass = getProperty(KEYSTORE_PASS_PROPERTY); 343 String privkeyAlias = getProperty(PRIVKEY_ALIAS_PROPERTY); 344 String privkeyPass = getProperty(PRIVKEY_PASS_PROPERTY); 345 if (isBlank(keystoreFile)) { 346 return null; 347 } 348 boolean confok = true; 349 if (keystorePass == null) { // could be blank 350 log.error("Keystore password missing"); 351 confok = false; 352 } 353 if (isBlank(privkeyAlias)) { 354 log.error("Key alias missing"); 355 confok = false; 356 } 357 if (privkeyPass == null) { // could be blank 358 log.error("Key password missing"); 359 confok = false; 360 } 361 if (!confok) { 362 throw new NuxeoException("S3 Crypto configuration incomplete"); 363 } 364 try { 365 // Open keystore 366 File ksFile = new File(keystoreFile); 367 KeyStore keystore; 368 try (FileInputStream ksStream = new FileInputStream(ksFile)) { 369 keystore = KeyStore.getInstance(KeyStore.getDefaultType()); 370 keystore.load(ksStream, keystorePass.toCharArray()); 371 } 372 // Get keypair for alias 373 if (!keystore.isKeyEntry(privkeyAlias)) { 374 throw new NuxeoException("Alias " + privkeyAlias + " is missing or not a key alias"); 375 } 376 PrivateKey privKey = (PrivateKey) keystore.getKey(privkeyAlias, privkeyPass.toCharArray()); 377 Certificate cert = keystore.getCertificate(privkeyAlias); 378 PublicKey pubKey = cert.getPublicKey(); 379 KeyPair keypair = new KeyPair(pubKey, privKey); 380 return new EncryptionMaterials(keypair); 381 } catch (IOException | GeneralSecurityException e) { 382 throw new NuxeoException("Could not read keystore: " + keystoreFile + ", alias: " + privkeyAlias, e); 383 } 384 } 385 386 protected void configurePathStyleAccess(AmazonS3Builder<?, ?> s3Builder) { 387 boolean pathStyleAccessEnabled = getBooleanProperty(PATHSTYLEACCESS_PROPERTY); 388 if (pathStyleAccessEnabled) { 389 log.debug("Path-style access enabled"); 390 s3Builder.enablePathStyleAccess(); 391 } 392 } 393 394 protected void configureRegionOrEndpoint(AmazonS3Builder<?, ?> s3Builder) { 395 String bucketRegion = getProperty(BUCKET_REGION_PROPERTY); 396 if (isBlank(bucketRegion)) { 397 bucketRegion = NuxeoAWSRegionProvider.getInstance().getRegion(); 398 } 399 String endpoint = getProperty(ENDPOINT_PROPERTY); 400 if (isNotBlank(endpoint)) { 401 s3Builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, bucketRegion)); 402 } else { 403 s3Builder.withRegion(bucketRegion); 404 } 405 } 406 407 protected void configureAccelerateMode(AmazonS3Builder<?, ?> s3Builder) { 408 boolean accelerateModeEnabled = getBooleanProperty(ACCELERATE_MODE_PROPERTY); 409 if (accelerateModeEnabled) { 410 log.debug("Accelerate mode enabled"); 411 s3Builder.enableAccelerateMode(); 412 } 413 } 414 415 protected AmazonS3 getAmazonS3(AmazonS3Builder<?, ?> s3Builder) { 416 return s3Builder.build(); 417 } 418 419 protected TransferManager createTransferManager() { 420 long minimumUploadPartSize = 5L * 1024 * 1024; // AWS SDK default = 5 MB 421 long multipartUploadThreshold = 16L * 1024 * 1024; // AWS SDK default = 16 MB 422 long multipartCopyThreshold = 5L * 1024 * 1024 * 1024; // AWS SDK default = 5 GB 423 long multipartCopyPartSize = 100L * 1024 * 1024; // AWS SDK default = 100 MB 424 ConfigurationService configurationService = Framework.getService(ConfigurationService.class); 425 if (configurationService != null) { 426 multipartCopyPartSize = configurationService.getLong(MULTIPART_COPY_PART_SIZE_PROPERTY, 427 multipartCopyPartSize); 428 } 429 // when the bucket has Object Lock active, uploads need to provide an MD5 430 boolean alwaysCalculateMultipartMd5 = bucketRetentionMode != null; 431 return TransferManagerBuilder.standard() 432 .withS3Client(amazonS3) 433 .withMinimumUploadPartSize(Long.valueOf(minimumUploadPartSize)) 434 .withMultipartUploadThreshold(Long.valueOf(multipartUploadThreshold)) 435 .withMultipartCopyThreshold(Long.valueOf(multipartCopyThreshold)) 436 .withMultipartCopyPartSize(Long.valueOf(multipartCopyPartSize)) 437 .withAlwaysCalculateMultipartMd5(alwaysCalculateMultipartMd5) 438 .build(); 439 } 440 441 /** @deprecated since 11.4, unused */ 442 @Deprecated 443 protected ObjectLockRetentionMode getRetentionMode() { 444 ObjectLockRetentionMode bucketRetentionMode = computeBucketRetentionMode(); 445 return bucketRetentionMode == null ? DEFAULT_RETENTION_MODE : bucketRetentionMode; 446 } 447 448 protected ObjectLockRetentionMode computeBucketRetentionMode() { 449 GetObjectLockConfigurationRequest request = new GetObjectLockConfigurationRequest().withBucketName(bucketName); 450 GetObjectLockConfigurationResult result; 451 try { 452 result = amazonS3.getObjectLockConfiguration(request); 453 } catch (AmazonServiceException e) { 454 log.debug("Failed to get ObjectLockConfiguration for bucket: {}", bucketName, e); 455 return null; 456 } 457 return Optional.ofNullable(result.getObjectLockConfiguration()) 458 .map(ObjectLockConfiguration::getRule) 459 .map(ObjectLockRule::getDefaultRetention) 460 .map(DefaultRetention::getMode) 461 .map(ObjectLockRetentionMode::valueOf) 462 .orElse(null); 463 } 464 465 /** 466 * Aborts uploads that crashed and are older than 1 day. 467 */ 468 protected void abortOldUploads() { 469 if (getBooleanProperty(MULTIPART_CLEANUP_DISABLED_PROPERTY)) { 470 log.debug("Cleanup of old multipart uploads is disabled"); 471 return; 472 } 473 // Async to avoid issues with transferManager.abortMultipartUploads taking a very long time. 474 // See NXP-28571. 475 new Thread(this::abortOldMultipartUploadsInternal, "Nuxeo-S3-abortOldMultipartUploads-" + bucketName).start(); 476 } 477 478 // executed in a separate thread 479 protected void abortOldMultipartUploadsInternal() { 480 long oneDay = Duration.ofDays(1).toMillis(); 481 try { 482 log.debug("Starting cleanup of old multipart uploads for bucket: {}", bucketName); 483 Date oneDayAgo = new Date(System.currentTimeMillis() - oneDay); 484 transferManager.abortMultipartUploads(bucketName, oneDayAgo); 485 log.debug("Cleanup done for bucket: {}", bucketName); 486 } catch (AmazonServiceException e) { 487 if (e.getStatusCode() == 400 || e.getStatusCode() == 404) { 488 log.warn("Aborting old uploads is not supported by this provider"); 489 return; 490 } 491 throw new NuxeoException("Failed to abort old uploads", e); 492 } 493 } 494 495}