001/*
002 * (C) Copyright 2019 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.blob.s3;
020
021import static org.apache.commons.lang3.StringUtils.isBlank;
022import static org.apache.commons.lang3.StringUtils.isNotBlank;
023import static org.nuxeo.ecm.blob.s3.S3BlobStoreConfiguration.DISABLE_PROXY_PROPERTY;
024
025import java.io.File;
026import java.io.FileInputStream;
027import java.io.IOException;
028import java.security.GeneralSecurityException;
029import java.security.KeyPair;
030import java.security.KeyStore;
031import java.security.PrivateKey;
032import java.security.PublicKey;
033import java.security.cert.Certificate;
034import java.time.Duration;
035import java.util.Date;
036import java.util.Map;
037import java.util.Optional;
038
039import org.apache.logging.log4j.LogManager;
040import org.apache.logging.log4j.Logger;
041import org.nuxeo.common.Environment;
042import org.nuxeo.ecm.blob.CloudBlobStoreConfiguration;
043import org.nuxeo.ecm.core.api.NuxeoException;
044import org.nuxeo.ecm.core.storage.sql.S3Utils;
045import org.nuxeo.runtime.api.Framework;
046import org.nuxeo.runtime.aws.NuxeoAWSRegionProvider;
047import org.nuxeo.runtime.services.config.ConfigurationService;
048
049import com.amazonaws.AmazonServiceException;
050import com.amazonaws.ClientConfiguration;
051import com.amazonaws.auth.AWSCredentialsProvider;
052import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
053import com.amazonaws.services.s3.AmazonS3;
054import com.amazonaws.services.s3.AmazonS3Builder;
055import com.amazonaws.services.s3.AmazonS3ClientBuilder;
056import com.amazonaws.services.s3.AmazonS3EncryptionClientBuilder;
057import com.amazonaws.services.s3.model.CryptoConfiguration;
058import com.amazonaws.services.s3.model.DefaultRetention;
059import com.amazonaws.services.s3.model.EncryptionMaterials;
060import com.amazonaws.services.s3.model.GetObjectLockConfigurationRequest;
061import com.amazonaws.services.s3.model.GetObjectLockConfigurationResult;
062import com.amazonaws.services.s3.model.ObjectLockConfiguration;
063import com.amazonaws.services.s3.model.ObjectLockRetentionMode;
064import com.amazonaws.services.s3.model.ObjectLockRule;
065import com.amazonaws.services.s3.model.StaticEncryptionMaterialsProvider;
066import com.amazonaws.services.s3.transfer.TransferManager;
067import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
068
069/**
070 * Blob storage configuration in S3.
071 *
072 * @since 11.1
073 */
074public class S3BlobStoreConfiguration extends CloudBlobStoreConfiguration {
075
076    private static final Logger log = LogManager.getLogger(S3BlobStoreConfiguration.class);
077
078    public static final String SYSTEM_PROPERTY_PREFIX = "nuxeo.s3storage";
079
080    public static final String BUCKET_NAME_PROPERTY = "bucket";
081
082    public static final String BUCKET_PREFIX_PROPERTY = "bucket_prefix";
083
084    public static final String BUCKET_REGION_PROPERTY = "region";
085
086    public static final String AWS_ID_PROPERTY = "awsid";
087
088    public static final String AWS_SECRET_PROPERTY = "awssecret";
089
090    public static final String AWS_SESSION_TOKEN_PROPERTY = "awstoken";
091
092    /** AWS ClientConfiguration default 50 */
093    public static final String CONNECTION_MAX_PROPERTY = "connection.max";
094
095    /** AWS ClientConfiguration default 3 (with exponential backoff) */
096    public static final String CONNECTION_RETRY_PROPERTY = "connection.retry";
097
098    /** AWS ClientConfiguration default 50*1000 = 50s */
099    public static final String CONNECTION_TIMEOUT_PROPERTY = "connection.timeout";
100
101    /** AWS ClientConfiguration default 50*1000 = 50s */
102    public static final String SOCKET_TIMEOUT_PROPERTY = "socket.timeout";
103
104    public static final String KEYSTORE_FILE_PROPERTY = "crypt.keystore.file";
105
106    public static final String KEYSTORE_PASS_PROPERTY = "crypt.keystore.password";
107
108    public static final String SERVERSIDE_ENCRYPTION_PROPERTY = "crypt.serverside";
109
110    public static final String SERVERSIDE_ENCRYPTION_KMS_KEY_PROPERTY = "crypt.kms.key";
111
112    public static final String PRIVKEY_ALIAS_PROPERTY = "crypt.key.alias";
113
114    public static final String PRIVKEY_PASS_PROPERTY = "crypt.key.password";
115
116    public static final String ENDPOINT_PROPERTY = "endpoint";
117
118    public static final String PATHSTYLEACCESS_PROPERTY = "pathstyleaccess";
119
120    public static final String ACCELERATE_MODE_PROPERTY = "accelerateMode";
121
122    public static final String DIRECTDOWNLOAD_PROPERTY_COMPAT = "downloadfroms3";
123
124    public static final String DIRECTDOWNLOAD_EXPIRE_PROPERTY_COMPAT = "downloadfroms3.expire";
125
126    public static final String METADATA_ADD_USERNAME_PROPERTY = "metadata.addusername";
127
128    /**
129     * Disable automatic abort of old multipart uploads at startup time.
130     *
131     * @since 11.1
132     */
133    public static final String MULTIPART_CLEANUP_DISABLED_PROPERTY = "multipart.cleanup.disabled";
134
135    public static final String DELIMITER = "/";
136
137    /**
138     * The configuration property to define the multipart copy part size.
139     */
140    public static final String MULTIPART_COPY_PART_SIZE_PROPERTY = "nuxeo.s3.multipart.copy.part.size";
141
142    /**
143     * Framework property to disable usage of the proxy environment variables ({@code nuxeo.http.proxy.*}) for the
144     * connection to the S3 endpoint.
145     *
146     * @since 11.1
147     */
148    public static final String DISABLE_PROXY_PROPERTY = "nuxeo.s3.proxy.disabled";
149
150    public static final ObjectLockRetentionMode DEFAULT_RETENTION_MODE = ObjectLockRetentionMode.GOVERNANCE;
151
152    public final CloudFrontConfiguration cloudFront;
153
154    public final AmazonS3 amazonS3;
155
156    public final TransferManager transferManager;
157
158    public final String bucketName;
159
160    public final String bucketPrefix;
161
162    public final boolean useServerSideEncryption;
163
164    public final String serverSideKMSKeyID;
165
166    public final boolean useClientSideEncryption;
167
168    public final boolean metadataAddUsername;
169
170    /**
171     * The retention mode with which the bucket is configured.
172     *
173     * @since 11.4
174     */
175    public final ObjectLockRetentionMode bucketRetentionMode;
176
177    /**
178     * The retention mode to use when setting the retention on an object.
179     */
180    public final ObjectLockRetentionMode retentionMode;
181
182    public S3BlobStoreConfiguration(Map<String, String> properties) throws IOException {
183        super(SYSTEM_PROPERTY_PREFIX, properties);
184        cloudFront = new CloudFrontConfiguration(SYSTEM_PROPERTY_PREFIX, properties);
185
186        bucketName = getBucketName();
187        bucketPrefix = getBucketPrefix();
188
189        String sseprop = getProperty(SERVERSIDE_ENCRYPTION_PROPERTY);
190        if (isNotBlank(sseprop)) {
191            useServerSideEncryption = Boolean.parseBoolean(sseprop);
192            serverSideKMSKeyID = getProperty(SERVERSIDE_ENCRYPTION_KMS_KEY_PROPERTY);
193        } else {
194            useServerSideEncryption = false;
195            serverSideKMSKeyID = null;
196        }
197
198        AWSCredentialsProvider awsCredentialsProvider = getAWSCredentialsProvider();
199        ClientConfiguration clientConfiguration = getClientConfiguration();
200        EncryptionMaterials encryptionMaterials = getEncryptionMaterials();
201        useClientSideEncryption = encryptionMaterials != null;
202
203        AmazonS3Builder<?, ?> s3Builder;
204        if (useClientSideEncryption) {
205            CryptoConfiguration cryptoConfiguration = new CryptoConfiguration();
206            s3Builder = AmazonS3EncryptionClientBuilder.standard()
207                                                       .withCredentials(awsCredentialsProvider)
208                                                       .withClientConfiguration(clientConfiguration)
209                                                       .withCryptoConfiguration(cryptoConfiguration)
210                                                       .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(
211                                                               encryptionMaterials));
212        } else {
213            s3Builder = AmazonS3ClientBuilder.standard()
214                                             .withCredentials(awsCredentialsProvider)
215                                             .withClientConfiguration(clientConfiguration);
216        }
217
218        configurePathStyleAccess(s3Builder);
219        configureRegionOrEndpoint(s3Builder);
220        configureAccelerateMode(s3Builder);
221
222        amazonS3 = getAmazonS3(s3Builder);
223
224        metadataAddUsername = getBooleanProperty(METADATA_ADD_USERNAME_PROPERTY);
225        bucketRetentionMode = computeBucketRetentionMode();
226        retentionMode = bucketRetentionMode == null ? DEFAULT_RETENTION_MODE : bucketRetentionMode;
227
228        transferManager = createTransferManager();
229
230        abortOldUploads();
231    }
232
233    /**
234     * Returns a copy of the S3BlobStoreConfiguration with a different namespace.
235     */
236    public S3BlobStoreConfiguration withNamespace(String ns) throws IOException {
237        return new S3BlobStoreConfiguration(propertiesWithNamespace(ns));
238    }
239
240    public void close() {
241        transferManager.shutdownNow();
242    }
243
244    @Override
245    protected boolean parseDirectDownload() {
246        String directDownloadCompat = getProperty(DIRECTDOWNLOAD_PROPERTY_COMPAT);
247        if (directDownloadCompat != null) {
248            return Boolean.parseBoolean(directDownloadCompat);
249        } else {
250            return super.parseDirectDownload();
251        }
252    }
253
254    @Override
255    protected long parseDirectDownloadExpire() {
256        int directDownloadExpireCompat = getIntProperty(DIRECTDOWNLOAD_EXPIRE_PROPERTY_COMPAT);
257        if (directDownloadExpireCompat >= 0) {
258            return directDownloadExpireCompat;
259        } else {
260            return super.parseDirectDownloadExpire();
261        }
262    }
263
264    protected String getBucketName() {
265        String bn = getProperty(BUCKET_NAME_PROPERTY);
266        if (isBlank(bn)) {
267            throw new NuxeoException("Missing configuration: " + BUCKET_NAME_PROPERTY);
268        }
269        return bn;
270    }
271
272    protected String getBucketPrefix() {
273        // bucket prefix is optional so we don't want to use the fallback mechanism to system properties,
274        // as there may be a globally defined bucket prefix for another blob provider
275        String value = properties.get(BUCKET_PREFIX_PROPERTY);
276        if (isBlank(value)) {
277            value = "";
278        } else if (!value.endsWith(DELIMITER)) {
279            log.debug(String.format("%s %s S3 bucket prefix should end with '/': added automatically.",
280                    BUCKET_PREFIX_PROPERTY, value));
281            value += DELIMITER;
282        }
283        if (isNotBlank(namespace)) {
284            // use namespace as an additional prefix
285            value += namespace;
286            if (!value.endsWith(DELIMITER)) {
287                value += DELIMITER;
288            }
289        }
290        return value;
291    }
292
293    protected AWSCredentialsProvider getAWSCredentialsProvider() {
294        String awsID = getProperty(AWS_ID_PROPERTY);
295        String awsSecret = getProperty(AWS_SECRET_PROPERTY);
296        String awsToken = getProperty(AWS_SESSION_TOKEN_PROPERTY);
297        return S3Utils.getAWSCredentialsProvider(awsID, awsSecret, awsToken);
298    }
299
300    protected ClientConfiguration getClientConfiguration() {
301        boolean proxyDisabled = Framework.isBooleanPropertyTrue(DISABLE_PROXY_PROPERTY);
302        String proxyHost = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_HOST);
303        String proxyPort = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_PORT);
304        String proxyLogin = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_LOGIN);
305        String proxyPassword = Framework.getProperty(Environment.NUXEO_HTTP_PROXY_PASSWORD);
306        int maxConnections = getIntProperty(CONNECTION_MAX_PROPERTY);
307        int maxErrorRetry = getIntProperty(CONNECTION_RETRY_PROPERTY);
308        int connectionTimeout = getIntProperty(CONNECTION_TIMEOUT_PROPERTY);
309        int socketTimeout = getIntProperty(SOCKET_TIMEOUT_PROPERTY);
310        ClientConfiguration clientConfiguration = new ClientConfiguration();
311        if (!proxyDisabled) {
312            if (isNotBlank(proxyHost)) {
313                clientConfiguration.setProxyHost(proxyHost);
314            }
315            if (isNotBlank(proxyPort)) {
316                clientConfiguration.setProxyPort(Integer.parseInt(proxyPort));
317            }
318            if (isNotBlank(proxyLogin)) {
319                clientConfiguration.setProxyUsername(proxyLogin);
320            }
321            if (proxyPassword != null) { // could be blank
322                clientConfiguration.setProxyPassword(proxyPassword);
323            }
324        }
325        if (maxConnections > 0) {
326            clientConfiguration.setMaxConnections(maxConnections);
327        }
328        if (maxErrorRetry >= 0) { // 0 is allowed
329            clientConfiguration.setMaxErrorRetry(maxErrorRetry);
330        }
331        if (connectionTimeout >= 0) { // 0 is allowed
332            clientConfiguration.setConnectionTimeout(connectionTimeout);
333        }
334        if (socketTimeout >= 0) { // 0 is allowed
335            clientConfiguration.setSocketTimeout(socketTimeout);
336        }
337        return clientConfiguration;
338    }
339
340    protected EncryptionMaterials getEncryptionMaterials() {
341        String keystoreFile = getProperty(KEYSTORE_FILE_PROPERTY);
342        String keystorePass = getProperty(KEYSTORE_PASS_PROPERTY);
343        String privkeyAlias = getProperty(PRIVKEY_ALIAS_PROPERTY);
344        String privkeyPass = getProperty(PRIVKEY_PASS_PROPERTY);
345        if (isBlank(keystoreFile)) {
346            return null;
347        }
348        boolean confok = true;
349        if (keystorePass == null) { // could be blank
350            log.error("Keystore password missing");
351            confok = false;
352        }
353        if (isBlank(privkeyAlias)) {
354            log.error("Key alias missing");
355            confok = false;
356        }
357        if (privkeyPass == null) { // could be blank
358            log.error("Key password missing");
359            confok = false;
360        }
361        if (!confok) {
362            throw new NuxeoException("S3 Crypto configuration incomplete");
363        }
364        try {
365            // Open keystore
366            File ksFile = new File(keystoreFile);
367            KeyStore keystore;
368            try (FileInputStream ksStream = new FileInputStream(ksFile)) {
369                keystore = KeyStore.getInstance(KeyStore.getDefaultType());
370                keystore.load(ksStream, keystorePass.toCharArray());
371            }
372            // Get keypair for alias
373            if (!keystore.isKeyEntry(privkeyAlias)) {
374                throw new NuxeoException("Alias " + privkeyAlias + " is missing or not a key alias");
375            }
376            PrivateKey privKey = (PrivateKey) keystore.getKey(privkeyAlias, privkeyPass.toCharArray());
377            Certificate cert = keystore.getCertificate(privkeyAlias);
378            PublicKey pubKey = cert.getPublicKey();
379            KeyPair keypair = new KeyPair(pubKey, privKey);
380            return new EncryptionMaterials(keypair);
381        } catch (IOException | GeneralSecurityException e) {
382            throw new NuxeoException("Could not read keystore: " + keystoreFile + ", alias: " + privkeyAlias, e);
383        }
384    }
385
386    protected void configurePathStyleAccess(AmazonS3Builder<?, ?> s3Builder) {
387        boolean pathStyleAccessEnabled = getBooleanProperty(PATHSTYLEACCESS_PROPERTY);
388        if (pathStyleAccessEnabled) {
389            log.debug("Path-style access enabled");
390            s3Builder.enablePathStyleAccess();
391        }
392    }
393
394    protected void configureRegionOrEndpoint(AmazonS3Builder<?, ?> s3Builder) {
395        String bucketRegion = getProperty(BUCKET_REGION_PROPERTY);
396        if (isBlank(bucketRegion)) {
397            bucketRegion = NuxeoAWSRegionProvider.getInstance().getRegion();
398        }
399        String endpoint = getProperty(ENDPOINT_PROPERTY);
400        if (isNotBlank(endpoint)) {
401            s3Builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, bucketRegion));
402        } else {
403            s3Builder.withRegion(bucketRegion);
404        }
405    }
406
407    protected void configureAccelerateMode(AmazonS3Builder<?, ?> s3Builder) {
408        boolean accelerateModeEnabled = getBooleanProperty(ACCELERATE_MODE_PROPERTY);
409        if (accelerateModeEnabled) {
410            log.debug("Accelerate mode enabled");
411            s3Builder.enableAccelerateMode();
412        }
413    }
414
415    protected AmazonS3 getAmazonS3(AmazonS3Builder<?, ?> s3Builder) {
416        return s3Builder.build();
417    }
418
419    protected TransferManager createTransferManager() {
420        long minimumUploadPartSize = 5L * 1024 * 1024; // AWS SDK default = 5 MB
421        long multipartUploadThreshold = 16L * 1024 * 1024; // AWS SDK default = 16 MB
422        long multipartCopyThreshold = 5L * 1024 * 1024 * 1024; // AWS SDK default = 5 GB
423        long multipartCopyPartSize = 100L * 1024 * 1024; // AWS SDK default = 100 MB
424        ConfigurationService configurationService = Framework.getService(ConfigurationService.class);
425        if (configurationService != null) {
426            multipartCopyPartSize = configurationService.getLong(MULTIPART_COPY_PART_SIZE_PROPERTY,
427                    multipartCopyPartSize);
428        }
429        // when the bucket has Object Lock active, uploads need to provide an MD5
430        boolean alwaysCalculateMultipartMd5 = bucketRetentionMode != null;
431        return TransferManagerBuilder.standard()
432                                     .withS3Client(amazonS3)
433                                     .withMinimumUploadPartSize(Long.valueOf(minimumUploadPartSize))
434                                     .withMultipartUploadThreshold(Long.valueOf(multipartUploadThreshold))
435                                     .withMultipartCopyThreshold(Long.valueOf(multipartCopyThreshold))
436                                     .withMultipartCopyPartSize(Long.valueOf(multipartCopyPartSize))
437                                     .withAlwaysCalculateMultipartMd5(alwaysCalculateMultipartMd5)
438                                     .build();
439    }
440
441    /** @deprecated since 11.4, unused */
442    @Deprecated
443    protected ObjectLockRetentionMode getRetentionMode() {
444        ObjectLockRetentionMode bucketRetentionMode = computeBucketRetentionMode();
445        return bucketRetentionMode == null ? DEFAULT_RETENTION_MODE : bucketRetentionMode;
446    }
447
448    protected ObjectLockRetentionMode computeBucketRetentionMode() {
449        GetObjectLockConfigurationRequest request = new GetObjectLockConfigurationRequest().withBucketName(bucketName);
450        GetObjectLockConfigurationResult result;
451        try {
452            result = amazonS3.getObjectLockConfiguration(request);
453        } catch (AmazonServiceException e) {
454            log.debug("Failed to get ObjectLockConfiguration for bucket: {}", bucketName, e);
455            return null;
456        }
457        return Optional.ofNullable(result.getObjectLockConfiguration())
458                       .map(ObjectLockConfiguration::getRule)
459                       .map(ObjectLockRule::getDefaultRetention)
460                       .map(DefaultRetention::getMode)
461                       .map(ObjectLockRetentionMode::valueOf)
462                       .orElse(null);
463    }
464
465    /**
466     * Aborts uploads that crashed and are older than 1 day.
467     */
468    protected void abortOldUploads() {
469        if (getBooleanProperty(MULTIPART_CLEANUP_DISABLED_PROPERTY)) {
470            log.debug("Cleanup of old multipart uploads is disabled");
471            return;
472        }
473        // Async to avoid issues with transferManager.abortMultipartUploads taking a very long time.
474        // See NXP-28571.
475        new Thread(this::abortOldMultipartUploadsInternal, "Nuxeo-S3-abortOldMultipartUploads-" + bucketName).start();
476    }
477
478    // executed in a separate thread
479    protected void abortOldMultipartUploadsInternal() {
480        long oneDay = Duration.ofDays(1).toMillis();
481        try {
482            log.debug("Starting cleanup of old multipart uploads for bucket: {}", bucketName);
483            Date oneDayAgo = new Date(System.currentTimeMillis() - oneDay);
484            transferManager.abortMultipartUploads(bucketName, oneDayAgo);
485            log.debug("Cleanup done for bucket: {}", bucketName);
486        } catch (AmazonServiceException e) {
487            if (e.getStatusCode() == 400 || e.getStatusCode() == 404) {
488                log.warn("Aborting old uploads is not supported by this provider");
489                return;
490            }
491            throw new NuxeoException("Failed to abort old uploads", e);
492        }
493    }
494
495}