001/*
002 * (C) Copyright 2019 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.blob.s3;
020
021import static java.lang.Boolean.TRUE;
022import static org.nuxeo.ecm.core.blob.KeyStrategy.VER_SEP;
023
024import java.io.IOException;
025import java.net.URI;
026import java.net.URISyntaxException;
027import java.net.URL;
028import java.time.Instant;
029import java.util.Date;
030import java.util.Map;
031
032import javax.servlet.http.HttpServletRequest;
033
034import org.apache.http.NameValuePair;
035import org.apache.http.client.utils.URIBuilder;
036import org.apache.logging.log4j.LogManager;
037import org.apache.logging.log4j.Logger;
038import org.nuxeo.common.utils.RFC2231;
039import org.nuxeo.ecm.core.api.Blob;
040import org.nuxeo.ecm.core.blob.BlobManager;
041import org.nuxeo.ecm.core.blob.BlobStatus;
042import org.nuxeo.ecm.core.blob.BlobStore;
043import org.nuxeo.ecm.core.blob.BlobStoreBlobProvider;
044import org.nuxeo.ecm.core.blob.CachingBlobStore;
045import org.nuxeo.ecm.core.blob.KeyStrategy;
046import org.nuxeo.ecm.core.blob.ManagedBlob;
047import org.nuxeo.ecm.core.blob.TransactionalBlobStore;
048import org.nuxeo.ecm.core.io.download.DownloadHelper;
049
050import com.amazonaws.AmazonServiceException;
051import com.amazonaws.HttpMethod;
052import com.amazonaws.services.cloudfront.CloudFrontUrlSigner;
053import com.amazonaws.services.cloudfront.util.SignerUtils.Protocol;
054import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
055import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
056import com.amazonaws.services.s3.model.ObjectMetadata;
057import com.amazonaws.services.s3.model.StorageClass;
058import com.amazonaws.services.s3.transfer.TransferManager;
059
060/**
061 * Blob provider that stores files in S3.
062 * <p>
063 * This blob provider supports transactional record mode.
064 *
065 * @since 11.1
066 */
067public class S3BlobProvider extends BlobStoreBlobProvider implements S3ManagedTransfer {
068
069    private static final Logger log = LogManager.getLogger(S3BlobProvider.class);
070
071    // public for tests
072    public S3BlobStoreConfiguration config;
073
074    @Override
075    protected BlobStore getBlobStore(String blobProviderId, Map<String, String> properties) throws IOException {
076        config = getConfiguration(properties);
077        log.info("Registering S3 blob provider '" + blobProviderId);
078        KeyStrategy keyStrategy = getKeyStrategy();
079
080        // main S3 blob store wrapped in a caching store
081        BlobStore store = new S3BlobStore("S3", config, keyStrategy);
082        boolean caching = !config.getBooleanProperty("test-nocaching"); // for tests
083        if (caching) {
084            store = new CachingBlobStore("Cache", store, config.cachingConfiguration);
085        }
086
087        // maybe wrap into a transactional store
088        if (isTransactional()) {
089            BlobStore transientStore;
090            if (store.hasVersioning()) {
091                // if versioning is used, we don't need a separate transient store for transactions
092                transientStore = store;
093            } else {
094                // transient store is another S3 blob store wrapped in a caching store
095                S3BlobStoreConfiguration transientConfig = config.withNamespace("tx");
096                transientStore = new S3BlobStore("S3_tmp", transientConfig, keyStrategy);
097                if (caching) {
098                    transientStore = new CachingBlobStore("Cache_tmp", transientStore, config.cachingConfiguration);
099                }
100            }
101            // transactional store
102            store = new TransactionalBlobStore(store, transientStore);
103        }
104        return store;
105    }
106
107    protected S3BlobStoreConfiguration getConfiguration(Map<String, String> properties) throws IOException {
108        return new S3BlobStoreConfiguration(properties);
109    }
110
111    @Override
112    public TransferManager getTransferManager() {
113        return config.transferManager;
114    }
115
116    @Override
117    public void close() {
118        config.close();
119    }
120
121    @Override
122    protected String getDigestAlgorithm() {
123        return config.digestConfiguration.digestAlgorithm;
124    }
125
126    /** Checks if the bucket exists (used in health check probes). */
127    public boolean canAccessBucket() {
128        return config.amazonS3.doesBucketExistV2(config.bucketName);
129    }
130
131    @Override
132    public URI getURI(ManagedBlob blob, BlobManager.UsageHint hint, HttpServletRequest servletRequest)
133            throws IOException {
134        if (hint != BlobManager.UsageHint.DOWNLOAD || !config.directDownload) {
135            return null;
136        }
137        String bucketKey = config.bucketPrefix + stripBlobKeyPrefix(blob.getKey());
138        Date expiration = new Date(System.currentTimeMillis() + config.directDownloadExpire * 1000);
139        try {
140            if (config.cloudFront.enabled) {
141                return getURICloudFront(bucketKey, blob, expiration);
142            } else {
143                return getURIS3(bucketKey, blob, expiration);
144            }
145        } catch (URISyntaxException e) {
146            throw new IOException(e);
147        }
148    }
149
150    protected URI getURICloudFront(String bucketKey, ManagedBlob blob, Date expiration)
151            throws URISyntaxException {
152        String[] parts = bucketKey.split(String.valueOf(VER_SEP));
153        bucketKey = parts[0];
154        CloudFrontConfiguration cloudFront = config.cloudFront;
155        Protocol protocol = cloudFront.protocol;
156        String baseURI = protocol == Protocol.http || protocol == Protocol.https
157                ? protocol + "://" + cloudFront.distributionDomain + "/" + bucketKey
158                : bucketKey;
159        URIBuilder uriBuilder = new URIBuilder(baseURI);
160        if (parts.length > 1) {
161            uriBuilder.addParameter("versionId", parts[1]);
162        }
163        uriBuilder.addParameter("response-content-type", getContentTypeHeader(blob));
164        uriBuilder.addParameter("response-content-disposition", getContentDispositionHeader(blob));
165        if (cloudFront.fixEncoding) {
166            // remove spaces in the values, as they're not encoded correctly due to a bug somewhere
167            // this happens in particular for the Content-Disposition header
168            for (NameValuePair p : uriBuilder.getQueryParams()) {
169                String value = p.getValue();
170                if (value != null && value.contains(" ")) {
171                    uriBuilder.setParameter(p.getName(), value.replace(" ", ""));
172                }
173            }
174        }
175        URI uri = uriBuilder.build();
176        if (cloudFront.privateKey == null) {
177            return uri;
178        } else {
179            String signedURL = CloudFrontUrlSigner.getSignedURLWithCannedPolicy(uri.toString(), cloudFront.keyPairId,
180                    cloudFront.privateKey, expiration);
181            return new URI(signedURL);
182        }
183    }
184
185    protected URI getURIS3(String bucketKey, ManagedBlob blob, Date expiration) throws URISyntaxException {
186        // split version id if part of file key
187        String[] parts = bucketKey.split(String.valueOf(VER_SEP));
188        GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(config.bucketName, parts[0],
189                HttpMethod.GET);
190        if (parts.length > 1) {
191            request.setVersionId(parts[1]);
192        }
193        request.addRequestParameter("response-content-type", getContentTypeHeader(blob));
194        request.addRequestParameter("response-content-disposition", getContentDispositionHeader(blob));
195        request.setExpiration(expiration);
196        URL url = config.amazonS3.generatePresignedUrl(request);
197        return url.toURI();
198    }
199
200    protected String getContentTypeHeader(Blob blob) {
201        return DownloadHelper.getContentTypeHeader(blob);
202    }
203
204    protected String getContentDispositionHeader(Blob blob) {
205        return RFC2231.encodeContentDisposition(blob.getFilename(), false, null);
206    }
207
208    @Override
209    public BlobStatus getStatus(ManagedBlob blob) throws IOException {
210        String key = stripBlobKeyPrefix(blob.getKey());
211        String objectKey;
212        String versionId;
213        int seppos = key.indexOf(VER_SEP);
214        if (seppos < 0) {
215            objectKey = key;
216            versionId = null;
217        } else {
218            objectKey = key.substring(0, seppos);
219            versionId = key.substring(seppos + 1);
220        }
221        String bucketKey = config.bucketPrefix + objectKey;
222        GetObjectMetadataRequest request = new GetObjectMetadataRequest(config.bucketName, bucketKey, versionId);
223        ObjectMetadata metadata;
224        try {
225            metadata = config.amazonS3.getObjectMetadata(request);
226        } catch (AmazonServiceException e) {
227            if (S3BlobStore.isMissingKey(e)) {
228                // don't crash for a missing blob, even though it means the storage is corrupted
229                log.error("Failed to get information on blob: {}", key, e);
230                return new BlobStatus().withDownloadable(false);
231            }
232            throw new IOException(e);
233        }
234        // storage class is null for STANDARD
235        String storageClass = metadata.getStorageClass();
236        if (StorageClass.Standard.toString().equals(storageClass)) {
237            storageClass = null;
238        }
239        // x-amz-restore absent
240        // x-amz-restore: ongoing-request="true"
241        // x-amz-restore: ongoing-request="false", expiry-date="Fri, 23 Dec 2012 00:00:00 GMT"
242        Boolean ongoingRestore = metadata.getOngoingRestore();
243        boolean downloadable = !TRUE.equals(ongoingRestore);
244        Date date = metadata.getRestoreExpirationTime();
245        Instant downloadableUntil = date == null ? null : date.toInstant();
246        return new BlobStatus().withStorageClass(storageClass)
247                               .withDownloadable(downloadable)
248                               .withDownloadableUntil(downloadableUntil);
249    }
250
251}