001/* 002 * (C) Copyright 2019 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.blob.s3; 020 021import static java.lang.Boolean.TRUE; 022import static org.nuxeo.ecm.core.blob.KeyStrategy.VER_SEP; 023 024import java.io.IOException; 025import java.net.URI; 026import java.net.URISyntaxException; 027import java.net.URL; 028import java.time.Instant; 029import java.util.Date; 030import java.util.Map; 031 032import javax.servlet.http.HttpServletRequest; 033 034import org.apache.http.NameValuePair; 035import org.apache.http.client.utils.URIBuilder; 036import org.apache.logging.log4j.LogManager; 037import org.apache.logging.log4j.Logger; 038import org.nuxeo.common.utils.RFC2231; 039import org.nuxeo.ecm.core.api.Blob; 040import org.nuxeo.ecm.core.blob.BlobManager; 041import org.nuxeo.ecm.core.blob.BlobStatus; 042import org.nuxeo.ecm.core.blob.BlobStore; 043import org.nuxeo.ecm.core.blob.BlobStoreBlobProvider; 044import org.nuxeo.ecm.core.blob.CachingBlobStore; 045import org.nuxeo.ecm.core.blob.KeyStrategy; 046import org.nuxeo.ecm.core.blob.ManagedBlob; 047import org.nuxeo.ecm.core.blob.TransactionalBlobStore; 048import org.nuxeo.ecm.core.io.download.DownloadHelper; 049 050import com.amazonaws.AmazonServiceException; 051import com.amazonaws.HttpMethod; 052import com.amazonaws.services.cloudfront.CloudFrontUrlSigner; 053import com.amazonaws.services.cloudfront.util.SignerUtils.Protocol; 054import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; 055import com.amazonaws.services.s3.model.GetObjectMetadataRequest; 056import com.amazonaws.services.s3.model.ObjectMetadata; 057import com.amazonaws.services.s3.model.StorageClass; 058import com.amazonaws.services.s3.transfer.TransferManager; 059 060/** 061 * Blob provider that stores files in S3. 062 * <p> 063 * This blob provider supports transactional record mode. 064 * 065 * @since 11.1 066 */ 067public class S3BlobProvider extends BlobStoreBlobProvider implements S3ManagedTransfer { 068 069 private static final Logger log = LogManager.getLogger(S3BlobProvider.class); 070 071 // public for tests 072 public S3BlobStoreConfiguration config; 073 074 @Override 075 protected BlobStore getBlobStore(String blobProviderId, Map<String, String> properties) throws IOException { 076 config = getConfiguration(properties); 077 log.info("Registering S3 blob provider '" + blobProviderId); 078 KeyStrategy keyStrategy = getKeyStrategy(); 079 080 // main S3 blob store wrapped in a caching store 081 BlobStore store = new S3BlobStore("S3", config, keyStrategy); 082 boolean caching = !config.getBooleanProperty("test-nocaching"); // for tests 083 if (caching) { 084 store = new CachingBlobStore("Cache", store, config.cachingConfiguration); 085 } 086 087 // maybe wrap into a transactional store 088 if (isTransactional()) { 089 BlobStore transientStore; 090 if (store.hasVersioning()) { 091 // if versioning is used, we don't need a separate transient store for transactions 092 transientStore = store; 093 } else { 094 // transient store is another S3 blob store wrapped in a caching store 095 S3BlobStoreConfiguration transientConfig = config.withNamespace("tx"); 096 transientStore = new S3BlobStore("S3_tmp", transientConfig, keyStrategy); 097 if (caching) { 098 transientStore = new CachingBlobStore("Cache_tmp", transientStore, config.cachingConfiguration); 099 } 100 } 101 // transactional store 102 store = new TransactionalBlobStore(store, transientStore); 103 } 104 return store; 105 } 106 107 protected S3BlobStoreConfiguration getConfiguration(Map<String, String> properties) throws IOException { 108 return new S3BlobStoreConfiguration(properties); 109 } 110 111 @Override 112 public TransferManager getTransferManager() { 113 return config.transferManager; 114 } 115 116 @Override 117 public void close() { 118 config.close(); 119 } 120 121 @Override 122 protected String getDigestAlgorithm() { 123 return config.digestConfiguration.digestAlgorithm; 124 } 125 126 /** Checks if the bucket exists (used in health check probes). */ 127 public boolean canAccessBucket() { 128 return config.amazonS3.doesBucketExistV2(config.bucketName); 129 } 130 131 @Override 132 public URI getURI(ManagedBlob blob, BlobManager.UsageHint hint, HttpServletRequest servletRequest) 133 throws IOException { 134 if (hint != BlobManager.UsageHint.DOWNLOAD || !config.directDownload) { 135 return null; 136 } 137 String bucketKey = config.bucketPrefix + stripBlobKeyPrefix(blob.getKey()); 138 Date expiration = new Date(System.currentTimeMillis() + config.directDownloadExpire * 1000); 139 try { 140 if (config.cloudFront.enabled) { 141 return getURICloudFront(bucketKey, blob, expiration); 142 } else { 143 return getURIS3(bucketKey, blob, expiration); 144 } 145 } catch (URISyntaxException e) { 146 throw new IOException(e); 147 } 148 } 149 150 protected URI getURICloudFront(String bucketKey, ManagedBlob blob, Date expiration) 151 throws URISyntaxException { 152 String[] parts = bucketKey.split(String.valueOf(VER_SEP)); 153 bucketKey = parts[0]; 154 CloudFrontConfiguration cloudFront = config.cloudFront; 155 Protocol protocol = cloudFront.protocol; 156 String baseURI = protocol == Protocol.http || protocol == Protocol.https 157 ? protocol + "://" + cloudFront.distributionDomain + "/" + bucketKey 158 : bucketKey; 159 URIBuilder uriBuilder = new URIBuilder(baseURI); 160 if (parts.length > 1) { 161 uriBuilder.addParameter("versionId", parts[1]); 162 } 163 uriBuilder.addParameter("response-content-type", getContentTypeHeader(blob)); 164 uriBuilder.addParameter("response-content-disposition", getContentDispositionHeader(blob)); 165 if (cloudFront.fixEncoding) { 166 // remove spaces in the values, as they're not encoded correctly due to a bug somewhere 167 // this happens in particular for the Content-Disposition header 168 for (NameValuePair p : uriBuilder.getQueryParams()) { 169 String value = p.getValue(); 170 if (value != null && value.contains(" ")) { 171 uriBuilder.setParameter(p.getName(), value.replace(" ", "")); 172 } 173 } 174 } 175 URI uri = uriBuilder.build(); 176 if (cloudFront.privateKey == null) { 177 return uri; 178 } else { 179 String signedURL = CloudFrontUrlSigner.getSignedURLWithCannedPolicy(uri.toString(), cloudFront.keyPairId, 180 cloudFront.privateKey, expiration); 181 return new URI(signedURL); 182 } 183 } 184 185 protected URI getURIS3(String bucketKey, ManagedBlob blob, Date expiration) throws URISyntaxException { 186 // split version id if part of file key 187 String[] parts = bucketKey.split(String.valueOf(VER_SEP)); 188 GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(config.bucketName, parts[0], 189 HttpMethod.GET); 190 if (parts.length > 1) { 191 request.setVersionId(parts[1]); 192 } 193 request.addRequestParameter("response-content-type", getContentTypeHeader(blob)); 194 request.addRequestParameter("response-content-disposition", getContentDispositionHeader(blob)); 195 request.setExpiration(expiration); 196 URL url = config.amazonS3.generatePresignedUrl(request); 197 return url.toURI(); 198 } 199 200 protected String getContentTypeHeader(Blob blob) { 201 return DownloadHelper.getContentTypeHeader(blob); 202 } 203 204 protected String getContentDispositionHeader(Blob blob) { 205 return RFC2231.encodeContentDisposition(blob.getFilename(), false, null); 206 } 207 208 @Override 209 public BlobStatus getStatus(ManagedBlob blob) throws IOException { 210 String key = stripBlobKeyPrefix(blob.getKey()); 211 String objectKey; 212 String versionId; 213 int seppos = key.indexOf(VER_SEP); 214 if (seppos < 0) { 215 objectKey = key; 216 versionId = null; 217 } else { 218 objectKey = key.substring(0, seppos); 219 versionId = key.substring(seppos + 1); 220 } 221 String bucketKey = config.bucketPrefix + objectKey; 222 GetObjectMetadataRequest request = new GetObjectMetadataRequest(config.bucketName, bucketKey, versionId); 223 ObjectMetadata metadata; 224 try { 225 metadata = config.amazonS3.getObjectMetadata(request); 226 } catch (AmazonServiceException e) { 227 if (S3BlobStore.isMissingKey(e)) { 228 // don't crash for a missing blob, even though it means the storage is corrupted 229 log.error("Failed to get information on blob: {}", key, e); 230 return new BlobStatus().withDownloadable(false); 231 } 232 throw new IOException(e); 233 } 234 // storage class is null for STANDARD 235 String storageClass = metadata.getStorageClass(); 236 if (StorageClass.Standard.toString().equals(storageClass)) { 237 storageClass = null; 238 } 239 // x-amz-restore absent 240 // x-amz-restore: ongoing-request="true" 241 // x-amz-restore: ongoing-request="false", expiry-date="Fri, 23 Dec 2012 00:00:00 GMT" 242 Boolean ongoingRestore = metadata.getOngoingRestore(); 243 boolean downloadable = !TRUE.equals(ongoingRestore); 244 Date date = metadata.getRestoreExpirationTime(); 245 Instant downloadableUntil = date == null ? null : date.toInstant(); 246 return new BlobStatus().withStorageClass(storageClass) 247 .withDownloadable(downloadable) 248 .withDownloadableUntil(downloadableUntil); 249 } 250 251}