001/* 002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Thierry Delprat <tdelprat@nuxeo.com> 018 * Antoine Taillefer <ataillefer@nuxeo.com> 019 */ 020 021package org.nuxeo.ecm.core.transientstore; 022 023import java.io.File; 024import java.io.IOException; 025import java.io.Serializable; 026import java.nio.file.DirectoryStream; 027import java.nio.file.Files; 028import java.nio.file.Path; 029import java.nio.file.Paths; 030import java.util.ArrayList; 031import java.util.HashMap; 032import java.util.List; 033import java.util.Map; 034import java.util.UUID; 035 036import org.apache.commons.codec.binary.Base64; 037import org.apache.commons.io.FileUtils; 038import org.apache.commons.logging.Log; 039import org.apache.commons.logging.LogFactory; 040import org.nuxeo.common.Environment; 041import org.nuxeo.ecm.core.api.Blob; 042import org.nuxeo.ecm.core.api.NuxeoException; 043import org.nuxeo.ecm.core.api.impl.blob.FileBlob; 044import org.nuxeo.ecm.core.transientstore.api.MaximumTransientSpaceExceeded; 045import org.nuxeo.ecm.core.transientstore.api.TransientStore; 046import org.nuxeo.ecm.core.transientstore.api.TransientStoreConfig; 047 048/** 049 * Base class for a {@link TransientStore} implementation. 050 * 051 * @since 7.2 052 */ 053public abstract class AbstractTransientStore implements TransientStore { 054 055 protected static final Log log = LogFactory.getLog(AbstractTransientStore.class); 056 057 protected TransientStoreConfig config; 058 059 protected File cacheDir; 060 061 @Override 062 public void init(TransientStoreConfig config) { 063 this.config = config; 064 065 // initialize caching directory 066 File transienStoreHome = new File(Environment.getDefault().getData(), "transientstores"); 067 File data = new File(transienStoreHome, config.getName()); 068 data.mkdirs(); 069 cacheDir = data.getAbsoluteFile(); 070 } 071 072 @Override 073 public abstract void shutdown(); 074 075 @Override 076 public abstract boolean exists(String key); 077 078 @Override 079 public abstract void putParameter(String key, String parameter, Serializable value); 080 081 @Override 082 public abstract Serializable getParameter(String key, String parameter); 083 084 @Override 085 public abstract void putParameters(String key, Map<String, Serializable> parameters); 086 087 @Override 088 public abstract Map<String, Serializable> getParameters(String key); 089 090 @Override 091 public abstract List<Blob> getBlobs(String key); 092 093 @Override 094 public abstract long getSize(String key); 095 096 @Override 097 public abstract boolean isCompleted(String key); 098 099 @Override 100 public abstract void setCompleted(String key, boolean completed); 101 102 @Override 103 public abstract void remove(String key); 104 105 @Override 106 public abstract void release(String key); 107 108 /** 109 * Updates the total storage size and the storage size of the entry with the given {@code key} according to 110 * {@code sizeOfBlobs} and stores the blob information in this entry. 111 */ 112 protected abstract void persistBlobs(String key, long sizeOfBlobs, List<Map<String, String>> blobInfos); 113 114 /** 115 * Returns the size of the disk storage in bytes. 116 */ 117 public abstract long getStorageSize(); 118 119 /** 120 * Sets the size of the disk storage in bytes. 121 */ 122 protected abstract void setStorageSize(long newSize); 123 124 protected abstract long incrementStorageSize(long size); 125 126 protected abstract long decrementStorageSize(long size); 127 128 protected abstract void removeAllEntries(); 129 130 @Override 131 public void putBlobs(String key, List<Blob> blobs) { 132 if (config.getAbsoluteMaxSizeMB() < 0 || getStorageSize() < config.getAbsoluteMaxSizeMB() * (1024 * 1024)) { 133 // Store blobs on the file system 134 List<Map<String, String>> blobInfos = storeBlobs(key, blobs); 135 // Persist blob information in the store 136 persistBlobs(key, getSizeOfBlobs(blobs), blobInfos); 137 } else { 138 throw new MaximumTransientSpaceExceeded(); 139 } 140 } 141 142 protected List<Map<String, String>> storeBlobs(String key, List<Blob> blobs) { 143 if (blobs == null) { 144 return null; 145 } 146 // Store blobs on the file system and compute blob information 147 List<Map<String, String>> blobInfos = new ArrayList<>(); 148 for (Blob blob : blobs) { 149 Map<String, String> blobInfo = new HashMap<>(); 150 File cachingDir = getCachingDirectory(key); 151 String uuid = UUID.randomUUID().toString(); 152 File cachedFile = new File(cachingDir, uuid); 153 try { 154 if (blob instanceof FileBlob && ((FileBlob) blob).isTemporary()) { 155 ((FileBlob) blob).moveTo(cachedFile); 156 } else { 157 blob.transferTo(cachedFile); 158 } 159 } catch (IOException e) { 160 throw new NuxeoException(e); 161 } 162 Path cachedFileRelativePath = Paths.get(cachingDir.getName(), uuid); 163 blobInfo.put("file", cachedFileRelativePath.toString()); 164 // Redis doesn't support null values 165 if (blob.getFilename() != null) { 166 blobInfo.put("filename", blob.getFilename()); 167 } 168 if (blob.getEncoding() != null) { 169 blobInfo.put("encoding", blob.getEncoding()); 170 } 171 if (blob.getMimeType() != null) { 172 blobInfo.put("mimetype", blob.getMimeType()); 173 } 174 if (blob.getDigest() != null) { 175 blobInfo.put("digest", blob.getDigest()); 176 } 177 blobInfos.add(blobInfo); 178 } 179 log.debug("Stored blobs on the file system: " + blobInfos); 180 return blobInfos; 181 } 182 183 public File getCachingDirectory(String key) { 184 try { 185 File cachingDir = new File(cacheDir.getCanonicalFile(), getCachingDirName(key)); 186 187 if (!cachingDir.getCanonicalPath().startsWith(cacheDir.getCanonicalPath())) { 188 throw new SecurityException("Trying to traverse illegal path"); 189 } 190 191 if (!cachingDir.exists()) { 192 cachingDir.mkdir(); 193 } 194 return cachingDir; 195 } catch (IOException e) { 196 throw new RuntimeException("Error when trying to access cache directory"); 197 } 198 } 199 200 protected String getCachingDirName(String key) { 201 String dirName = Base64.encodeBase64String(key.getBytes()); 202 dirName = dirName.replaceAll("/", "_"); 203 return dirName; 204 } 205 206 protected long getSizeOfBlobs(List<Blob> blobs) { 207 int size = 0; 208 if (blobs != null) { 209 for (Blob blob : blobs) { 210 long blobLength = blob.getLength(); 211 if (blobLength > -1) { 212 size += blobLength; 213 } 214 } 215 } 216 return size; 217 } 218 219 protected List<Blob> loadBlobs(List<Map<String, String>> blobInfos) { 220 log.debug("Loading blobs from the file system: " + blobInfos); 221 List<Blob> blobs = new ArrayList<>(); 222 for (Map<String, String> info : blobInfos) { 223 File blobFile = new File(cacheDir, info.get("file")); 224 Blob blob = new FileBlob(blobFile); 225 blob.setEncoding(info.get("encoding")); 226 blob.setMimeType(info.get("mimetype")); 227 blob.setFilename(info.get("filename")); 228 blob.setDigest(info.get("digest")); 229 blobs.add(blob); 230 } 231 return blobs; 232 } 233 234 @Override 235 public int getStorageSizeMB() { 236 return (int) getStorageSize() / (1024 * 1024); 237 } 238 239 @Override 240 public void doGC() { 241 log.debug(String.format("Performing GC for TransientStore %s", config.getName())); 242 long newSize = 0; 243 try { 244 try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(cacheDir.getAbsolutePath()))) { 245 for (Path entry : stream) { 246 String key = getKeyCachingDirName(entry.getFileName().toString()); 247 try { 248 if (exists(key)) { 249 newSize += getFilePathSize(entry); 250 continue; 251 } 252 FileUtils.deleteDirectory(entry.toFile()); 253 } catch (IOException e) { 254 log.error("Error while performing GC", e); 255 } 256 } 257 } 258 } catch (IOException e) { 259 log.error("Error while performing GC", e); 260 } 261 setStorageSize(newSize); 262 } 263 264 protected String getKeyCachingDirName(String dir) { 265 String key = dir.replaceAll("_", "/"); 266 return new String(Base64.decodeBase64(key)); 267 } 268 269 protected long getFilePathSize(Path entry) { 270 long size = 0; 271 for (File file : entry.toFile().listFiles()) { 272 size += file.length(); 273 } 274 return size; 275 } 276 277 @Override 278 public void removeAll() { 279 log.debug("Removing all entries from TransientStore " + config.getName()); 280 removeAllEntries(); 281 doGC(); 282 } 283 284}