001/* 002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and contributors. 003 * 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the GNU Lesser General Public License 006 * (LGPL) version 2.1 which accompanies this distribution, and is available at 007 * http://www.gnu.org/licenses/lgpl-2.1.html 008 * 009 * This library is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 012 * Lesser General Public License for more details. 013 * 014 * Contributors: 015 * Thierry Delprat <tdelprat@nuxeo.com> 016 * Antoine Taillefer <ataillefer@nuxeo.com> 017 */ 018 019package org.nuxeo.ecm.core.transientstore; 020 021import java.io.File; 022import java.io.IOException; 023import java.io.Serializable; 024import java.nio.file.DirectoryStream; 025import java.nio.file.Files; 026import java.nio.file.Path; 027import java.nio.file.Paths; 028import java.util.ArrayList; 029import java.util.HashMap; 030import java.util.List; 031import java.util.Map; 032import java.util.UUID; 033 034import org.apache.commons.codec.binary.Base64; 035import org.apache.commons.io.FileUtils; 036import org.apache.commons.logging.Log; 037import org.apache.commons.logging.LogFactory; 038import org.nuxeo.common.Environment; 039import org.nuxeo.ecm.core.api.Blob; 040import org.nuxeo.ecm.core.api.NuxeoException; 041import org.nuxeo.ecm.core.api.impl.blob.FileBlob; 042import org.nuxeo.ecm.core.transientstore.api.MaximumTransientSpaceExceeded; 043import org.nuxeo.ecm.core.transientstore.api.TransientStore; 044import org.nuxeo.ecm.core.transientstore.api.TransientStoreConfig; 045 046/** 047 * Base class for a {@link TransientStore} implementation. 048 * 049 * @since 7.2 050 */ 051public abstract class AbstractTransientStore implements TransientStore { 052 053 protected static final Log log = LogFactory.getLog(AbstractTransientStore.class); 054 055 protected TransientStoreConfig config; 056 057 protected File cacheDir; 058 059 @Override 060 public void init(TransientStoreConfig config) { 061 this.config = config; 062 063 // initialize caching directory 064 File transienStoreHome = new File(Environment.getDefault().getData(), "transientstores"); 065 File data = new File(transienStoreHome, config.getName()); 066 data.mkdirs(); 067 cacheDir = data.getAbsoluteFile(); 068 } 069 070 @Override 071 public abstract void shutdown(); 072 073 @Override 074 public abstract boolean exists(String key); 075 076 @Override 077 public abstract void putParameter(String key, String parameter, Serializable value); 078 079 @Override 080 public abstract Serializable getParameter(String key, String parameter); 081 082 @Override 083 public abstract void putParameters(String key, Map<String, Serializable> parameters); 084 085 @Override 086 public abstract Map<String, Serializable> getParameters(String key); 087 088 @Override 089 public abstract List<Blob> getBlobs(String key); 090 091 @Override 092 public abstract long getSize(String key); 093 094 @Override 095 public abstract boolean isCompleted(String key); 096 097 @Override 098 public abstract void setCompleted(String key, boolean completed); 099 100 @Override 101 public abstract void remove(String key); 102 103 @Override 104 public abstract void release(String key); 105 106 /** 107 * Updates the total storage size and the storage size of the entry with the given {@code key} according to 108 * {@code sizeOfBlobs} and stores the blob information in this entry. 109 */ 110 protected abstract void persistBlobs(String key, long sizeOfBlobs, List<Map<String, String>> blobInfos); 111 112 /** 113 * Returns the size of the disk storage in bytes. 114 */ 115 public abstract long getStorageSize(); 116 117 /** 118 * Sets the size of the disk storage in bytes. 119 */ 120 protected abstract void setStorageSize(long newSize); 121 122 protected abstract long incrementStorageSize(long size); 123 124 protected abstract long decrementStorageSize(long size); 125 126 protected abstract void removeAllEntries(); 127 128 @Override 129 public void putBlobs(String key, List<Blob> blobs) { 130 if (config.getAbsoluteMaxSizeMB() < 0 || getStorageSize() < config.getAbsoluteMaxSizeMB() * (1024 * 1024)) { 131 // Store blobs on the file system 132 List<Map<String, String>> blobInfos = storeBlobs(key, blobs); 133 // Persist blob information in the store 134 persistBlobs(key, getSizeOfBlobs(blobs), blobInfos); 135 } else { 136 throw new MaximumTransientSpaceExceeded(); 137 } 138 } 139 140 protected List<Map<String, String>> storeBlobs(String key, List<Blob> blobs) { 141 if (blobs == null) { 142 return null; 143 } 144 // Store blobs on the file system and compute blob information 145 List<Map<String, String>> blobInfos = new ArrayList<>(); 146 for (Blob blob : blobs) { 147 Map<String, String> blobInfo = new HashMap<>(); 148 File cachingDir = getCachingDirectory(key); 149 String uuid = UUID.randomUUID().toString(); 150 File cachedFile = new File(cachingDir, uuid); 151 try { 152 if (blob instanceof FileBlob && ((FileBlob) blob).isTemporary()) { 153 ((FileBlob) blob).moveTo(cachedFile); 154 } else { 155 blob.transferTo(cachedFile); 156 } 157 } catch (IOException e) { 158 throw new NuxeoException(e); 159 } 160 Path cachedFileRelativePath = Paths.get(cachingDir.getName(), uuid); 161 blobInfo.put("file", cachedFileRelativePath.toString()); 162 // Redis doesn't support null values 163 if (blob.getFilename() != null) { 164 blobInfo.put("filename", blob.getFilename()); 165 } 166 if (blob.getEncoding() != null) { 167 blobInfo.put("encoding", blob.getEncoding()); 168 } 169 if (blob.getMimeType() != null) { 170 blobInfo.put("mimetype", blob.getMimeType()); 171 } 172 if (blob.getDigest() != null) { 173 blobInfo.put("digest", blob.getDigest()); 174 } 175 blobInfos.add(blobInfo); 176 } 177 log.debug("Stored blobs on the file system: " + blobInfos); 178 return blobInfos; 179 } 180 181 public File getCachingDirectory(String key) { 182 try { 183 File cachingDir = new File(cacheDir.getCanonicalFile(), getCachingDirName(key)); 184 185 if (!cachingDir.getCanonicalPath().startsWith(cacheDir.getCanonicalPath())) { 186 throw new SecurityException("Trying to traverse illegal path"); 187 } 188 189 if (!cachingDir.exists()) { 190 cachingDir.mkdir(); 191 } 192 return cachingDir; 193 } catch (IOException e) { 194 throw new RuntimeException("Error when trying to access cache directory"); 195 } 196 } 197 198 protected String getCachingDirName(String key) { 199 String dirName = Base64.encodeBase64String(key.getBytes()); 200 dirName = dirName.replaceAll("/", "_"); 201 return dirName; 202 } 203 204 protected long getSizeOfBlobs(List<Blob> blobs) { 205 int size = 0; 206 if (blobs != null) { 207 for (Blob blob : blobs) { 208 long blobLength = blob.getLength(); 209 if (blobLength > -1) { 210 size += blobLength; 211 } 212 } 213 } 214 return size; 215 } 216 217 protected List<Blob> loadBlobs(List<Map<String, String>> blobInfos) { 218 log.debug("Loading blobs from the file system: " + blobInfos); 219 List<Blob> blobs = new ArrayList<>(); 220 for (Map<String, String> info : blobInfos) { 221 File blobFile = new File(cacheDir, info.get("file")); 222 Blob blob = new FileBlob(blobFile); 223 blob.setEncoding(info.get("encoding")); 224 blob.setMimeType(info.get("mimetype")); 225 blob.setFilename(info.get("filename")); 226 blob.setDigest(info.get("digest")); 227 blobs.add(blob); 228 } 229 return blobs; 230 } 231 232 @Override 233 public int getStorageSizeMB() { 234 return (int) getStorageSize() / (1024 * 1024); 235 } 236 237 @Override 238 public void doGC() { 239 log.debug(String.format("Performing GC for TransientStore %s", config.getName())); 240 long newSize = 0; 241 try { 242 try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(cacheDir.getAbsolutePath()))) { 243 for (Path entry : stream) { 244 String key = getKeyCachingDirName(entry.getFileName().toString()); 245 try { 246 if (exists(key)) { 247 newSize += getFilePathSize(entry); 248 continue; 249 } 250 FileUtils.deleteDirectory(entry.toFile()); 251 } catch (IOException e) { 252 log.error("Error while performing GC", e); 253 } 254 } 255 } 256 } catch (IOException e) { 257 log.error("Error while performing GC", e); 258 } 259 setStorageSize(newSize); 260 } 261 262 protected String getKeyCachingDirName(String dir) { 263 String key = dir.replaceAll("_", "/"); 264 return new String(Base64.decodeBase64(key)); 265 } 266 267 protected long getFilePathSize(Path entry) { 268 long size = 0; 269 for (File file : entry.toFile().listFiles()) { 270 size += file.length(); 271 } 272 return size; 273 } 274 275 @Override 276 public void removeAll() { 277 log.debug("Removing all entries from TransientStore " + config.getName()); 278 removeAllEntries(); 279 doGC(); 280 } 281 282}