001/* 002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Thierry Delprat <tdelprat@nuxeo.com> 018 * Antoine Taillefer <ataillefer@nuxeo.com> 019 */ 020 021package org.nuxeo.ecm.core.transientstore; 022 023import java.io.File; 024import java.io.IOException; 025import java.io.Serializable; 026import java.nio.file.DirectoryStream; 027import java.nio.file.Files; 028import java.nio.file.Path; 029import java.nio.file.Paths; 030import java.util.ArrayList; 031import java.util.HashMap; 032import java.util.List; 033import java.util.Map; 034import java.util.UUID; 035 036import org.apache.commons.codec.binary.Base64; 037import org.apache.commons.io.FileUtils; 038import org.apache.commons.lang3.StringUtils; 039import org.apache.commons.logging.Log; 040import org.apache.commons.logging.LogFactory; 041import org.nuxeo.common.Environment; 042import org.nuxeo.ecm.core.api.Blob; 043import org.nuxeo.ecm.core.api.NuxeoException; 044import org.nuxeo.ecm.core.api.impl.blob.FileBlob; 045import org.nuxeo.ecm.core.transientstore.api.MaximumTransientSpaceExceeded; 046import org.nuxeo.ecm.core.transientstore.api.TransientStore; 047import org.nuxeo.ecm.core.transientstore.api.TransientStoreConfig; 048import org.nuxeo.ecm.core.transientstore.api.TransientStoreProvider; 049 050/** 051 * Base class for a {@link TransientStore} implementation. 052 * 053 * @since 7.2 054 */ 055public abstract class AbstractTransientStore implements TransientStoreProvider { 056 057 protected static final Log log = LogFactory.getLog(AbstractTransientStore.class); 058 059 protected TransientStoreConfig config; 060 061 protected File cacheDir; 062 063 @Override 064 public void init(TransientStoreConfig config) { 065 this.config = config; 066 File data = getDataDir(config); 067 data.mkdirs(); 068 cacheDir = data.getAbsoluteFile(); 069 } 070 071 private File getDataDir(TransientStoreConfig config) { 072 String dataDirPath = config.getDataDir(); 073 if (StringUtils.isBlank(dataDirPath)) { 074 File transienStoreHome = new File(Environment.getDefault().getData(), "transientstores"); 075 return new File(transienStoreHome, config.getName()); 076 } else { 077 return new File(dataDirPath); 078 } 079 } 080 081 @Override 082 public abstract void shutdown(); 083 084 @Override 085 public abstract boolean exists(String key); 086 087 @Override 088 public abstract void putParameter(String key, String parameter, Serializable value); 089 090 @Override 091 public abstract Serializable getParameter(String key, String parameter); 092 093 @Override 094 public abstract void putParameters(String key, Map<String, Serializable> parameters); 095 096 @Override 097 public abstract Map<String, Serializable> getParameters(String key); 098 099 @Override 100 public abstract List<Blob> getBlobs(String key); 101 102 @Override 103 public abstract long getSize(String key); 104 105 @Override 106 public abstract boolean isCompleted(String key); 107 108 @Override 109 public abstract void setCompleted(String key, boolean completed); 110 111 @Override 112 public abstract void release(String key); 113 114 /** 115 * Updates the total storage size and the storage size of the entry with the given {@code key} according to 116 * {@code sizeOfBlobs} and stores the blob information in this entry. 117 */ 118 protected abstract void persistBlobs(String key, long sizeOfBlobs, List<Map<String, String>> blobInfos); 119 120 /** 121 * Sets the size of the disk storage in bytes. 122 */ 123 protected abstract void setStorageSize(long newSize); 124 125 protected abstract long incrementStorageSize(long size); 126 127 protected abstract long decrementStorageSize(long size); 128 129 protected abstract void removeAllEntries(); 130 131 protected abstract void removeEntry(String key); 132 133 @Override 134 public void putBlobs(String key, List<Blob> blobs) { 135 if (config.getAbsoluteMaxSizeMB() < 0 || getStorageSize() < config.getAbsoluteMaxSizeMB() * (1024 * 1024)) { 136 // Store blobs on the file system 137 List<Map<String, String>> blobInfos = storeBlobs(key, blobs); 138 // Persist blob information in the store 139 persistBlobs(key, getSizeOfBlobs(blobs), blobInfos); 140 } else { 141 throw new MaximumTransientSpaceExceeded(); 142 } 143 } 144 145 protected List<Map<String, String>> storeBlobs(String key, List<Blob> blobs) { 146 if (blobs == null) { 147 return null; 148 } 149 // Store blobs on the file system and compute blob information 150 List<Map<String, String>> blobInfos = new ArrayList<>(); 151 for (Blob blob : blobs) { 152 Map<String, String> blobInfo = new HashMap<>(); 153 File cachingDir = getCachingDirectory(key); 154 String uuid = UUID.randomUUID().toString(); 155 File cachedFile = new File(cachingDir, uuid); 156 try { 157 if (blob instanceof FileBlob && ((FileBlob) blob).isTemporary()) { 158 ((FileBlob) blob).moveTo(cachedFile); 159 } else { 160 blob.transferTo(cachedFile); 161 } 162 } catch (IOException e) { 163 throw new NuxeoException(e); 164 } 165 Path cachedFileRelativePath = Paths.get(cachingDir.getName(), uuid); 166 blobInfo.put("file", cachedFileRelativePath.toString()); 167 // Redis doesn't support null values 168 if (blob.getFilename() != null) { 169 blobInfo.put("filename", blob.getFilename()); 170 } 171 if (blob.getEncoding() != null) { 172 blobInfo.put("encoding", blob.getEncoding()); 173 } 174 if (blob.getMimeType() != null) { 175 blobInfo.put("mimetype", blob.getMimeType()); 176 } 177 if (blob.getDigest() != null) { 178 blobInfo.put("digest", blob.getDigest()); 179 } 180 blobInfos.add(blobInfo); 181 } 182 log.debug("Stored blobs on the file system: " + blobInfos); 183 return blobInfos; 184 } 185 186 public File getCachingDirectory(String key) { 187 String cachingDirName = getCachingDirName(key); 188 try { 189 File cachingDir = new File(cacheDir.getCanonicalFile(), cachingDirName); 190 if (!cachingDir.getCanonicalPath().startsWith(cacheDir.getCanonicalPath())) { 191 throw new NuxeoException("Trying to traverse illegal path: " + cachingDir + " for key: " + key); 192 } 193 if (!cachingDir.exists()) { 194 cachingDir.mkdir(); 195 } 196 return cachingDir; 197 } catch (IOException e) { 198 throw new NuxeoException("Error when trying to access cache directory: " + cacheDir + "/" + cachingDirName 199 + " for key: " + key, e); 200 } 201 } 202 203 protected String getCachingDirName(String key) { 204 String dirName = Base64.encodeBase64String(key.getBytes()); 205 dirName = dirName.replaceAll("/", "_"); 206 return dirName; 207 } 208 209 protected long getSizeOfBlobs(List<Blob> blobs) { 210 int size = 0; 211 if (blobs != null) { 212 for (Blob blob : blobs) { 213 long blobLength = blob.getLength(); 214 if (blobLength > -1) { 215 size += blobLength; 216 } 217 } 218 } 219 return size; 220 } 221 222 protected List<Blob> loadBlobs(List<Map<String, String>> blobInfos) { 223 log.debug("Loading blobs from the file system: " + blobInfos); 224 List<Blob> blobs = new ArrayList<>(); 225 for (Map<String, String> info : blobInfos) { 226 File blobFile = new File(cacheDir, info.get("file")); 227 Blob blob = new FileBlob(blobFile); 228 blob.setEncoding(info.get("encoding")); 229 blob.setMimeType(info.get("mimetype")); 230 blob.setFilename(info.get("filename")); 231 blob.setDigest(info.get("digest")); 232 blobs.add(blob); 233 } 234 return blobs; 235 } 236 237 @Override 238 public void doGC() { 239 log.debug(String.format("Performing GC for TransientStore %s", config.getName())); 240 long newSize = 0; 241 try { 242 try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(cacheDir.getAbsolutePath()))) { 243 for (Path entry : stream) { 244 String key = getKeyCachingDirName(entry.getFileName().toString()); 245 if (exists(key)) { 246 newSize += getFilePathSize(entry); 247 continue; 248 } 249 FileUtils.deleteQuietly(entry.toFile()); 250 } 251 } 252 } catch (IOException e) { 253 log.error("Error while performing GC", e); 254 } 255 setStorageSize(newSize); 256 } 257 258 protected String getKeyCachingDirName(String dir) { 259 String key = dir.replaceAll("_", "/"); 260 return new String(Base64.decodeBase64(key)); 261 } 262 263 protected long getFilePathSize(Path entry) { 264 long size = 0; 265 for (File file : entry.toFile().listFiles()) { 266 size += file.length(); 267 } 268 return size; 269 } 270 271 @Override 272 public void remove(String key) { 273 removeEntry(key); 274 removeBlobs(key); 275 } 276 277 protected void removeBlobs(String key) { 278 FileUtils.deleteQuietly(getCachingDirectory(key)); 279 } 280 281 @Override 282 public void removeAll() { 283 log.debug("Removing all entries from TransientStore " + config.getName()); 284 removeAllEntries(); 285 doGC(); 286 } 287 288 public File getCacheDir() { 289 return cacheDir; 290 } 291 292}