001/* 002 * (C) Copyright 2011-2016 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Stephane Lacoin 018 * Florent Guillaume 019 */ 020package org.nuxeo.ecm.core.blob.binary; 021 022import java.io.File; 023import java.io.FileInputStream; 024import java.io.FileOutputStream; 025import java.io.IOException; 026import java.io.InputStream; 027import java.io.OutputStream; 028import java.io.OutputStreamWriter; 029import java.io.Writer; 030import java.util.Map; 031 032import org.apache.commons.io.FileUtils; 033import org.apache.commons.io.IOUtils; 034import org.apache.commons.logging.Log; 035import org.apache.commons.logging.LogFactory; 036import org.nuxeo.common.file.FileCache; 037import org.nuxeo.common.file.LRUFileCache; 038import org.nuxeo.common.utils.SizeUtils; 039import org.nuxeo.ecm.core.api.NuxeoException; 040import org.nuxeo.runtime.api.Framework; 041import org.nuxeo.runtime.trackers.files.FileEventTracker; 042 043/** 044 * Abstract class for a {@link BinaryManager} that uses a cache for its files because fetching them is expensive. 045 * <p> 046 * Initialization of the {@link BinaryManager} must call {@link #initializeCache} from the {@link #initialize} method. 047 * 048 * @since 5.7 049 */ 050public abstract class CachingBinaryManager extends AbstractBinaryManager { 051 052 private static final Log log = LogFactory.getLog(CachingBinaryManager.class); 053 054 protected static final String LEN_DIGEST_SUFFIX = "-len"; 055 056 protected File cachedir; 057 058 public FileCache fileCache; 059 060 protected FileStorage fileStorage; 061 062 @Override 063 public void initialize(String blobProviderId, Map<String, String> properties) throws IOException { 064 super.initialize(blobProviderId, properties); 065 descriptor = new BinaryManagerRootDescriptor(); 066 descriptor.digest = getDefaultDigestAlgorithm(); 067 log.info("Registering binary manager '" + blobProviderId + "' using " + getClass().getSimpleName()); 068 } 069 070 /** 071 * Initialize the cache. 072 * 073 * @param dir the directory to use to store cached files 074 * @param maxSize the maximum size of the cache (in bytes) 075 * @param maxCount the maximum number of files in the cache 076 * @param minAge the minimum age of a file in the cache to be eligible for removal (in seconds) 077 * @param fileStorage the file storage mechanism to use to store and fetch files 078 * @since 5.9.2 079 */ 080 protected void initializeCache(File dir, long maxSize, long maxCount, long minAge, FileStorage fileStorage) { 081 fileCache = new LRUFileCache(dir, maxSize, maxCount, minAge); 082 this.fileStorage = fileStorage; 083 } 084 085 /** 086 * Initialize the cache. 087 * 088 * @param maxSizeStr the maximum size of the cache (as a String) 089 * @param fileStorage the file storage mechanism to use to store and fetch files 090 * @see #initializeCache(String, String, String, FileStorage) 091 * @since 6.0 092 */ 093 public void initializeCache(String maxSizeStr, FileStorage fileStorage) throws IOException { 094 String maxCountStr = "10000"; // default for legacy code 095 String minAgeStr = "3600"; // default for legacy code 096 initializeCache(maxSizeStr, maxCountStr, minAgeStr, fileStorage); 097 } 098 099 /** 100 * Initializes the cache. 101 * 102 * @param maxSizeStr the maximum size of the cache (as a String) 103 * @param maxCountStr the maximum number of files in the cache 104 * @param minAgeStr the minimum age of a file in the cache to be eligible for removal (in seconds) 105 * @param fileStorage the file storage mechanism to use to store and fetch files 106 * @see SizeUtils#parseSizeInBytes(String) 107 * @since 7.10-HF03, 8.1 108 */ 109 public void initializeCache(String maxSizeStr, String maxCountStr, String minAgeStr, FileStorage fileStorage) 110 throws IOException { 111 cachedir = Framework.createTempFile("nxbincache.", ""); 112 cachedir.delete(); 113 cachedir.mkdir(); 114 long maxSize = SizeUtils.parseSizeInBytes(maxSizeStr); 115 long maxCount = Long.parseLong(maxCountStr); 116 long minAge = Long.parseLong(minAgeStr); 117 initializeCache(cachedir, maxSize, maxCount, minAge, fileStorage); 118 log.info("Using binary cache directory: " + cachedir.getPath() + " size: " + maxSizeStr + " maxCount: " 119 + maxCount + " minAge: " + minAge); 120 121 // be sure FileTracker won't steal our files ! 122 FileEventTracker.registerProtectedPath(cachedir.getAbsolutePath()); 123 } 124 125 @Override 126 public void close() { 127 fileCache.clear(); 128 if (cachedir != null) { 129 try { 130 FileUtils.deleteDirectory(cachedir); 131 } catch (IOException e) { 132 throw new NuxeoException(e); 133 } 134 } 135 } 136 137 @Override 138 protected Binary getBinary(InputStream in) throws IOException { 139 // write the input stream to a temporary file, while computing a digest 140 File tmp = fileCache.getTempFile(); 141 OutputStream out = new FileOutputStream(tmp); 142 String digest; 143 try { 144 digest = storeAndDigest(in, out); 145 } finally { 146 in.close(); 147 out.close(); 148 } 149 long length = tmp.length(); 150 151 File cachedFile = fileCache.getFile(digest); 152 if (cachedFile != null) { 153 // file already in cache 154 if (Framework.isTestModeSet()) { 155 Framework.getProperties().setProperty("cachedBinary", digest); 156 } 157 // delete tmp file, not needed anymore 158 tmp.delete(); 159 } else { 160 // send the file to storage 161 fileStorage.storeFile(digest, tmp); 162 // register the file in the file cache 163 fileCache.putFile(digest, tmp); 164 } 165 return new LazyBinary(digest, length, blobProviderId, this); 166 } 167 168 @Override 169 public Binary getBinary(String digest) { 170 return new LazyBinary(digest, blobProviderId, this); 171 } 172 173 /* =============== Methods used by LazyBinary =============== */ 174 175 /** 176 * Gets a file from cache or storage. 177 * <p> 178 * Used by {@link LazyBinary}. 179 */ 180 public File getFile(String digest) throws IOException { 181 // get file from cache 182 File file = fileCache.getFile(digest); 183 if (file != null) { 184 return file; 185 } 186 // fetch file from storage 187 File tmp = fileCache.getTempFile(); 188 if (fileStorage.fetchFile(digest, tmp)) { 189 // put file in cache 190 file = fileCache.putFile(digest, tmp); 191 return file; 192 } else { 193 // file not in storage 194 tmp.delete(); 195 return null; 196 } 197 } 198 199 /** 200 * Gets a file length from cache or storage. 201 * <p> 202 * Use by {@link LazyBinary}. 203 */ 204 public Long getLength(String digest) throws IOException { 205 // get length from cache 206 Long length = getLengthFromCache(digest); 207 if (length != null) { 208 return length; 209 } 210 // fetch length from storage 211 length = fileStorage.fetchLength(digest); 212 // put length in cache 213 putLengthInCache(digest, length); 214 return length; 215 } 216 217 protected Long getLengthFromCache(String digest) throws IOException { 218 File f = fileCache.getFile(digest + LEN_DIGEST_SUFFIX); 219 if (f == null) { 220 return null; 221 } 222 // read decimal length from file 223 InputStream in = null; 224 try { 225 in = new FileInputStream(f); 226 String len = IOUtils.toString(in); 227 return Long.valueOf(len); 228 } catch (NumberFormatException e) { 229 throw new IOException("Invalid length in " + f, e); 230 } finally { 231 IOUtils.closeQuietly(in); 232 } 233 } 234 235 protected void putLengthInCache(String digest, Long len) throws IOException { 236 if (len == null) { 237 return; 238 } 239 // write decimal length in file 240 OutputStream out = null; 241 try { 242 File tmp = fileCache.getTempFile(); 243 out = new FileOutputStream(tmp); 244 Writer writer = new OutputStreamWriter(out); 245 writer.write(len.toString()); 246 writer.flush(); 247 writer.close(); 248 fileCache.putFile(digest + LEN_DIGEST_SUFFIX, tmp); 249 } finally { 250 IOUtils.closeQuietly(out); 251 } 252 } 253 254}