001/* 002 * (C) Copyright 2006-2011 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.common.file; 020 021import java.io.File; 022import java.io.FileOutputStream; 023import java.io.IOException; 024import java.io.InputStream; 025import java.nio.file.DirectoryStream; 026import java.nio.file.FileAlreadyExistsException; 027import java.nio.file.Files; 028import java.nio.file.Path; 029import java.nio.file.attribute.FileTime; 030import java.util.ArrayList; 031import java.util.Collections; 032import java.util.List; 033import java.util.concurrent.locks.Lock; 034import java.util.concurrent.locks.ReentrantLock; 035import java.util.regex.Pattern; 036 037import org.apache.commons.io.IOUtils; 038import org.apache.commons.logging.Log; 039import org.apache.commons.logging.LogFactory; 040 041/** 042 * A LRU cache of {@link File}s with maximum filesystem size. 043 * <p> 044 * Cache entries that are old enough and whose size makes the cache bigger than its maximum size are deleted. 045 * <p> 046 * The cache keys are restricted to a subset of ASCII: letters, digits and dashes. Usually a MD5 or SHA1 hash is used. 047 */ 048public class LRUFileCache implements FileCache { 049 050 private static final Log log = LogFactory.getLog(LRUFileCache.class); 051 052 /** Allowed key pattern, used as file path. */ 053 public static final Pattern SIMPLE_ASCII = Pattern.compile("[-_a-zA-Z0-9]+"); 054 055 private static final String TMP_PREFIX = "nxbin_"; 056 057 private static final String TMP_SUFFIX = ".tmp"; 058 059 public static final long CLEAR_OLD_ENTRIES_INTERVAL_MILLIS_DEFAULT = 5000; // 5 s 060 061 protected long clearOldEntriesIntervalMillis = CLEAR_OLD_ENTRIES_INTERVAL_MILLIS_DEFAULT; 062 063 protected static class PathInfo implements Comparable<PathInfo> { 064 065 protected final Path path; 066 067 protected final long time; 068 069 protected final long size; 070 071 public PathInfo(Path path) throws IOException { 072 this.path = path; 073 this.time = Files.getLastModifiedTime(path).toMillis(); 074 this.size = Files.size(path); 075 } 076 077 @Override 078 public int compareTo(PathInfo other) { 079 return Long.compare(other.time, time); // compare in reverse order (most recent first) 080 } 081 } 082 083 protected final Path dir; 084 085 protected final long maxSize; 086 087 protected final long maxCount; 088 089 protected final long minAgeMillis; 090 091 protected Lock clearOldEntriesLock = new ReentrantLock(); 092 093 protected long clearOldEntriesLast; 094 095 /** 096 * Constructs a cache in the given directory with the given maximum size (in bytes). 097 * 098 * @param dir the directory to use to store cached files 099 * @param maxSize the maximum size of the cache (in bytes) 100 * @param maxCount the maximum number of files in the cache 101 * @param minAge the minimum age of a file in the cache to be eligible for removal (in seconds) 102 */ 103 public LRUFileCache(File dir, long maxSize, long maxCount, long minAge) { 104 this.dir = dir.toPath(); 105 this.maxSize = maxSize; 106 this.maxCount = maxCount; 107 this.minAgeMillis = minAge * 1000; 108 } 109 110 // for tests 111 public void setClearOldEntriesIntervalMillis(long millis) { 112 clearOldEntriesIntervalMillis = millis; 113 } 114 115 /** 116 * Filter keeping regular files that aren't temporary. 117 */ 118 protected static class RegularFileFilter implements DirectoryStream.Filter<Path> { 119 120 protected static final RegularFileFilter INSTANCE = new RegularFileFilter(); 121 122 @Override 123 public boolean accept(Path path) { 124 if (!Files.isRegularFile(path)) { 125 return false; 126 } 127 String filename = path.getFileName().toString(); 128 if (filename.startsWith(TMP_PREFIX) && filename.endsWith(TMP_SUFFIX)) { 129 return false; 130 } 131 return true; 132 } 133 } 134 135 @Override 136 public long getSize() { 137 long size = 0; 138 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) { 139 for (Path path : ds) { 140 size += Files.size(path); 141 } 142 } catch (IOException e) { 143 log.error(e, e); 144 } 145 return size; 146 } 147 148 @Override 149 public int getNumberOfItems() { 150 int count = 0; 151 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) { 152 for (Path path : ds) { 153 count++; 154 } 155 } catch (IOException e) { 156 log.error(e, e); 157 } 158 return count; 159 } 160 161 @Override 162 public void clear() { 163 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) { 164 for (Path path : ds) { 165 try { 166 Files.delete(path); 167 } catch (IOException e) { 168 log.error(e, e); 169 } 170 } 171 } catch (IOException e) { 172 log.error(e, e); 173 } 174 } 175 176 /** 177 * Clears cache entries if they are old enough and their size makes the cache bigger than its maximum size. 178 */ 179 protected void clearOldEntries() { 180 if (clearOldEntriesLock.tryLock()) { 181 try { 182 if (System.currentTimeMillis() > clearOldEntriesLast + clearOldEntriesIntervalMillis) { 183 doClearOldEntries(); 184 clearOldEntriesLast = System.currentTimeMillis(); 185 return; 186 } 187 } finally { 188 clearOldEntriesLock.unlock(); 189 } 190 } 191 // else don't do anything, another thread is already clearing old entries 192 } 193 194 protected void doClearOldEntries() { 195 List<PathInfo> files = new ArrayList<>(); 196 try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) { 197 for (Path path : ds) { 198 try { 199 files.add(new PathInfo(path)); 200 } catch (IOException e) { 201 log.error(e, e); 202 } 203 } 204 } catch (IOException e) { 205 log.error(e, e); 206 } 207 Collections.sort(files); // sort by most recent first 208 209 long size = 0; 210 long count = 0; 211 long threshold = System.currentTimeMillis() - minAgeMillis; 212 for (PathInfo pi : files) { 213 size += pi.size; 214 count++; 215 if (pi.time < threshold) { 216 // old enough to be candidate 217 if (size > maxSize || count > maxCount) { 218 // delete file 219 try { 220 Files.delete(pi.path); 221 size -= pi.size; 222 count--; 223 } catch (IOException e) { 224 log.error(e, e); 225 } 226 } 227 } 228 } 229 } 230 231 @Override 232 public File getTempFile() throws IOException { 233 // make sure we have a temporary directory 234 // even if it's been deleted by an external process doing cleanup 235 Files.createDirectories(dir); 236 return Files.createTempFile(dir, TMP_PREFIX, TMP_SUFFIX).toFile(); 237 } 238 239 protected void checkKey(String key) throws IllegalArgumentException { 240 if (!SIMPLE_ASCII.matcher(key).matches() || ".".equals(key) || "..".equals(key)) { 241 throw new IllegalArgumentException("Invalid key: " + key); 242 } 243 } 244 245 /** 246 * {@inheritDoc} 247 * <p> 248 * The key is used as a file name in the directory cache. 249 */ 250 @Override 251 public File putFile(String key, InputStream in) throws IOException { 252 File tmp; 253 try { 254 // check the cache 255 checkKey(key); 256 Path path = dir.resolve(key); 257 if (Files.exists(path)) { 258 recordAccess(path); 259 return path.toFile(); 260 } 261 262 // store the stream in a temporary file 263 tmp = getTempFile(); 264 try (FileOutputStream out = new FileOutputStream(tmp)) { 265 IOUtils.copy(in, out); 266 } 267 } finally { 268 in.close(); 269 } 270 return putFile(key, tmp); 271 } 272 273 /** 274 * {@inheritDoc} 275 * <p> 276 * The key is used as a file name in the directory cache. 277 */ 278 @Override 279 public File putFile(String key, File file) throws IllegalArgumentException, IOException { 280 Path source = file.toPath(); 281 282 // put file in cache 283 checkKey(key); 284 Path path = dir.resolve(key); 285 try { 286 Files.move(source, path); 287 recordAccess(path); 288 clearOldEntries(); 289 } catch (FileAlreadyExistsException faee) { 290 // already something there 291 recordAccess(path); 292 // remove unused tmp file 293 try { 294 Files.delete(source); 295 } catch (IOException e) { 296 log.error(e, e); 297 } 298 } 299 return path.toFile(); 300 } 301 302 @Override 303 public File getFile(String key) { 304 checkKey(key); 305 Path path = dir.resolve(key); 306 if (!Files.exists(path)) { 307 return null; 308 } 309 recordAccess(path); 310 return path.toFile(); 311 } 312 313 /** Records access to a file by changing its modification time. */ 314 protected void recordAccess(Path path) { 315 try { 316 Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis())); 317 } catch (IOException e) { 318 log.error(e, e); 319 } 320 } 321 322}