Source code

001/*
002 * (C) Copyright 2006-2011 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.common.file;
020
021import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
022import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
023
024import java.io.File;
025import java.io.FileOutputStream;
026import java.io.IOException;
027import java.io.InputStream;
028import java.nio.file.DirectoryStream;
029import java.nio.file.FileAlreadyExistsException;
030import java.nio.file.Files;
031import java.nio.file.Path;
032import java.nio.file.attribute.FileTime;
033import java.util.ArrayList;
034import java.util.Collections;
035import java.util.List;
036import java.util.concurrent.locks.Lock;
037import java.util.concurrent.locks.ReentrantLock;
038import java.util.regex.Pattern;
039
040import org.apache.commons.io.IOUtils;
041import org.apache.commons.logging.Log;
042import org.apache.commons.logging.LogFactory;
043
044/**
045 * A LRU cache of {@link File}s with maximum filesystem size.
046 * <p>
047 * Cache entries that are old enough and whose size makes the cache bigger than its maximum size are deleted.
048 * <p>
049 * The cache keys are restricted to a subset of ASCII: letters, digits and dashes. Usually a MD5 or SHA1 hash is used.
050 */
051public class LRUFileCache implements FileCache {
052
053    private static final Log log = LogFactory.getLog(LRUFileCache.class);
054
055    /** Allowed key pattern, used as file path. */
056    public static final Pattern SIMPLE_ASCII = Pattern.compile("[-_.@;a-zA-Z0-9]+");
057
058    private static final String TMP_PREFIX = "nxbin_";
059
060    private static final String TMP_SUFFIX = ".tmp";
061
062    public static final long CLEAR_OLD_ENTRIES_INTERVAL_MILLIS_DEFAULT = 5000; // 5 s
063
064    protected long clearOldEntriesIntervalMillis = CLEAR_OLD_ENTRIES_INTERVAL_MILLIS_DEFAULT;
065
066    protected static class PathInfo implements Comparable<PathInfo> {
067
068        protected final Path path;
069
070        protected final long time;
071
072        protected final long size;
073
074        public PathInfo(Path path) throws IOException {
075            this.path = path;
076            this.time = Files.getLastModifiedTime(path).toMillis();
077            this.size = Files.size(path);
078        }
079
080        @Override
081        public int compareTo(PathInfo other) {
082            return Long.compare(other.time, time); // compare in reverse order (most recent first)
083        }
084    }
085
086    protected final Path dir;
087
088    protected final long maxSize;
089
090    protected final long maxCount;
091
092    protected final long minAgeMillis;
093
094    protected Lock clearOldEntriesLock = new ReentrantLock();
095
096    protected long clearOldEntriesLast;
097
098    /**
099     * Constructs a cache in the given directory with the given maximum size (in bytes).
100     *
101     * @param dir the directory to use to store cached files
102     * @param maxSize the maximum size of the cache (in bytes)
103     * @param maxCount the maximum number of files in the cache
104     * @param minAge the minimum age of a file in the cache to be eligible for removal (in seconds)
105     */
106    public LRUFileCache(File dir, long maxSize, long maxCount, long minAge) {
107        this.dir = dir.toPath();
108        this.maxSize = maxSize;
109        this.maxCount = maxCount;
110        this.minAgeMillis = minAge * 1000;
111    }
112
113    // for tests
114    public void setClearOldEntriesIntervalMillis(long millis) {
115        clearOldEntriesIntervalMillis = millis;
116    }
117
118    /**
119     * Filter keeping regular files that aren't temporary.
120     */
121    protected static class RegularFileFilter implements DirectoryStream.Filter<Path> {
122
123        protected static final RegularFileFilter INSTANCE = new RegularFileFilter();
124
125        @Override
126        public boolean accept(Path path) {
127            if (!Files.isRegularFile(path)) {
128                return false;
129            }
130            String filename = path.getFileName().toString();
131            if (filename.startsWith(TMP_PREFIX) && filename.endsWith(TMP_SUFFIX)) {
132                return false;
133            }
134            return true;
135        }
136    }
137
138    @Override
139    public long getSize() {
140        long size = 0;
141        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
142            for (Path path : ds) {
143                size += Files.size(path);
144            }
145        } catch (IOException e) {
146            log.error(e, e);
147        }
148        return size;
149    }
150
151    @Override
152    public int getNumberOfItems() {
153        int count = 0;
154        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
155            for (Path path : ds) {
156                count++;
157            }
158        } catch (IOException e) {
159            log.error(e, e);
160        }
161        return count;
162    }
163
164    @Override
165    public void clear() {
166        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
167            for (Path path : ds) {
168                try {
169                    Files.delete(path);
170                } catch (IOException e) {
171                    log.error(e, e);
172                }
173            }
174        } catch (IOException e) {
175            log.error(e, e);
176        }
177    }
178
179    /**
180     * Clears cache entries if they are old enough and their size makes the cache bigger than its maximum size.
181     */
182    protected void clearOldEntries() {
183        if (clearOldEntriesLock.tryLock()) {
184            try {
185                if (System.currentTimeMillis() > clearOldEntriesLast + clearOldEntriesIntervalMillis) {
186                    doClearOldEntries();
187                    clearOldEntriesLast = System.currentTimeMillis();
188                    return;
189                }
190            } finally {
191                clearOldEntriesLock.unlock();
192            }
193        }
194        // else don't do anything, another thread is already clearing old entries
195    }
196
197    protected void doClearOldEntries() {
198        List<PathInfo> files = new ArrayList<>();
199        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
200            for (Path path : ds) {
201                try {
202                    files.add(new PathInfo(path));
203                } catch (IOException e) {
204                    log.error(e, e);
205                }
206            }
207        } catch (IOException e) {
208            log.error(e, e);
209        }
210        Collections.sort(files); // sort by most recent first
211
212        long size = 0;
213        long count = 0;
214        long threshold = System.currentTimeMillis() - minAgeMillis;
215        for (PathInfo pi : files) {
216            size += pi.size;
217            count++;
218            if (pi.time < threshold) {
219                // old enough to be candidate
220                if (size > maxSize || count > maxCount) {
221                    // delete file
222                    try {
223                        Files.delete(pi.path);
224                        size -= pi.size;
225                        count--;
226                    } catch (IOException e) {
227                        log.error(e, e);
228                    }
229                }
230            }
231        }
232    }
233
234    @Override
235    public File getTempFile() throws IOException {
236        // make sure we have a temporary directory
237        // even if it's been deleted by an external process doing cleanup
238        Files.createDirectories(dir);
239        return Files.createTempFile(dir, TMP_PREFIX, TMP_SUFFIX).toFile();
240    }
241
242    protected void checkKey(String key) throws IllegalArgumentException {
243        if (!SIMPLE_ASCII.matcher(key).matches() || ".".equals(key) || "..".equals(key)) {
244            throw new IllegalArgumentException("Invalid key: " + key);
245        }
246    }
247
248    /**
249     * {@inheritDoc}
250     * <p>
251     * The key is used as a file name in the directory cache.
252     */
253    @Override
254    public File putFile(String key, InputStream in) throws IOException {
255        File tmp;
256        try {
257            // check the cache
258            checkKey(key);
259            Path path = dir.resolve(key);
260            if (Files.exists(path)) {
261                recordAccess(path);
262                return path.toFile();
263            }
264
265            // store the stream in a temporary file
266            tmp = getTempFile();
267            try (FileOutputStream out = new FileOutputStream(tmp)) {
268                IOUtils.copy(in, out);
269            }
270        } finally {
271            in.close();
272        }
273        return putFile(key, tmp);
274    }
275
276    /**
277     * {@inheritDoc}
278     * <p>
279     * The key is used as a file name in the directory cache.
280     */
281    @Override
282    public File putFile(String key, File file) throws IllegalArgumentException, IOException {
283        Path source = file.toPath();
284
285        // put file in cache
286        checkKey(key);
287        Path path = dir.resolve(key);
288        try {
289            Files.move(source, path, ATOMIC_MOVE, REPLACE_EXISTING);
290            recordAccess(path);
291            clearOldEntries();
292        } catch (FileAlreadyExistsException faee) {
293            // already something there
294            recordAccess(path);
295            // remove unused tmp file
296            try {
297                Files.delete(source);
298            } catch (IOException e) {
299                log.error(e, e);
300            }
301        }
302        return path.toFile();
303    }
304
305    @Override
306    public File getFile(String key) {
307        checkKey(key);
308        Path path = dir.resolve(key);
309        if (!Files.exists(path)) {
310            return null;
311        }
312        recordAccess(path);
313        return path.toFile();
314    }
315
316    /** Records access to a file by changing its modification time. */
317    protected void recordAccess(Path path) {
318        try {
319            Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
320        } catch (IOException e) {
321            log.error(e, e);
322        }
323    }
324
325}