001/*
002 * (C) Copyright 2006-2011 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.common.file;
020
021import java.io.File;
022import java.io.FileOutputStream;
023import java.io.IOException;
024import java.io.InputStream;
025import java.nio.file.DirectoryStream;
026import java.nio.file.FileAlreadyExistsException;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.nio.file.attribute.FileTime;
030import java.util.ArrayList;
031import java.util.Collections;
032import java.util.List;
033import java.util.concurrent.locks.Lock;
034import java.util.concurrent.locks.ReentrantLock;
035import java.util.regex.Pattern;
036
037import org.apache.commons.io.IOUtils;
038import org.apache.commons.logging.Log;
039import org.apache.commons.logging.LogFactory;
040
041/**
042 * A LRU cache of {@link File}s with maximum filesystem size.
043 * <p>
044 * Cache entries that are old enough and whose size makes the cache bigger than its maximum size are deleted.
045 * <p>
046 * The cache keys are restricted to a subset of ASCII: letters, digits and dashes. Usually a MD5 or SHA1 hash is used.
047 */
048public class LRUFileCache implements FileCache {
049
050    private static final Log log = LogFactory.getLog(LRUFileCache.class);
051
052    /** Allowed key pattern, used as file path. */
053    public static final Pattern SIMPLE_ASCII = Pattern.compile("[-_a-zA-Z0-9]+");
054
055    private static final String TMP_PREFIX = "nxbin_";
056
057    private static final String TMP_SUFFIX = ".tmp";
058
059    // not final for tests
060    public static long CLEAR_OLD_ENTRIES_INTERVAL_MILLIS = 5000; // 5 s
061
062    protected static class PathInfo implements Comparable<PathInfo> {
063
064        protected final Path path;
065
066        protected final long time;
067
068        protected final long size;
069
070        public PathInfo(Path path) throws IOException {
071            this.path = path;
072            this.time = Files.getLastModifiedTime(path).toMillis();
073            this.size = Files.size(path);
074        }
075
076        @Override
077        public int compareTo(PathInfo other) {
078            return Long.compare(other.time, time); // compare in reverse order (most recent first)
079        }
080    }
081
082    protected final Path dir;
083
084    protected final long maxSize;
085
086    protected final long maxCount;
087
088    protected final long minAgeMillis;
089
090    protected Lock clearOldEntriesLock = new ReentrantLock();
091
092    protected long clearOldEntriesLast;
093
094    /**
095     * Constructs a cache in the given directory with the given maximum size (in bytes).
096     *
097     * @param dir the directory to use to store cached files
098     * @param maxSize the maximum size of the cache (in bytes)
099     * @param maxCount the maximum number of files in the cache
100     * @param minAge the minimum age of a file in the cache to be eligible for removal (in seconds)
101     */
102    public LRUFileCache(File dir, long maxSize, long maxCount, long minAge) {
103        this.dir = dir.toPath();
104        this.maxSize = maxSize;
105        this.maxCount = maxCount;
106        this.minAgeMillis = minAge * 1000;
107    }
108
109    /**
110     * Filter keeping regular files that aren't temporary.
111     */
112    protected static class RegularFileFilter implements DirectoryStream.Filter<Path> {
113
114        protected static final RegularFileFilter INSTANCE = new RegularFileFilter();
115
116        @Override
117        public boolean accept(Path path) {
118            if (!Files.isRegularFile(path)) {
119                return false;
120            }
121            String filename = path.getFileName().toString();
122            if (filename.startsWith(TMP_PREFIX) && filename.endsWith(TMP_SUFFIX)) {
123                return false;
124            }
125            return true;
126        }
127    }
128
129    @Override
130    public long getSize() {
131        long size = 0;
132        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
133            for (Path path : ds) {
134                size += Files.size(path);
135            }
136        } catch (IOException e) {
137            log.error(e, e);
138        }
139        return size;
140    }
141
142    @Override
143    public int getNumberOfItems() {
144        int count = 0;
145        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
146            for (Path path : ds) {
147                count++;
148            }
149        } catch (IOException e) {
150            log.error(e, e);
151        }
152        return count;
153    }
154
155    @Override
156    public void clear() {
157        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
158            for (Path path : ds) {
159                try {
160                    Files.delete(path);
161                } catch (IOException e) {
162                    log.error(e, e);
163                }
164            }
165        } catch (IOException e) {
166            log.error(e, e);
167        }
168    }
169
170    /**
171     * Clears cache entries if they are old enough and their size makes the cache bigger than its maximum size.
172     */
173    protected void clearOldEntries() {
174        if (clearOldEntriesLock.tryLock()) {
175            try {
176                if (System.currentTimeMillis() > clearOldEntriesLast + CLEAR_OLD_ENTRIES_INTERVAL_MILLIS) {
177                    doClearOldEntries();
178                    clearOldEntriesLast = System.currentTimeMillis();
179                    return;
180                }
181            } finally {
182                clearOldEntriesLock.unlock();
183            }
184        }
185        // else don't do anything, another thread is already clearing old entries
186    }
187
188    protected void doClearOldEntries() {
189        List<PathInfo> files = new ArrayList<>();
190        try (DirectoryStream<Path> ds = Files.newDirectoryStream(dir, RegularFileFilter.INSTANCE)) {
191            for (Path path : ds) {
192                try {
193                    files.add(new PathInfo(path));
194                } catch (IOException e) {
195                    log.error(e, e);
196                }
197            }
198        } catch (IOException e) {
199            log.error(e, e);
200        }
201        Collections.sort(files); // sort by most recent first
202
203        long size = 0;
204        long count = 0;
205        long threshold = System.currentTimeMillis() - minAgeMillis;
206        for (PathInfo pi : files) {
207            size += pi.size;
208            count++;
209            if (pi.time < threshold) {
210                // old enough to be candidate
211                if (size > maxSize || count > maxCount) {
212                    // delete file
213                    try {
214                        Files.delete(pi.path);
215                        size -= pi.size;
216                        count--;
217                    } catch (IOException e) {
218                        log.error(e, e);
219                    }
220                }
221            }
222        }
223    }
224
225    @Override
226    public File getTempFile() throws IOException {
227        return Files.createTempFile(dir, TMP_PREFIX, TMP_SUFFIX).toFile();
228    }
229
230    protected void checkKey(String key) throws IllegalArgumentException {
231        if (!SIMPLE_ASCII.matcher(key).matches() || ".".equals(key) || "..".equals(key)) {
232            throw new IllegalArgumentException("Invalid key: " + key);
233        }
234    }
235
236    /**
237     * {@inheritDoc}
238     * <p>
239     * The key is used as a file name in the directory cache.
240     */
241    @Override
242    public File putFile(String key, InputStream in) throws IOException {
243        File tmp;
244        try {
245            // check the cache
246            checkKey(key);
247            Path path = dir.resolve(key);
248            if (Files.exists(path)) {
249                recordAccess(path);
250                return path.toFile();
251            }
252
253            // store the stream in a temporary file
254            tmp = getTempFile();
255            try (FileOutputStream out = new FileOutputStream(tmp)) {
256                IOUtils.copy(in, out);
257            }
258        } finally {
259            in.close();
260        }
261        return putFile(key, tmp);
262    }
263
264    /**
265     * {@inheritDoc}
266     * <p>
267     * The key is used as a file name in the directory cache.
268     */
269    @Override
270    public File putFile(String key, File file) throws IllegalArgumentException, IOException {
271        Path source = file.toPath();
272
273        // put file in cache
274        checkKey(key);
275        Path path = dir.resolve(key);
276        try {
277            Files.move(source, path);
278            recordAccess(path);
279            clearOldEntries();
280        } catch (FileAlreadyExistsException faee) {
281            // already something there
282            recordAccess(path);
283            // remove unused tmp file
284            try {
285                Files.delete(source);
286            } catch (IOException e) {
287                log.error(e, e);
288            }
289        }
290        return path.toFile();
291    }
292
293    @Override
294    public File getFile(String key) {
295        checkKey(key);
296        Path path = dir.resolve(key);
297        if (!Files.exists(path)) {
298            return null;
299        }
300        recordAccess(path);
301        return path.toFile();
302    }
303
304    /** Records access to a file by changing its modification time. */
305    protected void recordAccess(Path path) {
306        try {
307            Files.setLastModifiedTime(path, FileTime.fromMillis(System.currentTimeMillis()));
308        } catch (IOException e) {
309            log.error(e, e);
310        }
311    }
312
313}