001/*
002 * (C) Copyright 2011-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Stephane Lacoin
018 *     Florent Guillaume
019 */
020package org.nuxeo.ecm.core.blob.binary;
021
022import java.io.File;
023import java.io.FileInputStream;
024import java.io.FileOutputStream;
025import java.io.IOException;
026import java.io.InputStream;
027import java.io.OutputStream;
028import java.io.OutputStreamWriter;
029import java.io.Writer;
030import java.util.Map;
031
032import org.apache.commons.io.FileUtils;
033import org.apache.commons.io.IOUtils;
034import org.apache.commons.logging.Log;
035import org.apache.commons.logging.LogFactory;
036import org.nuxeo.common.file.FileCache;
037import org.nuxeo.common.file.LRUFileCache;
038import org.nuxeo.common.utils.SizeUtils;
039import org.nuxeo.ecm.core.api.NuxeoException;
040import org.nuxeo.runtime.api.Framework;
041import org.nuxeo.runtime.trackers.files.FileEventTracker;
042
043/**
044 * Abstract class for a {@link BinaryManager} that uses a cache for its files because fetching them is expensive.
045 * <p>
046 * Initialization of the {@link BinaryManager} must call {@link #initializeCache} from the {@link #initialize} method.
047 *
048 * @since 5.7
049 */
050public abstract class CachingBinaryManager extends AbstractBinaryManager {
051
052    private static final Log log = LogFactory.getLog(CachingBinaryManager.class);
053
054    protected static final String LEN_DIGEST_SUFFIX = "-len";
055
056    protected File cachedir;
057
058    public FileCache fileCache;
059
060    protected FileStorage fileStorage;
061
062    @Override
063    public void initialize(String blobProviderId, Map<String, String> properties) throws IOException {
064        super.initialize(blobProviderId, properties);
065        descriptor = new BinaryManagerRootDescriptor();
066        descriptor.digest = getDefaultDigestAlgorithm();
067        log.info("Registering binary manager '" + blobProviderId + "' using " + getClass().getSimpleName());
068    }
069
070    /**
071     * Initialize the cache.
072     *
073     * @param dir the directory to use to store cached files
074     * @param maxSize the maximum size of the cache (in bytes)
075     * @param maxCount the maximum number of files in the cache
076     * @param minAge the minimum age of a file in the cache to be eligible for removal (in seconds)
077     * @param fileStorage the file storage mechanism to use to store and fetch files
078     * @since 5.9.2
079     */
080    protected void initializeCache(File dir, long maxSize, long maxCount, long minAge, FileStorage fileStorage) {
081        fileCache = new LRUFileCache(dir, maxSize, maxCount, minAge);
082        this.fileStorage = fileStorage;
083    }
084
085    /**
086     * Initialize the cache.
087     *
088     * @param maxSizeStr the maximum size of the cache (as a String)
089     * @param fileStorage the file storage mechanism to use to store and fetch files
090     * @see #initializeCache(String, String, String, FileStorage)
091     * @since 6.0
092     */
093    public void initializeCache(String maxSizeStr, FileStorage fileStorage) throws IOException {
094        String maxCountStr = "10000"; // default for legacy code
095        String minAgeStr = "3600"; // default for legacy code
096        initializeCache(maxSizeStr, maxCountStr, minAgeStr, fileStorage);
097    }
098
099    /**
100     * Initializes the cache.
101     *
102     * @param maxSizeStr the maximum size of the cache (as a String)
103     * @param maxCountStr the maximum number of files in the cache
104     * @param minAgeStr the minimum age of a file in the cache to be eligible for removal (in seconds)
105     * @param fileStorage the file storage mechanism to use to store and fetch files
106     * @see SizeUtils#parseSizeInBytes(String)
107     * @since 7.10-HF03, 8.1
108     */
109    public void initializeCache(String maxSizeStr, String maxCountStr, String minAgeStr, FileStorage fileStorage)
110            throws IOException {
111        cachedir = Framework.createTempFile("nxbincache.", "");
112        cachedir.delete();
113        cachedir.mkdir();
114        long maxSize = SizeUtils.parseSizeInBytes(maxSizeStr);
115        long maxCount = Long.parseLong(maxCountStr);
116        long minAge = Long.parseLong(minAgeStr);
117        initializeCache(cachedir, maxSize, maxCount, minAge, fileStorage);
118        log.info("Using binary cache directory: " + cachedir.getPath() + " size: " + maxSizeStr + " maxCount: "
119                + maxCount + " minAge: " + minAge);
120
121        // be sure FileTracker won't steal our files !
122        FileEventTracker.registerProtectedPath(cachedir.getAbsolutePath());
123    }
124
125    @Override
126    public void close() {
127        fileCache.clear();
128        if (cachedir != null) {
129            try {
130                FileUtils.deleteDirectory(cachedir);
131            } catch (IOException e) {
132                throw new NuxeoException(e);
133            }
134        }
135    }
136
137    @Override
138    protected Binary getBinary(InputStream in) throws IOException {
139        // write the input stream to a temporary file, while computing a digest
140        File tmp = fileCache.getTempFile();
141        OutputStream out = new FileOutputStream(tmp);
142        String digest;
143        try {
144            digest = storeAndDigest(in, out);
145        } finally {
146            in.close();
147            out.close();
148        }
149        long length = tmp.length();
150
151        File cachedFile = fileCache.getFile(digest);
152        if (cachedFile != null) {
153            // file already in cache
154            if (Framework.isTestModeSet()) {
155                Framework.getProperties().setProperty("cachedBinary", digest);
156            }
157            // delete tmp file, not needed anymore
158            tmp.delete();
159        } else {
160            // send the file to storage
161            fileStorage.storeFile(digest, tmp);
162            // register the file in the file cache
163            fileCache.putFile(digest, tmp);
164        }
165        return new LazyBinary(digest, length, blobProviderId, this);
166    }
167
168    @Override
169    public Binary getBinary(String digest) {
170        return new LazyBinary(digest, blobProviderId, this);
171    }
172
173    /* =============== Methods used by LazyBinary =============== */
174
175    /**
176     * Gets a file from cache or storage.
177     * <p>
178     * Used by {@link LazyBinary}.
179     */
180    public File getFile(String digest) throws IOException {
181        // get file from cache
182        File file = fileCache.getFile(digest);
183        if (file != null) {
184            return file;
185        }
186        // fetch file from storage
187        File tmp = fileCache.getTempFile();
188        if (fileStorage.fetchFile(digest, tmp)) {
189            // put file in cache
190            file = fileCache.putFile(digest, tmp);
191            return file;
192        } else {
193            // file not in storage
194            tmp.delete();
195            return null;
196        }
197    }
198
199    /**
200     * Gets a file length from cache or storage.
201     * <p>
202     * Use by {@link LazyBinary}.
203     */
204    public Long getLength(String digest) throws IOException {
205        // get length from cache
206        Long length = getLengthFromCache(digest);
207        if (length != null) {
208            return length;
209        }
210        // fetch length from storage
211        length = fileStorage.fetchLength(digest);
212        // put length in cache
213        putLengthInCache(digest, length);
214        return length;
215    }
216
217    protected Long getLengthFromCache(String digest) throws IOException {
218        File f = fileCache.getFile(digest + LEN_DIGEST_SUFFIX);
219        if (f == null) {
220            return null;
221        }
222        // read decimal length from file
223        InputStream in = null;
224        try {
225            in = new FileInputStream(f);
226            String len = IOUtils.toString(in);
227            return Long.valueOf(len);
228        } catch (NumberFormatException e) {
229            throw new IOException("Invalid length in " + f, e);
230        } finally {
231            IOUtils.closeQuietly(in);
232        }
233    }
234
235    protected void putLengthInCache(String digest, Long len) throws IOException {
236        if (len == null) {
237            return;
238        }
239        // write decimal length in file
240        OutputStream out = null;
241        try {
242            File tmp = fileCache.getTempFile();
243            out = new FileOutputStream(tmp);
244            Writer writer = new OutputStreamWriter(out);
245            writer.write(len.toString());
246            writer.flush();
247            writer.close();
248            fileCache.putFile(digest + LEN_DIGEST_SUFFIX, tmp);
249        } finally {
250            IOUtils.closeQuietly(out);
251        }
252    }
253
254}