001/*
002 * Copyright (c) 2006-2012 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the Eclipse Public License v1.0
006 * which accompanies this distribution, and is available at
007 * http://www.eclipse.org/legal/epl-v10.html
008 *
009 * Contributors:
010 *     Florent Guillaume, Mathieu Guillaume, jcarsique
011 */
012
013package org.nuxeo.ecm.core.blob.binary;
014
015import java.io.File;
016import java.io.FileInputStream;
017import java.io.FileOutputStream;
018import java.io.IOException;
019import java.io.InputStream;
020import java.io.OutputStream;
021import java.io.RandomAccessFile;
022import java.util.Map;
023import java.util.regex.Pattern;
024
025import org.apache.commons.io.FileUtils;
026import org.apache.commons.io.IOUtils;
027import org.apache.commons.lang.StringUtils;
028import org.apache.commons.logging.Log;
029import org.apache.commons.logging.LogFactory;
030import org.nuxeo.common.Environment;
031import org.nuxeo.ecm.core.api.NuxeoException;
032import org.nuxeo.runtime.api.Framework;
033import org.nuxeo.runtime.trackers.files.FileEventTracker;
034
035/**
036 * A simple filesystem-based binary manager. It stores the binaries according to their digest (hash), which means that
037 * no transactional behavior needs to be implemented.
038 * <p>
039 * A garbage collection is needed to purge unused binaries.
040 * <p>
041 * The format of the <em>binaries</em> directory is:
042 * <ul>
043 * <li><em>data/</em> hierarchy with the actual binaries in subdirectories,</li>
044 * <li><em>tmp/</em> temporary storage during creation,</li>
045 * <li><em>config.xml</em> a file containing the configuration used.</li>
046 * </ul>
047 *
048 * @author Florent Guillaume
049 * @since 5.6
050 */
051public class LocalBinaryManager extends AbstractBinaryManager {
052
053    private static final Log log = LogFactory.getLog(LocalBinaryManager.class);
054
055    public static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("[a-zA-Z]:[/\\\\].*");
056
057    public static final String DEFAULT_PATH = "binaries";
058
059    public static final String DATA = "data";
060
061    public static final String TMP = "tmp";
062
063    public static final String CONFIG_FILE = "config.xml";
064
065    protected File storageDir;
066
067    protected File tmpDir;
068
069    @Override
070    public void initialize(String blobProviderId, Map<String, String> properties) throws IOException {
071        super.initialize(blobProviderId, properties);
072        String path = properties.get(BinaryManager.PROP_PATH);
073        if (StringUtils.isBlank(path)) {
074            path = DEFAULT_PATH;
075        }
076        path = Framework.expandVars(path);
077        path = path.trim();
078        File base;
079        if (path.startsWith("/") || path.startsWith("\\") || path.contains("://") || path.contains(":\\")
080                || WINDOWS_ABSOLUTE_PATH.matcher(path).matches()) {
081            // absolute
082            base = new File(path);
083        } else {
084            // relative
085            File home = Environment.getDefault().getData();
086            base = new File(home, path);
087
088            // Backward compliance with versions before 5.4 (NXP-5370)
089            File oldBase = new File(Framework.getRuntime().getHome().getPath(), path);
090            if (oldBase.exists()) {
091                log.warn("Old binaries path used (NXP-5370). Please move " + oldBase + " to " + base);
092                base = oldBase;
093            }
094        }
095
096        // be sure FileTracker won't steal our files !
097        FileEventTracker.registerProtectedPath(base.getAbsolutePath());
098
099        log.info("Registering binary manager '" + blobProviderId + "' using "
100                + (this.getClass().equals(LocalBinaryManager.class) ? "" : (this.getClass().getSimpleName() + " and "))
101                + "binary store: " + base);
102        storageDir = new File(base, DATA);
103        tmpDir = new File(base, TMP);
104        storageDir.mkdirs();
105        tmpDir.mkdirs();
106        descriptor = getDescriptor(new File(base, CONFIG_FILE));
107        createGarbageCollector();
108    }
109
110    @Override
111    public void close() {
112        if (tmpDir != null) {
113            try {
114                FileUtils.cleanDirectory(tmpDir);
115            } catch (IOException e) {
116                throw new NuxeoException(e);
117            }
118        }
119    }
120
121    public File getStorageDir() {
122        return storageDir;
123    }
124
125    @Override
126    protected Binary getBinary(InputStream in) throws IOException {
127        String digest = storeAndDigest(in);
128        File file = getFileForDigest(digest, false);
129        /*
130         * Now we can build the Binary.
131         */
132        return new Binary(file, digest, blobProviderId);
133    }
134
135    @Override
136    public Binary getBinary(String digest) {
137        File file = getFileForDigest(digest, false);
138        if (file == null) {
139            // invalid digest
140            return null;
141        }
142        if (!file.exists()) {
143            log.warn("cannot fetch content at " + file.getPath() + " (file does not exist), check your configuration");
144            return null;
145        }
146        return new Binary(file, digest, blobProviderId);
147    }
148
149    /**
150     * Gets a file representing the storage for a given digest.
151     *
152     * @param digest the digest
153     * @param createDir {@code true} if the directory containing the file itself must be created
154     * @return the file for this digest
155     */
156    public File getFileForDigest(String digest, boolean createDir) {
157        int depth = descriptor.depth;
158        if (digest.length() < 2 * depth) {
159            return null;
160        }
161        StringBuilder buf = new StringBuilder(3 * depth - 1);
162        for (int i = 0; i < depth; i++) {
163            if (i != 0) {
164                buf.append(File.separatorChar);
165            }
166            buf.append(digest.substring(2 * i, 2 * i + 2));
167        }
168        File dir = new File(storageDir, buf.toString());
169        if (createDir) {
170            dir.mkdirs();
171        }
172        return new File(dir, digest);
173    }
174
175    protected String storeAndDigest(InputStream in) throws IOException {
176        File tmp = File.createTempFile("create_", ".tmp", tmpDir);
177        OutputStream out = new FileOutputStream(tmp);
178        /*
179         * First, write the input stream to a temporary file, while computing a digest.
180         */
181        try {
182            String digest;
183            try {
184                digest = storeAndDigest(in, out);
185            } finally {
186                in.close();
187                out.close();
188            }
189            /*
190             * Move the tmp file to its destination.
191             */
192            File file = getFileForDigest(digest, true);
193            atomicMove(tmp, file);
194            return digest;
195        } finally {
196            tmp.delete();
197        }
198
199    }
200
201    /**
202     * Does an atomic move of the tmp (or source) file to the final file.
203     * <p>
204     * Tries to work well with NFS mounts and different filesystems.
205     */
206    protected void atomicMove(File source, File dest) throws IOException {
207        if (dest.exists()) {
208            // The file with the proper digest is already there so don't do
209            // anything. This is to avoid "Stale NFS File Handle" problems
210            // which would occur if we tried to overwrite it anyway.
211            // Note that this doesn't try to protect from the case where
212            // two identical files are uploaded at the same time.
213            // Update date for the GC.
214            dest.setLastModified(source.lastModified());
215            return;
216        }
217        if (!source.renameTo(dest)) {
218            // Failed to rename, probably a different filesystem.
219            // Do *NOT* use Apache Commons IO's FileUtils.moveFile()
220            // because it rewrites the destination file so is not atomic.
221            // Do a copy through a tmp file on the same filesystem then
222            // atomic rename.
223            File tmp = File.createTempFile(dest.getName(), ".tmp", dest.getParentFile());
224            try {
225                try (InputStream in = new FileInputStream(source); //
226                        OutputStream out = new FileOutputStream(tmp)) {
227                    IOUtils.copy(in, out);
228                }
229                // then do the atomic rename
230                tmp.renameTo(dest);
231            } finally {
232                tmp.delete();
233            }
234            // finally remove the original source
235            source.delete();
236        }
237        if (!dest.exists()) {
238            throw new IOException("Could not create file: " + dest);
239        }
240    }
241
242    protected void createGarbageCollector() {
243        garbageCollector = new DefaultBinaryGarbageCollector(this);
244    }
245
246    public static class DefaultBinaryGarbageCollector implements BinaryGarbageCollector {
247
248        /**
249         * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s.
250         */
251        public static int TIME_RESOLUTION = 2000;
252
253        protected final LocalBinaryManager binaryManager;
254
255        protected volatile long startTime;
256
257        protected BinaryManagerStatus status;
258
259        public DefaultBinaryGarbageCollector(LocalBinaryManager binaryManager) {
260            this.binaryManager = binaryManager;
261        }
262
263        @Override
264        public String getId() {
265            return binaryManager.getStorageDir().toURI().toString();
266        }
267
268        @Override
269        public BinaryManagerStatus getStatus() {
270            return status;
271        }
272
273        @Override
274        public boolean isInProgress() {
275            // volatile as this is designed to be called from another thread
276            return startTime != 0;
277        }
278
279        @Override
280        public void start() {
281            if (startTime != 0) {
282                throw new RuntimeException("Alread started");
283            }
284            startTime = System.currentTimeMillis();
285            status = new BinaryManagerStatus();
286        }
287
288        @Override
289        public void mark(String digest) {
290            File file = binaryManager.getFileForDigest(digest, false);
291            if (!file.exists()) {
292                log.error("Unknown file digest: " + digest);
293                return;
294            }
295            touch(file);
296        }
297
298        @Override
299        public void stop(boolean delete) {
300            if (startTime == 0) {
301                throw new RuntimeException("Not started");
302            }
303            deleteOld(binaryManager.getStorageDir(), startTime - TIME_RESOLUTION, 0, delete);
304            status.gcDuration = System.currentTimeMillis() - startTime;
305            startTime = 0;
306        }
307
308        protected void deleteOld(File file, long minTime, int depth, boolean delete) {
309            if (file.isDirectory()) {
310                for (File f : file.listFiles()) {
311                    deleteOld(f, minTime, depth + 1, delete);
312                }
313                if (depth > 0 && file.list().length == 0) {
314                    // empty directory
315                    file.delete();
316                }
317            } else if (file.isFile() && file.canWrite()) {
318                long lastModified = file.lastModified();
319                long length = file.length();
320                if (lastModified == 0) {
321                    log.error("Cannot read last modified for file: " + file);
322                } else if (lastModified < minTime) {
323                    status.sizeBinariesGC += length;
324                    status.numBinariesGC++;
325                    if (delete && !file.delete()) {
326                        log.warn("Cannot gc file: " + file);
327                    }
328                } else {
329                    status.sizeBinaries += length;
330                    status.numBinaries++;
331                }
332            }
333        }
334    }
335
336    /**
337     * Sets the last modification date to now on a file
338     *
339     * @param file the file
340     */
341    public static void touch(File file) {
342        long time = System.currentTimeMillis();
343        if (file.setLastModified(time)) {
344            // ok
345            return;
346        }
347        if (!file.canWrite()) {
348            // cannot write -> stop won't be able to delete anyway
349            return;
350        }
351        try {
352            // Windows: the file may be open for reading
353            // workaround found by Thomas Mueller, see JCR-2872
354            RandomAccessFile r = new RandomAccessFile(file, "rw");
355            try {
356                r.setLength(r.length());
357            } finally {
358                r.close();
359            }
360        } catch (IOException e) {
361            log.error("Cannot set last modified for file: " + file, e);
362        }
363    }
364
365}