001/*
002 * (C) Copyright 2006-2018 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 *     Mathieu Guillaume
019 *     jcarsique
020 */
021
022package org.nuxeo.ecm.core.blob.binary;
023
024import java.io.File;
025import java.io.FileInputStream;
026import java.io.FileOutputStream;
027import java.io.IOException;
028import java.io.InputStream;
029import java.io.OutputStream;
030import java.io.RandomAccessFile;
031import java.util.Map;
032import java.util.regex.Pattern;
033
034import org.apache.commons.io.FileUtils;
035import org.apache.commons.io.IOUtils;
036import org.apache.commons.logging.Log;
037import org.apache.commons.logging.LogFactory;
038import org.nuxeo.ecm.core.api.NuxeoException;
039import org.nuxeo.ecm.core.blob.LocalBlobStoreConfiguration;
040import org.nuxeo.runtime.trackers.files.FileEventTracker;
041
042/**
043 * A simple filesystem-based binary manager. It stores the binaries according to their digest (hash), which means that
044 * no transactional behavior needs to be implemented.
045 * <p>
046 * A garbage collection is needed to purge unused binaries.
047 * <p>
048 * The format of the <em>binaries</em> directory is:
049 * <ul>
050 * <li><em>data/</em> hierarchy with the actual binaries in subdirectories,</li>
051 * <li><em>tmp/</em> temporary storage during creation,</li>
052 * <li><em>config.xml</em> a file containing the configuration used.</li>
053 * </ul>
054 *
055 * @author Florent Guillaume
056 * @since 5.6
057 */
058public class LocalBinaryManager extends AbstractBinaryManager {
059
060    private static final Log log = LogFactory.getLog(LocalBinaryManager.class);
061
062    /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */
063    @Deprecated
064    public static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("[a-zA-Z]:[/\\\\].*");
065
066    /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */
067    @Deprecated
068    public static final String DEFAULT_PATH = "binaries";
069
070    /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */
071    @Deprecated
072    public static final String DATA = "data";
073
074    /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */
075    @Deprecated
076    public static final String TMP = "tmp";
077
078    /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */
079    @Deprecated
080    public static final String CONFIG_FILE = "config.xml";
081
082    protected File storageDir;
083
084    protected File tmpDir;
085
086    @Override
087    public void initialize(String blobProviderId, Map<String, String> properties) throws IOException {
088        super.initialize(blobProviderId, properties);
089        LocalBlobStoreConfiguration config = new LocalBlobStoreConfiguration(properties);
090
091        log.info("Registering binary manager '" + blobProviderId + "' using "
092                + (this.getClass().equals(LocalBinaryManager.class) ? "" : (this.getClass().getSimpleName() + " and "))
093                + "binary store: " + config.storageDir.getParent());
094        storageDir = config.storageDir.toFile();
095        tmpDir = config.tmpDir.toFile();
096        storageDir.mkdirs();
097        tmpDir.mkdirs();
098        setDescriptor(config.descriptor);
099        createGarbageCollector();
100
101        // be sure FileTracker won't steal our files !
102        FileEventTracker.registerProtectedPath(storageDir.getAbsolutePath());
103    }
104
105    @Override
106    public void close() {
107        if (tmpDir != null) {
108            try {
109                FileUtils.cleanDirectory(tmpDir);
110            } catch (IOException e) {
111                throw new NuxeoException(e);
112            }
113        }
114    }
115
116    public File getStorageDir() {
117        return storageDir;
118    }
119
120    @Override
121    protected Binary getBinary(InputStream in) throws IOException {
122        String digest = storeAndDigest(in);
123        File file = getFileForDigest(digest, false);
124        /*
125         * Now we can build the Binary.
126         */
127        return new Binary(file, digest, blobProviderId);
128    }
129
130    @Override
131    public Binary getBinary(String digest) {
132        File file = getFileForDigest(digest, false);
133        if (file == null) {
134            // invalid digest
135            return null;
136        }
137        if (!file.exists()) {
138            log.warn("cannot fetch content at " + file.getPath() + " (file does not exist), check your configuration");
139            return null;
140        }
141        return new Binary(file, digest, blobProviderId);
142    }
143
144    /**
145     * Gets a file representing the storage for a given digest.
146     *
147     * @param digest the digest
148     * @param createDir {@code true} if the directory containing the file itself must be created
149     * @return the file for this digest
150     */
151    public File getFileForDigest(String digest, boolean createDir) {
152        int depth = descriptor.depth;
153        if (digest.length() < 2 * depth) {
154            return null;
155        }
156        StringBuilder sb = new StringBuilder(3 * depth - 1);
157        for (int i = 0; i < depth; i++) {
158            if (i != 0) {
159                sb.append(File.separatorChar);
160            }
161            sb.append(digest.substring(2 * i, 2 * i + 2));
162        }
163        File dir = new File(storageDir, sb.toString());
164        if (createDir) {
165            dir.mkdirs();
166        }
167        return new File(dir, digest);
168    }
169
170    protected String storeAndDigest(InputStream in) throws IOException {
171        File tmp = File.createTempFile("create_", ".tmp", tmpDir);
172        /*
173         * First, write the input stream to a temporary file, while computing a digest.
174         */
175        try {
176            String digest;
177            try (OutputStream out = new FileOutputStream(tmp)) {
178                digest = storeAndDigest(in, out);
179            } finally {
180                in.close();
181            }
182            /*
183             * Move the tmp file to its destination.
184             */
185            File file = getFileForDigest(digest, true);
186            atomicMove(tmp, file);
187            return digest;
188        } finally {
189            tmp.delete();
190        }
191
192    }
193
194    /**
195     * Does an atomic move of the tmp (or source) file to the final file.
196     * <p>
197     * Tries to work well with NFS mounts and different filesystems.
198     */
199    protected void atomicMove(File source, File dest) throws IOException {
200        if (dest.exists()) {
201            // The file with the proper digest is already there so don't do
202            // anything. This is to avoid "Stale NFS File Handle" problems
203            // which would occur if we tried to overwrite it anyway.
204            // Note that this doesn't try to protect from the case where
205            // two identical files are uploaded at the same time.
206            // Update date for the GC.
207            dest.setLastModified(source.lastModified());
208            return;
209        }
210        if (!source.renameTo(dest)) {
211            // Failed to rename, probably a different filesystem.
212            // Do *NOT* use Apache Commons IO's FileUtils.moveFile()
213            // because it rewrites the destination file so is not atomic.
214            // Do a copy through a tmp file on the same filesystem then
215            // atomic rename.
216            File tmp = File.createTempFile(dest.getName(), ".tmp", dest.getParentFile());
217            try {
218                try (InputStream in = new FileInputStream(source); //
219                        OutputStream out = new FileOutputStream(tmp)) {
220                    IOUtils.copy(in, out);
221                }
222                // then do the atomic rename
223                tmp.renameTo(dest);
224            } finally {
225                tmp.delete();
226            }
227            // finally remove the original source
228            source.delete();
229        }
230        if (!dest.exists()) {
231            throw new IOException("Could not create file: " + dest);
232        }
233    }
234
235    protected void createGarbageCollector() {
236        garbageCollector = new DefaultBinaryGarbageCollector(this);
237    }
238
239    public static class DefaultBinaryGarbageCollector implements BinaryGarbageCollector {
240
241        /**
242         * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s.
243         */
244        public static final int TIME_RESOLUTION = 2000;
245
246        protected final LocalBinaryManager binaryManager;
247
248        protected volatile long startTime;
249
250        protected BinaryManagerStatus status;
251
252        public DefaultBinaryGarbageCollector(LocalBinaryManager binaryManager) {
253            this.binaryManager = binaryManager;
254        }
255
256        @Override
257        public String getId() {
258            return binaryManager.getStorageDir().toURI().toString();
259        }
260
261        @Override
262        public BinaryManagerStatus getStatus() {
263            return status;
264        }
265
266        @Override
267        public boolean isInProgress() {
268            // volatile as this is designed to be called from another thread
269            return startTime != 0;
270        }
271
272        @Override
273        public void start() {
274            if (startTime != 0) {
275                throw new RuntimeException("Alread started");
276            }
277            startTime = System.currentTimeMillis();
278            status = new BinaryManagerStatus();
279        }
280
281        @Override
282        public void mark(String digest) {
283            File file = binaryManager.getFileForDigest(digest, false);
284            if (!file.exists()) {
285                log.error("Unknown file digest: " + digest);
286                return;
287            }
288            touch(file);
289        }
290
291        @Override
292        public void stop(boolean delete) {
293            if (startTime == 0) {
294                throw new RuntimeException("Not started");
295            }
296            deleteOld(binaryManager.getStorageDir(), startTime - TIME_RESOLUTION, 0, delete);
297            status.gcDuration = System.currentTimeMillis() - startTime;
298            startTime = 0;
299        }
300
301        protected void deleteOld(File file, long minTime, int depth, boolean delete) {
302            if (file.isDirectory()) {
303                for (File f : file.listFiles()) {
304                    deleteOld(f, minTime, depth + 1, delete);
305                }
306                if (depth > 0 && file.list().length == 0) {
307                    // empty directory
308                    file.delete();
309                }
310            } else if (file.isFile() && file.canWrite()) {
311                long lastModified = file.lastModified();
312                long length = file.length();
313                if (lastModified == 0) {
314                    log.error("Cannot read last modified for file: " + file);
315                } else if (lastModified < minTime) {
316                    status.sizeBinariesGC += length;
317                    status.numBinariesGC++;
318                    if (delete && !file.delete()) {
319                        log.warn("Cannot gc file: " + file);
320                    }
321                } else {
322                    status.sizeBinaries += length;
323                    status.numBinaries++;
324                }
325            }
326        }
327    }
328
329    /**
330     * Sets the last modification date to now on a file
331     *
332     * @param file the file
333     */
334    public static void touch(File file) {
335        long time = System.currentTimeMillis();
336        if (file.setLastModified(time)) {
337            // ok
338            return;
339        }
340        if (!file.canWrite()) {
341            // cannot write -> stop won't be able to delete anyway
342            return;
343        }
344        try {
345            // Windows: the file may be open for reading
346            // workaround found by Thomas Mueller, see JCR-2872
347            try (RandomAccessFile r = new RandomAccessFile(file, "rw")) {
348                r.setLength(r.length());
349            }
350        } catch (IOException e) {
351            log.error("Cannot set last modified for file: " + file, e);
352        }
353    }
354
355}