001/*
002 * (C) Copyright 2019 Nuxeo (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Florent Guillaume
018 */
019package org.nuxeo.ecm.core.blob;
020
021import static java.nio.file.StandardCopyOption.ATOMIC_MOVE;
022import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
023
024import java.io.BufferedInputStream;
025import java.io.File;
026import java.io.IOException;
027import java.io.InputStream;
028import java.io.RandomAccessFile;
029import java.nio.file.Files;
030import java.nio.file.NoSuchFileException;
031import java.nio.file.Path;
032
033import org.apache.logging.log4j.LogManager;
034import org.apache.logging.log4j.Logger;
035import org.nuxeo.ecm.core.api.NuxeoException;
036import org.nuxeo.ecm.core.blob.binary.BinaryGarbageCollector;
037import org.nuxeo.ecm.core.blob.binary.BinaryManagerStatus;
038
039/**
040 * Blob storage as files on a local filesystem. The actual storage path chosen for a given key is decided based on a
041 * {@link PathStrategy}.
042 *
043 * @since 11.1
044 */
045public class LocalBlobStore extends AbstractBlobStore {
046
047    private static final Logger log = LogManager.getLogger(LocalBlobStore.class);
048
049    protected final PathStrategy pathStrategy;
050
051    protected final LocalBlobGarbageCollector gc;
052
053    public LocalBlobStore(String name, KeyStrategy keyStrategy, PathStrategy pathStrategy) {
054        super(name, keyStrategy);
055        this.pathStrategy = pathStrategy;
056        gc = new LocalBlobGarbageCollector();
057    }
058
059    @Override
060    public String writeBlob(BlobWriteContext blobWriteContext) throws IOException {
061        Path tmp = pathStrategy.createTempFile();
062        try {
063            write(blobWriteContext, tmp);
064            logTrace("->", "write " + Files.size(tmp) + " bytes");
065            logTrace("hnote right: " + tmp.getFileName().toString());
066            String key = blobWriteContext.getKey(); // may depend on WriteObserver, for example for digests
067            Path dest = pathStrategy.getPathForKey(key);
068            Files.createDirectories(dest.getParent());
069            logTrace(name, "-->", name, "rename");
070            logTrace("hnote right of " + name + ": " + dest.getFileName().toString());
071            Files.move(tmp, dest, ATOMIC_MOVE);
072            return key;
073        } finally {
074            try {
075                Files.deleteIfExists(tmp);
076            } catch (IOException e) {
077                log.warn(e, e);
078            }
079        }
080    }
081
082    // overridden for encrypted storage
083    protected void write(BlobWriteContext blobWriteContext, Path file) throws IOException {
084        transfer(blobWriteContext, file);
085    }
086
087    @Override
088    public boolean copyBlobIsOptimized(BlobStore sourceStore) {
089        return sourceStore instanceof LocalBlobStore;
090    }
091
092    @Override
093    public boolean copyBlob(String key, BlobStore sourceStore, String sourceKey, boolean atomicMove)
094            throws IOException {
095        BlobStore unwrappedSourceStore = sourceStore.unwrap();
096        if (unwrappedSourceStore instanceof LocalBlobStore) {
097            LocalBlobStore sourceLocalBlobStore = (LocalBlobStore) unwrappedSourceStore;
098            return copyBlob(key, sourceLocalBlobStore, sourceKey, atomicMove);
099        } else {
100            return copyBlobGeneric(key, sourceStore, sourceKey, atomicMove);
101        }
102    }
103
104    /**
105     * Optimized file-to-file copy/move.
106     */
107    protected boolean copyBlob(String key, LocalBlobStore sourceStore, String sourceKey, boolean atomicMove)
108            throws IOException {
109        Path dest = pathStrategy.getPathForKey(key);
110        Files.createDirectories(dest.getParent());
111        Path source = sourceStore.pathStrategy.getPathForKey(sourceKey);
112        if (!Files.exists(source)) { // NOSONAR (squid:S3725)
113            return false;
114        }
115        if (atomicMove) {
116            logTrace("hnote right of " + sourceStore.name + ": " + sourceKey);
117            logTrace(sourceStore.name, "->", name, "move");
118            logTrace("hnote right: " + key);
119            PathStrategy.atomicMove(source, dest);
120        } else {
121            logTrace("hnote right of " + sourceStore.name + ": " + sourceKey);
122            logTrace(sourceStore.name, "->", name, "copy");
123            logTrace("hnote right: " + key);
124            Files.copy(source, dest, REPLACE_EXISTING);
125        }
126        return true;
127    }
128
129    /**
130     * Generic copy/move to a local file.
131     */
132    protected boolean copyBlobGeneric(String key, BlobStore sourceStore, String sourceKey, boolean atomicMove)
133            throws IOException {
134        Path dest = pathStrategy.getPathForKey(key);
135        Files.createDirectories(dest.getParent());
136        Path tmp = null;
137        try {
138            Path readTo;
139            if (atomicMove) {
140                readTo = tmp = pathStrategy.createTempFile();
141            } else {
142                readTo = dest;
143            }
144            OptionalOrUnknown<Path> fileOpt = sourceStore.getFile(sourceKey);
145            if (fileOpt.isPresent()) {
146                Files.copy(fileOpt.get(), readTo, REPLACE_EXISTING);
147            } else {
148                boolean found = sourceStore.readBlob(sourceKey, readTo);
149                if (!found) {
150                    return false;
151                }
152            }
153            if (atomicMove) {
154                Files.move(readTo, dest, ATOMIC_MOVE);
155                sourceStore.deleteBlob(sourceKey);
156            }
157            return true;
158        } finally {
159            if (tmp != null) {
160                try {
161                    Files.deleteIfExists(tmp);
162                } catch (IOException e) {
163                    log.warn(e, e);
164                }
165            }
166        }
167    }
168
169    @Override
170    public OptionalOrUnknown<Path> getFile(String key) {
171        return getStoredFile(key);
172    }
173
174    protected OptionalOrUnknown<Path> getStoredFile(String key) {
175        Path file = pathStrategy.getPathForKey(key);
176        return Files.exists(file) ? OptionalOrUnknown.of(file) : OptionalOrUnknown.missing(); // NOSONAR (squid:S3725)
177    }
178
179    @Override
180    public OptionalOrUnknown<InputStream> getStream(String key) throws IOException {
181        Path file = pathStrategy.getPathForKey(key);
182        try {
183            return OptionalOrUnknown.of(new BufferedInputStream(Files.newInputStream(file)));
184        } catch (NoSuchFileException e) {
185            return OptionalOrUnknown.missing();
186        }
187    }
188
189    @Override
190    public boolean readBlob(String key, Path dest) throws IOException {
191        Path file = pathStrategy.getPathForKey(key);
192        if (Files.exists(file)) { // NOSONAR (squid:S3725)
193            logTrace("<-", "read " + Files.size(file) + " bytes");
194            logTrace("hnote right: " + key);
195            Files.copy(file, dest, REPLACE_EXISTING);
196            return true;
197        } else {
198            logTrace("<--", "missing");
199            logTrace("hnote right: " + key);
200            return false;
201        }
202    }
203
204    @Override
205    public void deleteBlob(String key) {
206        Path file = pathStrategy.getPathForKey(key);
207        try {
208            logTrace("->", "delete");
209            logTrace("hnote right: " + key);
210            Files.deleteIfExists(file);
211        } catch (IOException e) {
212            log.warn(e, e);
213        }
214    }
215
216    @Override
217    public BinaryGarbageCollector getBinaryGarbageCollector() {
218        return gc;
219    }
220
221    public class LocalBlobGarbageCollector implements BinaryGarbageCollector {
222
223        /**
224         * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s.
225         */
226        public static final long TIME_RESOLUTION = 2000;
227
228        protected volatile long startTime;
229
230        protected BinaryManagerStatus status;
231
232        @Override
233        public String getId() {
234            return pathStrategy.dir.toUri().toString();
235        }
236
237        @Override
238        public BinaryManagerStatus getStatus() {
239            return status;
240        }
241
242        @Override
243        public boolean isInProgress() {
244            // volatile as this is designed to be called from another thread
245            return startTime != 0;
246        }
247
248        @Override
249        public void start() {
250            if (startTime != 0) {
251                throw new NuxeoException("Already started");
252            }
253            startTime = System.currentTimeMillis();
254            status = new BinaryManagerStatus();
255        }
256
257        @Override
258        public void mark(String key) {
259            OptionalOrUnknown<Path> fileOpt = getStoredFile(key);
260            if (!fileOpt.isPresent()) {
261                log.warn("Unknown blob for key: " + key);
262                return;
263            }
264            // mark the blob by touching the file
265            touch(fileOpt.get().toFile());
266        }
267
268        @Override
269        public void stop(boolean delete) {
270            if (startTime == 0) {
271                throw new NuxeoException("Not started");
272            }
273            deleteOld(pathStrategy.dir.toFile(), startTime - TIME_RESOLUTION, 0, delete);
274            status.gcDuration = System.currentTimeMillis() - startTime;
275            startTime = 0;
276        }
277
278        protected void deleteOld(File file, long minTime, int depth, boolean delete) {
279            if (file.isDirectory()) {
280                for (File f : file.listFiles()) {
281                    deleteOld(f, minTime, depth + 1, delete);
282                }
283                if (depth > 0 && file.list().length == 0) {
284                    // empty directory
285                    file.delete(); // NOSONAR
286                }
287            } else if (file.isFile() && file.canWrite()) {
288                long lastModified = file.lastModified();
289                long length = file.length();
290                if (lastModified == 0) {
291                    log.warn("Cannot read last modified for file: " + file);
292                } else if (lastModified < minTime) {
293                    status.sizeBinariesGC += length;
294                    status.numBinariesGC++;
295                    if (delete && !file.delete()) { // NOSONAR
296                        log.warn("Cannot gc file: " + file);
297                    }
298                } else {
299                    status.sizeBinaries += length;
300                    status.numBinaries++;
301                }
302            }
303        }
304
305        /** Sets the last modification date to now on a file. */
306        protected void touch(File file) {
307            long time = System.currentTimeMillis();
308            if (file.setLastModified(time)) {
309                // ok
310                return;
311            }
312            if (!file.canWrite()) {
313                // cannot write -> stop won't be able to delete anyway
314                return;
315            }
316            try {
317                // Windows: the file may be open for reading
318                // workaround found by Thomas Mueller, see JCR-2872
319                try (RandomAccessFile r = new RandomAccessFile(file, "rw")) {
320                    r.setLength(r.length());
321                }
322            } catch (IOException e) {
323                log.warn("Cannot set last modified for file: " + file, e);
324            }
325        }
326
327    }
328
329}