001/* 002 * (C) Copyright 2019 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 */ 019package org.nuxeo.ecm.core.blob; 020 021import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; 022import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; 023 024import java.io.BufferedInputStream; 025import java.io.File; 026import java.io.IOException; 027import java.io.InputStream; 028import java.io.RandomAccessFile; 029import java.nio.file.Files; 030import java.nio.file.NoSuchFileException; 031import java.nio.file.Path; 032 033import org.apache.logging.log4j.LogManager; 034import org.apache.logging.log4j.Logger; 035import org.nuxeo.ecm.core.api.NuxeoException; 036import org.nuxeo.ecm.core.blob.binary.BinaryGarbageCollector; 037import org.nuxeo.ecm.core.blob.binary.BinaryManagerStatus; 038 039/** 040 * Blob storage as files on a local filesystem. The actual storage path chosen for a given key is decided based on a 041 * {@link PathStrategy}. 042 * 043 * @since 11.1 044 */ 045public class LocalBlobStore extends AbstractBlobStore { 046 047 private static final Logger log = LogManager.getLogger(LocalBlobStore.class); 048 049 protected final PathStrategy pathStrategy; 050 051 protected final LocalBlobGarbageCollector gc; 052 053 public LocalBlobStore(String name, KeyStrategy keyStrategy, PathStrategy pathStrategy) { 054 super(name, keyStrategy); 055 this.pathStrategy = pathStrategy; 056 gc = new LocalBlobGarbageCollector(); 057 } 058 059 @Override 060 public String writeBlob(BlobWriteContext blobWriteContext) throws IOException { 061 Path tmp = pathStrategy.createTempFile(); 062 try { 063 write(blobWriteContext, tmp); 064 logTrace("->", "write " + Files.size(tmp) + " bytes"); 065 logTrace("hnote right: " + tmp.getFileName().toString()); 066 String key = blobWriteContext.getKey(); // may depend on WriteObserver, for example for digests 067 Path dest = pathStrategy.getPathForKey(key); 068 Files.createDirectories(dest.getParent()); 069 logTrace(name, "-->", name, "rename"); 070 logTrace("hnote right of " + name + ": " + dest.getFileName().toString()); 071 Files.move(tmp, dest, ATOMIC_MOVE); 072 return key; 073 } finally { 074 try { 075 Files.deleteIfExists(tmp); 076 } catch (IOException e) { 077 log.warn(e, e); 078 } 079 } 080 } 081 082 // overridden for encrypted storage 083 protected void write(BlobWriteContext blobWriteContext, Path file) throws IOException { 084 transfer(blobWriteContext, file); 085 } 086 087 @Override 088 public boolean copyBlobIsOptimized(BlobStore sourceStore) { 089 return sourceStore instanceof LocalBlobStore; 090 } 091 092 @Override 093 public boolean copyBlob(String key, BlobStore sourceStore, String sourceKey, boolean atomicMove) 094 throws IOException { 095 BlobStore unwrappedSourceStore = sourceStore.unwrap(); 096 if (unwrappedSourceStore instanceof LocalBlobStore) { 097 LocalBlobStore sourceLocalBlobStore = (LocalBlobStore) unwrappedSourceStore; 098 return copyBlob(key, sourceLocalBlobStore, sourceKey, atomicMove); 099 } else { 100 return copyBlobGeneric(key, sourceStore, sourceKey, atomicMove); 101 } 102 } 103 104 /** 105 * Optimized file-to-file copy/move. 106 */ 107 protected boolean copyBlob(String key, LocalBlobStore sourceStore, String sourceKey, boolean atomicMove) 108 throws IOException { 109 Path dest = pathStrategy.getPathForKey(key); 110 Files.createDirectories(dest.getParent()); 111 Path source = sourceStore.pathStrategy.getPathForKey(sourceKey); 112 if (!Files.exists(source)) { // NOSONAR (squid:S3725) 113 return false; 114 } 115 if (atomicMove) { 116 logTrace("hnote right of " + sourceStore.name + ": " + sourceKey); 117 logTrace(sourceStore.name, "->", name, "move"); 118 logTrace("hnote right: " + key); 119 PathStrategy.atomicMove(source, dest); 120 } else { 121 logTrace("hnote right of " + sourceStore.name + ": " + sourceKey); 122 logTrace(sourceStore.name, "->", name, "copy"); 123 logTrace("hnote right: " + key); 124 Files.copy(source, dest, REPLACE_EXISTING); 125 } 126 return true; 127 } 128 129 /** 130 * Generic copy/move to a local file. 131 */ 132 protected boolean copyBlobGeneric(String key, BlobStore sourceStore, String sourceKey, boolean atomicMove) 133 throws IOException { 134 Path dest = pathStrategy.getPathForKey(key); 135 Files.createDirectories(dest.getParent()); 136 Path tmp = null; 137 try { 138 Path readTo; 139 if (atomicMove) { 140 readTo = tmp = pathStrategy.createTempFile(); 141 } else { 142 readTo = dest; 143 } 144 OptionalOrUnknown<Path> fileOpt = sourceStore.getFile(sourceKey); 145 if (fileOpt.isPresent()) { 146 Files.copy(fileOpt.get(), readTo, REPLACE_EXISTING); 147 } else { 148 boolean found = sourceStore.readBlob(sourceKey, readTo); 149 if (!found) { 150 return false; 151 } 152 } 153 if (atomicMove) { 154 Files.move(readTo, dest, ATOMIC_MOVE); 155 sourceStore.deleteBlob(sourceKey); 156 } 157 return true; 158 } finally { 159 if (tmp != null) { 160 try { 161 Files.deleteIfExists(tmp); 162 } catch (IOException e) { 163 log.warn(e, e); 164 } 165 } 166 } 167 } 168 169 @Override 170 public OptionalOrUnknown<Path> getFile(String key) { 171 return getStoredFile(key); 172 } 173 174 protected OptionalOrUnknown<Path> getStoredFile(String key) { 175 Path file = pathStrategy.getPathForKey(key); 176 return Files.exists(file) ? OptionalOrUnknown.of(file) : OptionalOrUnknown.missing(); // NOSONAR (squid:S3725) 177 } 178 179 @Override 180 public OptionalOrUnknown<InputStream> getStream(String key) throws IOException { 181 Path file = pathStrategy.getPathForKey(key); 182 try { 183 return OptionalOrUnknown.of(new BufferedInputStream(Files.newInputStream(file))); 184 } catch (NoSuchFileException e) { 185 return OptionalOrUnknown.missing(); 186 } 187 } 188 189 @Override 190 public boolean readBlob(String key, Path dest) throws IOException { 191 Path file = pathStrategy.getPathForKey(key); 192 if (Files.exists(file)) { // NOSONAR (squid:S3725) 193 logTrace("<-", "read " + Files.size(file) + " bytes"); 194 logTrace("hnote right: " + key); 195 Files.copy(file, dest, REPLACE_EXISTING); 196 return true; 197 } else { 198 logTrace("<--", "missing"); 199 logTrace("hnote right: " + key); 200 return false; 201 } 202 } 203 204 @Override 205 public void deleteBlob(String key) { 206 Path file = pathStrategy.getPathForKey(key); 207 try { 208 logTrace("->", "delete"); 209 logTrace("hnote right: " + key); 210 Files.deleteIfExists(file); 211 } catch (IOException e) { 212 log.warn(e, e); 213 } 214 } 215 216 @Override 217 public BinaryGarbageCollector getBinaryGarbageCollector() { 218 return gc; 219 } 220 221 public class LocalBlobGarbageCollector implements BinaryGarbageCollector { 222 223 /** 224 * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s. 225 */ 226 public static final long TIME_RESOLUTION = 2000; 227 228 protected volatile long startTime; 229 230 protected BinaryManagerStatus status; 231 232 @Override 233 public String getId() { 234 return pathStrategy.dir.toUri().toString(); 235 } 236 237 @Override 238 public BinaryManagerStatus getStatus() { 239 return status; 240 } 241 242 @Override 243 public boolean isInProgress() { 244 // volatile as this is designed to be called from another thread 245 return startTime != 0; 246 } 247 248 @Override 249 public void start() { 250 if (startTime != 0) { 251 throw new NuxeoException("Already started"); 252 } 253 startTime = System.currentTimeMillis(); 254 status = new BinaryManagerStatus(); 255 } 256 257 @Override 258 public void mark(String key) { 259 OptionalOrUnknown<Path> fileOpt = getStoredFile(key); 260 if (!fileOpt.isPresent()) { 261 log.warn("Unknown blob for key: " + key); 262 return; 263 } 264 // mark the blob by touching the file 265 touch(fileOpt.get().toFile()); 266 } 267 268 @Override 269 public void stop(boolean delete) { 270 if (startTime == 0) { 271 throw new NuxeoException("Not started"); 272 } 273 deleteOld(pathStrategy.dir.toFile(), startTime - TIME_RESOLUTION, 0, delete); 274 status.gcDuration = System.currentTimeMillis() - startTime; 275 startTime = 0; 276 } 277 278 protected void deleteOld(File file, long minTime, int depth, boolean delete) { 279 if (file.isDirectory()) { 280 for (File f : file.listFiles()) { 281 deleteOld(f, minTime, depth + 1, delete); 282 } 283 if (depth > 0 && file.list().length == 0) { 284 // empty directory 285 file.delete(); // NOSONAR 286 } 287 } else if (file.isFile() && file.canWrite()) { 288 long lastModified = file.lastModified(); 289 long length = file.length(); 290 if (lastModified == 0) { 291 log.warn("Cannot read last modified for file: " + file); 292 } else if (lastModified < minTime) { 293 status.sizeBinariesGC += length; 294 status.numBinariesGC++; 295 if (delete && !file.delete()) { // NOSONAR 296 log.warn("Cannot gc file: " + file); 297 } 298 } else { 299 status.sizeBinaries += length; 300 status.numBinaries++; 301 } 302 } 303 } 304 305 /** Sets the last modification date to now on a file. */ 306 protected void touch(File file) { 307 long time = System.currentTimeMillis(); 308 if (file.setLastModified(time)) { 309 // ok 310 return; 311 } 312 if (!file.canWrite()) { 313 // cannot write -> stop won't be able to delete anyway 314 return; 315 } 316 try { 317 // Windows: the file may be open for reading 318 // workaround found by Thomas Mueller, see JCR-2872 319 try (RandomAccessFile r = new RandomAccessFile(file, "rw")) { 320 r.setLength(r.length()); 321 } 322 } catch (IOException e) { 323 log.warn("Cannot set last modified for file: " + file, e); 324 } 325 } 326 327 } 328 329}