001/* 002 * (C) Copyright 2006-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 * Mathieu Guillaume 019 * jcarsique 020 */ 021 022package org.nuxeo.ecm.core.blob.binary; 023 024import java.io.File; 025import java.io.FileInputStream; 026import java.io.FileOutputStream; 027import java.io.IOException; 028import java.io.InputStream; 029import java.io.OutputStream; 030import java.io.RandomAccessFile; 031import java.util.Map; 032import java.util.regex.Pattern; 033 034import org.apache.commons.io.FileUtils; 035import org.apache.commons.io.IOUtils; 036import org.apache.commons.logging.Log; 037import org.apache.commons.logging.LogFactory; 038import org.nuxeo.ecm.core.api.NuxeoException; 039import org.nuxeo.ecm.core.blob.LocalBlobStoreConfiguration; 040import org.nuxeo.runtime.trackers.files.FileEventTracker; 041 042/** 043 * A simple filesystem-based binary manager. It stores the binaries according to their digest (hash), which means that 044 * no transactional behavior needs to be implemented. 045 * <p> 046 * A garbage collection is needed to purge unused binaries. 047 * <p> 048 * The format of the <em>binaries</em> directory is: 049 * <ul> 050 * <li><em>data/</em> hierarchy with the actual binaries in subdirectories,</li> 051 * <li><em>tmp/</em> temporary storage during creation,</li> 052 * <li><em>config.xml</em> a file containing the configuration used.</li> 053 * </ul> 054 * 055 * @author Florent Guillaume 056 * @since 5.6 057 */ 058public class LocalBinaryManager extends AbstractBinaryManager { 059 060 private static final Log log = LogFactory.getLog(LocalBinaryManager.class); 061 062 /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */ 063 @Deprecated 064 public static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("[a-zA-Z]:[/\\\\].*"); 065 066 /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */ 067 @Deprecated 068 public static final String DEFAULT_PATH = "binaries"; 069 070 /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */ 071 @Deprecated 072 public static final String DATA = "data"; 073 074 /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */ 075 @Deprecated 076 public static final String TMP = "tmp"; 077 078 /** @deprecated since 11.1, use {@link LocalBlobStoreConfiguration} instead */ 079 @Deprecated 080 public static final String CONFIG_FILE = "config.xml"; 081 082 protected File storageDir; 083 084 protected File tmpDir; 085 086 @Override 087 public void initialize(String blobProviderId, Map<String, String> properties) throws IOException { 088 super.initialize(blobProviderId, properties); 089 LocalBlobStoreConfiguration config = new LocalBlobStoreConfiguration(properties); 090 091 log.info("Registering binary manager '" + blobProviderId + "' using " 092 + (this.getClass().equals(LocalBinaryManager.class) ? "" : (this.getClass().getSimpleName() + " and ")) 093 + "binary store: " + config.storageDir.getParent()); 094 storageDir = config.storageDir.toFile(); 095 tmpDir = config.tmpDir.toFile(); 096 storageDir.mkdirs(); 097 tmpDir.mkdirs(); 098 setDescriptor(config.descriptor); 099 createGarbageCollector(); 100 101 // be sure FileTracker won't steal our files ! 102 FileEventTracker.registerProtectedPath(storageDir.getAbsolutePath()); 103 } 104 105 @Override 106 public void close() { 107 if (tmpDir != null) { 108 try { 109 FileUtils.cleanDirectory(tmpDir); 110 } catch (IOException e) { 111 throw new NuxeoException(e); 112 } 113 } 114 } 115 116 public File getStorageDir() { 117 return storageDir; 118 } 119 120 @Override 121 protected Binary getBinary(InputStream in) throws IOException { 122 String digest = storeAndDigest(in); 123 File file = getFileForDigest(digest, false); 124 /* 125 * Now we can build the Binary. 126 */ 127 return new Binary(file, digest, blobProviderId); 128 } 129 130 @Override 131 public Binary getBinary(String digest) { 132 File file = getFileForDigest(digest, false); 133 if (file == null) { 134 // invalid digest 135 return null; 136 } 137 if (!file.exists()) { 138 log.warn("cannot fetch content at " + file.getPath() + " (file does not exist), check your configuration"); 139 return null; 140 } 141 return new Binary(file, digest, blobProviderId); 142 } 143 144 /** 145 * Gets a file representing the storage for a given digest. 146 * 147 * @param digest the digest 148 * @param createDir {@code true} if the directory containing the file itself must be created 149 * @return the file for this digest 150 */ 151 public File getFileForDigest(String digest, boolean createDir) { 152 int depth = descriptor.depth; 153 if (digest.length() < 2 * depth) { 154 return null; 155 } 156 StringBuilder sb = new StringBuilder(3 * depth - 1); 157 for (int i = 0; i < depth; i++) { 158 if (i != 0) { 159 sb.append(File.separatorChar); 160 } 161 sb.append(digest.substring(2 * i, 2 * i + 2)); 162 } 163 File dir = new File(storageDir, sb.toString()); 164 if (createDir) { 165 dir.mkdirs(); 166 } 167 return new File(dir, digest); 168 } 169 170 protected String storeAndDigest(InputStream in) throws IOException { 171 File tmp = File.createTempFile("create_", ".tmp", tmpDir); 172 /* 173 * First, write the input stream to a temporary file, while computing a digest. 174 */ 175 try { 176 String digest; 177 try (OutputStream out = new FileOutputStream(tmp)) { 178 digest = storeAndDigest(in, out); 179 } finally { 180 in.close(); 181 } 182 /* 183 * Move the tmp file to its destination. 184 */ 185 File file = getFileForDigest(digest, true); 186 atomicMove(tmp, file); 187 return digest; 188 } finally { 189 tmp.delete(); 190 } 191 192 } 193 194 /** 195 * Does an atomic move of the tmp (or source) file to the final file. 196 * <p> 197 * Tries to work well with NFS mounts and different filesystems. 198 */ 199 protected void atomicMove(File source, File dest) throws IOException { 200 if (dest.exists()) { 201 // The file with the proper digest is already there so don't do 202 // anything. This is to avoid "Stale NFS File Handle" problems 203 // which would occur if we tried to overwrite it anyway. 204 // Note that this doesn't try to protect from the case where 205 // two identical files are uploaded at the same time. 206 // Update date for the GC. 207 dest.setLastModified(source.lastModified()); 208 return; 209 } 210 if (!source.renameTo(dest)) { 211 // Failed to rename, probably a different filesystem. 212 // Do *NOT* use Apache Commons IO's FileUtils.moveFile() 213 // because it rewrites the destination file so is not atomic. 214 // Do a copy through a tmp file on the same filesystem then 215 // atomic rename. 216 File tmp = File.createTempFile(dest.getName(), ".tmp", dest.getParentFile()); 217 try { 218 try (InputStream in = new FileInputStream(source); // 219 OutputStream out = new FileOutputStream(tmp)) { 220 IOUtils.copy(in, out); 221 } 222 // then do the atomic rename 223 tmp.renameTo(dest); 224 } finally { 225 tmp.delete(); 226 } 227 // finally remove the original source 228 source.delete(); 229 } 230 if (!dest.exists()) { 231 throw new IOException("Could not create file: " + dest); 232 } 233 } 234 235 protected void createGarbageCollector() { 236 garbageCollector = new DefaultBinaryGarbageCollector(this); 237 } 238 239 public static class DefaultBinaryGarbageCollector implements BinaryGarbageCollector { 240 241 /** 242 * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s. 243 */ 244 public static final int TIME_RESOLUTION = 2000; 245 246 protected final LocalBinaryManager binaryManager; 247 248 protected volatile long startTime; 249 250 protected BinaryManagerStatus status; 251 252 public DefaultBinaryGarbageCollector(LocalBinaryManager binaryManager) { 253 this.binaryManager = binaryManager; 254 } 255 256 @Override 257 public String getId() { 258 return binaryManager.getStorageDir().toURI().toString(); 259 } 260 261 @Override 262 public BinaryManagerStatus getStatus() { 263 return status; 264 } 265 266 @Override 267 public boolean isInProgress() { 268 // volatile as this is designed to be called from another thread 269 return startTime != 0; 270 } 271 272 @Override 273 public void start() { 274 if (startTime != 0) { 275 throw new RuntimeException("Alread started"); 276 } 277 startTime = System.currentTimeMillis(); 278 status = new BinaryManagerStatus(); 279 } 280 281 @Override 282 public void mark(String digest) { 283 File file = binaryManager.getFileForDigest(digest, false); 284 if (!file.exists()) { 285 log.error("Unknown file digest: " + digest); 286 return; 287 } 288 touch(file); 289 } 290 291 @Override 292 public void stop(boolean delete) { 293 if (startTime == 0) { 294 throw new RuntimeException("Not started"); 295 } 296 deleteOld(binaryManager.getStorageDir(), startTime - TIME_RESOLUTION, 0, delete); 297 status.gcDuration = System.currentTimeMillis() - startTime; 298 startTime = 0; 299 } 300 301 protected void deleteOld(File file, long minTime, int depth, boolean delete) { 302 if (file.isDirectory()) { 303 for (File f : file.listFiles()) { 304 deleteOld(f, minTime, depth + 1, delete); 305 } 306 if (depth > 0 && file.list().length == 0) { 307 // empty directory 308 file.delete(); 309 } 310 } else if (file.isFile() && file.canWrite()) { 311 long lastModified = file.lastModified(); 312 long length = file.length(); 313 if (lastModified == 0) { 314 log.error("Cannot read last modified for file: " + file); 315 } else if (lastModified < minTime) { 316 status.sizeBinariesGC += length; 317 status.numBinariesGC++; 318 if (delete && !file.delete()) { 319 log.warn("Cannot gc file: " + file); 320 } 321 } else { 322 status.sizeBinaries += length; 323 status.numBinaries++; 324 } 325 } 326 } 327 } 328 329 /** 330 * Sets the last modification date to now on a file 331 * 332 * @param file the file 333 */ 334 public static void touch(File file) { 335 long time = System.currentTimeMillis(); 336 if (file.setLastModified(time)) { 337 // ok 338 return; 339 } 340 if (!file.canWrite()) { 341 // cannot write -> stop won't be able to delete anyway 342 return; 343 } 344 try { 345 // Windows: the file may be open for reading 346 // workaround found by Thomas Mueller, see JCR-2872 347 try (RandomAccessFile r = new RandomAccessFile(file, "rw")) { 348 r.setLength(r.length()); 349 } 350 } catch (IOException e) { 351 log.error("Cannot set last modified for file: " + file, e); 352 } 353 } 354 355}