001/* 002 * (C) Copyright 2006-2012 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume, Mathieu Guillaume, jcarsique 018 */ 019 020package org.nuxeo.ecm.core.blob.binary; 021 022import java.io.File; 023import java.io.FileInputStream; 024import java.io.FileOutputStream; 025import java.io.IOException; 026import java.io.InputStream; 027import java.io.OutputStream; 028import java.io.RandomAccessFile; 029import java.util.Map; 030import java.util.regex.Pattern; 031 032import org.apache.commons.io.FileUtils; 033import org.apache.commons.io.IOUtils; 034import org.apache.commons.lang.StringUtils; 035import org.apache.commons.logging.Log; 036import org.apache.commons.logging.LogFactory; 037import org.nuxeo.common.Environment; 038import org.nuxeo.ecm.core.api.NuxeoException; 039import org.nuxeo.runtime.api.Framework; 040import org.nuxeo.runtime.trackers.files.FileEventTracker; 041 042/** 043 * A simple filesystem-based binary manager. It stores the binaries according to their digest (hash), which means that 044 * no transactional behavior needs to be implemented. 045 * <p> 046 * A garbage collection is needed to purge unused binaries. 047 * <p> 048 * The format of the <em>binaries</em> directory is: 049 * <ul> 050 * <li><em>data/</em> hierarchy with the actual binaries in subdirectories,</li> 051 * <li><em>tmp/</em> temporary storage during creation,</li> 052 * <li><em>config.xml</em> a file containing the configuration used.</li> 053 * </ul> 054 * 055 * @author Florent Guillaume 056 * @since 5.6 057 */ 058public class LocalBinaryManager extends AbstractBinaryManager { 059 060 private static final Log log = LogFactory.getLog(LocalBinaryManager.class); 061 062 public static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("[a-zA-Z]:[/\\\\].*"); 063 064 public static final String DEFAULT_PATH = "binaries"; 065 066 public static final String DATA = "data"; 067 068 public static final String TMP = "tmp"; 069 070 public static final String CONFIG_FILE = "config.xml"; 071 072 protected File storageDir; 073 074 protected File tmpDir; 075 076 @Override 077 public void initialize(String blobProviderId, Map<String, String> properties) throws IOException { 078 super.initialize(blobProviderId, properties); 079 String path = properties.get(BinaryManager.PROP_PATH); 080 if (StringUtils.isBlank(path)) { 081 path = DEFAULT_PATH; 082 } 083 path = Framework.expandVars(path); 084 path = path.trim(); 085 File base; 086 if (path.startsWith("/") || path.startsWith("\\") || path.contains("://") || path.contains(":\\") 087 || WINDOWS_ABSOLUTE_PATH.matcher(path).matches()) { 088 // absolute 089 base = new File(path); 090 } else { 091 // relative 092 File home = Environment.getDefault().getData(); 093 base = new File(home, path); 094 095 // Backward compliance with versions before 5.4 (NXP-5370) 096 File oldBase = new File(Framework.getRuntime().getHome().getPath(), path); 097 if (oldBase.exists()) { 098 log.warn("Old binaries path used (NXP-5370). Please move " + oldBase + " to " + base); 099 base = oldBase; 100 } 101 } 102 103 log.info("Registering binary manager '" + blobProviderId + "' using " 104 + (this.getClass().equals(LocalBinaryManager.class) ? "" : (this.getClass().getSimpleName() + " and ")) 105 + "binary store: " + base); 106 storageDir = new File(base, DATA); 107 tmpDir = new File(base, TMP); 108 storageDir.mkdirs(); 109 tmpDir.mkdirs(); 110 descriptor = getDescriptor(new File(base, CONFIG_FILE)); 111 createGarbageCollector(); 112 113 // be sure FileTracker won't steal our files ! 114 FileEventTracker.registerProtectedPath(storageDir.getAbsolutePath()); 115 } 116 117 @Override 118 public void close() { 119 if (tmpDir != null) { 120 try { 121 FileUtils.cleanDirectory(tmpDir); 122 } catch (IOException e) { 123 throw new NuxeoException(e); 124 } 125 } 126 } 127 128 public File getStorageDir() { 129 return storageDir; 130 } 131 132 @Override 133 protected Binary getBinary(InputStream in) throws IOException { 134 String digest = storeAndDigest(in); 135 File file = getFileForDigest(digest, false); 136 /* 137 * Now we can build the Binary. 138 */ 139 return new Binary(file, digest, blobProviderId); 140 } 141 142 @Override 143 public Binary getBinary(String digest) { 144 File file = getFileForDigest(digest, false); 145 if (file == null) { 146 // invalid digest 147 return null; 148 } 149 if (!file.exists()) { 150 log.warn("cannot fetch content at " + file.getPath() + " (file does not exist), check your configuration"); 151 return null; 152 } 153 return new Binary(file, digest, blobProviderId); 154 } 155 156 /** 157 * Gets a file representing the storage for a given digest. 158 * 159 * @param digest the digest 160 * @param createDir {@code true} if the directory containing the file itself must be created 161 * @return the file for this digest 162 */ 163 public File getFileForDigest(String digest, boolean createDir) { 164 int depth = descriptor.depth; 165 if (digest.length() < 2 * depth) { 166 return null; 167 } 168 StringBuilder buf = new StringBuilder(3 * depth - 1); 169 for (int i = 0; i < depth; i++) { 170 if (i != 0) { 171 buf.append(File.separatorChar); 172 } 173 buf.append(digest.substring(2 * i, 2 * i + 2)); 174 } 175 File dir = new File(storageDir, buf.toString()); 176 if (createDir) { 177 dir.mkdirs(); 178 } 179 return new File(dir, digest); 180 } 181 182 protected String storeAndDigest(InputStream in) throws IOException { 183 File tmp = File.createTempFile("create_", ".tmp", tmpDir); 184 OutputStream out = new FileOutputStream(tmp); 185 /* 186 * First, write the input stream to a temporary file, while computing a digest. 187 */ 188 try { 189 String digest; 190 try { 191 digest = storeAndDigest(in, out); 192 } finally { 193 in.close(); 194 out.close(); 195 } 196 /* 197 * Move the tmp file to its destination. 198 */ 199 File file = getFileForDigest(digest, true); 200 atomicMove(tmp, file); 201 return digest; 202 } finally { 203 tmp.delete(); 204 } 205 206 } 207 208 /** 209 * Does an atomic move of the tmp (or source) file to the final file. 210 * <p> 211 * Tries to work well with NFS mounts and different filesystems. 212 */ 213 protected void atomicMove(File source, File dest) throws IOException { 214 if (dest.exists()) { 215 // The file with the proper digest is already there so don't do 216 // anything. This is to avoid "Stale NFS File Handle" problems 217 // which would occur if we tried to overwrite it anyway. 218 // Note that this doesn't try to protect from the case where 219 // two identical files are uploaded at the same time. 220 // Update date for the GC. 221 dest.setLastModified(source.lastModified()); 222 return; 223 } 224 if (!source.renameTo(dest)) { 225 // Failed to rename, probably a different filesystem. 226 // Do *NOT* use Apache Commons IO's FileUtils.moveFile() 227 // because it rewrites the destination file so is not atomic. 228 // Do a copy through a tmp file on the same filesystem then 229 // atomic rename. 230 File tmp = File.createTempFile(dest.getName(), ".tmp", dest.getParentFile()); 231 try { 232 try (InputStream in = new FileInputStream(source); // 233 OutputStream out = new FileOutputStream(tmp)) { 234 IOUtils.copy(in, out); 235 } 236 // then do the atomic rename 237 tmp.renameTo(dest); 238 } finally { 239 tmp.delete(); 240 } 241 // finally remove the original source 242 source.delete(); 243 } 244 if (!dest.exists()) { 245 throw new IOException("Could not create file: " + dest); 246 } 247 } 248 249 protected void createGarbageCollector() { 250 garbageCollector = new DefaultBinaryGarbageCollector(this); 251 } 252 253 public static class DefaultBinaryGarbageCollector implements BinaryGarbageCollector { 254 255 /** 256 * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s. 257 */ 258 public static int TIME_RESOLUTION = 2000; 259 260 protected final LocalBinaryManager binaryManager; 261 262 protected volatile long startTime; 263 264 protected BinaryManagerStatus status; 265 266 public DefaultBinaryGarbageCollector(LocalBinaryManager binaryManager) { 267 this.binaryManager = binaryManager; 268 } 269 270 @Override 271 public String getId() { 272 return binaryManager.getStorageDir().toURI().toString(); 273 } 274 275 @Override 276 public BinaryManagerStatus getStatus() { 277 return status; 278 } 279 280 @Override 281 public boolean isInProgress() { 282 // volatile as this is designed to be called from another thread 283 return startTime != 0; 284 } 285 286 @Override 287 public void start() { 288 if (startTime != 0) { 289 throw new RuntimeException("Alread started"); 290 } 291 startTime = System.currentTimeMillis(); 292 status = new BinaryManagerStatus(); 293 } 294 295 @Override 296 public void mark(String digest) { 297 File file = binaryManager.getFileForDigest(digest, false); 298 if (!file.exists()) { 299 log.error("Unknown file digest: " + digest); 300 return; 301 } 302 touch(file); 303 } 304 305 @Override 306 public void stop(boolean delete) { 307 if (startTime == 0) { 308 throw new RuntimeException("Not started"); 309 } 310 deleteOld(binaryManager.getStorageDir(), startTime - TIME_RESOLUTION, 0, delete); 311 status.gcDuration = System.currentTimeMillis() - startTime; 312 startTime = 0; 313 } 314 315 protected void deleteOld(File file, long minTime, int depth, boolean delete) { 316 if (file.isDirectory()) { 317 for (File f : file.listFiles()) { 318 deleteOld(f, minTime, depth + 1, delete); 319 } 320 if (depth > 0 && file.list().length == 0) { 321 // empty directory 322 file.delete(); 323 } 324 } else if (file.isFile() && file.canWrite()) { 325 long lastModified = file.lastModified(); 326 long length = file.length(); 327 if (lastModified == 0) { 328 log.error("Cannot read last modified for file: " + file); 329 } else if (lastModified < minTime) { 330 status.sizeBinariesGC += length; 331 status.numBinariesGC++; 332 if (delete && !file.delete()) { 333 log.warn("Cannot gc file: " + file); 334 } 335 } else { 336 status.sizeBinaries += length; 337 status.numBinaries++; 338 } 339 } 340 } 341 } 342 343 /** 344 * Sets the last modification date to now on a file 345 * 346 * @param file the file 347 */ 348 public static void touch(File file) { 349 long time = System.currentTimeMillis(); 350 if (file.setLastModified(time)) { 351 // ok 352 return; 353 } 354 if (!file.canWrite()) { 355 // cannot write -> stop won't be able to delete anyway 356 return; 357 } 358 try { 359 // Windows: the file may be open for reading 360 // workaround found by Thomas Mueller, see JCR-2872 361 RandomAccessFile r = new RandomAccessFile(file, "rw"); 362 try { 363 r.setLength(r.length()); 364 } finally { 365 r.close(); 366 } 367 } catch (IOException e) { 368 log.error("Cannot set last modified for file: " + file, e); 369 } 370 } 371 372}