001/* 002 * (C) Copyright 2006-2018 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Florent Guillaume 018 * Mathieu Guillaume 019 * jcarsique 020 */ 021 022package org.nuxeo.ecm.core.blob.binary; 023 024import java.io.File; 025import java.io.FileInputStream; 026import java.io.FileOutputStream; 027import java.io.IOException; 028import java.io.InputStream; 029import java.io.OutputStream; 030import java.io.RandomAccessFile; 031import java.util.Map; 032import java.util.regex.Pattern; 033 034import org.apache.commons.io.FileUtils; 035import org.apache.commons.io.IOUtils; 036import org.apache.commons.lang3.StringUtils; 037import org.apache.commons.logging.Log; 038import org.apache.commons.logging.LogFactory; 039import org.nuxeo.common.Environment; 040import org.nuxeo.ecm.core.api.NuxeoException; 041import org.nuxeo.ecm.core.blob.BlobProviderDescriptor; 042import org.nuxeo.runtime.api.Framework; 043import org.nuxeo.runtime.trackers.files.FileEventTracker; 044 045/** 046 * A simple filesystem-based binary manager. It stores the binaries according to their digest (hash), which means that 047 * no transactional behavior needs to be implemented. 048 * <p> 049 * A garbage collection is needed to purge unused binaries. 050 * <p> 051 * The format of the <em>binaries</em> directory is: 052 * <ul> 053 * <li><em>data/</em> hierarchy with the actual binaries in subdirectories,</li> 054 * <li><em>tmp/</em> temporary storage during creation,</li> 055 * <li><em>config.xml</em> a file containing the configuration used.</li> 056 * </ul> 057 * 058 * @author Florent Guillaume 059 * @since 5.6 060 */ 061public class LocalBinaryManager extends AbstractBinaryManager { 062 063 private static final Log log = LogFactory.getLog(LocalBinaryManager.class); 064 065 public static final Pattern WINDOWS_ABSOLUTE_PATH = Pattern.compile("[a-zA-Z]:[/\\\\].*"); 066 067 public static final String DEFAULT_PATH = "binaries"; 068 069 public static final String DATA = "data"; 070 071 public static final String TMP = "tmp"; 072 073 public static final String CONFIG_FILE = "config.xml"; 074 075 protected File storageDir; 076 077 protected File tmpDir; 078 079 @Override 080 public void initialize(String blobProviderId, Map<String, String> properties) throws IOException { 081 super.initialize(blobProviderId, properties); 082 String path = properties.get(BinaryManager.PROP_PATH); 083 if (StringUtils.isBlank(path)) { 084 path = DEFAULT_PATH; 085 } 086 path = Framework.expandVars(path); 087 path = path.trim(); 088 File base; 089 if (path.startsWith("/") || path.startsWith("\\") || path.contains("://") || path.contains(":\\") 090 || WINDOWS_ABSOLUTE_PATH.matcher(path).matches()) { 091 // absolute 092 base = new File(path); 093 } else { 094 // relative 095 File home = Environment.getDefault().getData(); 096 base = new File(home, path); 097 098 // Backward compliance with versions before 5.4 (NXP-5370) 099 File oldBase = new File(Framework.getRuntime().getHome().getPath(), path); 100 if (oldBase.exists()) { 101 log.warn("Old binaries path used (NXP-5370). Please move " + oldBase + " to " + base); 102 base = oldBase; 103 } 104 } 105 // take namespace into account 106 String namespace = properties.get(BlobProviderDescriptor.NAMESPACE); 107 if (StringUtils.isNotBlank(namespace)) { 108 base = new File(base.getParentFile(), base.getName() + "_" + namespace.trim()); 109 } 110 111 log.info("Registering binary manager '" + blobProviderId + "' using " 112 + (this.getClass().equals(LocalBinaryManager.class) ? "" : (this.getClass().getSimpleName() + " and ")) 113 + "binary store: " + base); 114 storageDir = new File(base, DATA); 115 tmpDir = new File(base, TMP); 116 storageDir.mkdirs(); 117 tmpDir.mkdirs(); 118 descriptor = getDescriptor(new File(base, CONFIG_FILE)); 119 createGarbageCollector(); 120 121 // be sure FileTracker won't steal our files ! 122 FileEventTracker.registerProtectedPath(storageDir.getAbsolutePath()); 123 } 124 125 @Override 126 public void close() { 127 if (tmpDir != null) { 128 try { 129 FileUtils.cleanDirectory(tmpDir); 130 } catch (IOException e) { 131 throw new NuxeoException(e); 132 } 133 } 134 } 135 136 public File getStorageDir() { 137 return storageDir; 138 } 139 140 @Override 141 protected Binary getBinary(InputStream in) throws IOException { 142 String digest = storeAndDigest(in); 143 File file = getFileForDigest(digest, false); 144 /* 145 * Now we can build the Binary. 146 */ 147 return new Binary(file, digest, blobProviderId); 148 } 149 150 @Override 151 public Binary getBinary(String digest) { 152 File file = getFileForDigest(digest, false); 153 if (file == null) { 154 // invalid digest 155 return null; 156 } 157 if (!file.exists()) { 158 log.warn("cannot fetch content at " + file.getPath() + " (file does not exist), check your configuration"); 159 return null; 160 } 161 return new Binary(file, digest, blobProviderId); 162 } 163 164 /** 165 * Gets a file representing the storage for a given digest. 166 * 167 * @param digest the digest 168 * @param createDir {@code true} if the directory containing the file itself must be created 169 * @return the file for this digest 170 */ 171 public File getFileForDigest(String digest, boolean createDir) { 172 int depth = descriptor.depth; 173 if (digest.length() < 2 * depth) { 174 return null; 175 } 176 StringBuilder buf = new StringBuilder(3 * depth - 1); 177 for (int i = 0; i < depth; i++) { 178 if (i != 0) { 179 buf.append(File.separatorChar); 180 } 181 buf.append(digest.substring(2 * i, 2 * i + 2)); 182 } 183 File dir = new File(storageDir, buf.toString()); 184 if (createDir) { 185 dir.mkdirs(); 186 } 187 return new File(dir, digest); 188 } 189 190 protected String storeAndDigest(InputStream in) throws IOException { 191 File tmp = File.createTempFile("create_", ".tmp", tmpDir); 192 /* 193 * First, write the input stream to a temporary file, while computing a digest. 194 */ 195 try { 196 String digest; 197 try (OutputStream out = new FileOutputStream(tmp)) { 198 digest = storeAndDigest(in, out); 199 } finally { 200 in.close(); 201 } 202 /* 203 * Move the tmp file to its destination. 204 */ 205 File file = getFileForDigest(digest, true); 206 atomicMove(tmp, file); 207 return digest; 208 } finally { 209 tmp.delete(); 210 } 211 212 } 213 214 /** 215 * Does an atomic move of the tmp (or source) file to the final file. 216 * <p> 217 * Tries to work well with NFS mounts and different filesystems. 218 */ 219 protected void atomicMove(File source, File dest) throws IOException { 220 if (dest.exists()) { 221 // The file with the proper digest is already there so don't do 222 // anything. This is to avoid "Stale NFS File Handle" problems 223 // which would occur if we tried to overwrite it anyway. 224 // Note that this doesn't try to protect from the case where 225 // two identical files are uploaded at the same time. 226 // Update date for the GC. 227 dest.setLastModified(source.lastModified()); 228 return; 229 } 230 if (!source.renameTo(dest)) { 231 // Failed to rename, probably a different filesystem. 232 // Do *NOT* use Apache Commons IO's FileUtils.moveFile() 233 // because it rewrites the destination file so is not atomic. 234 // Do a copy through a tmp file on the same filesystem then 235 // atomic rename. 236 File tmp = File.createTempFile(dest.getName(), ".tmp", dest.getParentFile()); 237 try { 238 try (InputStream in = new FileInputStream(source); // 239 OutputStream out = new FileOutputStream(tmp)) { 240 IOUtils.copy(in, out); 241 } 242 // then do the atomic rename 243 tmp.renameTo(dest); 244 } finally { 245 tmp.delete(); 246 } 247 // finally remove the original source 248 source.delete(); 249 } 250 if (!dest.exists()) { 251 throw new IOException("Could not create file: " + dest); 252 } 253 } 254 255 protected void createGarbageCollector() { 256 garbageCollector = new DefaultBinaryGarbageCollector(this); 257 } 258 259 public static class DefaultBinaryGarbageCollector implements BinaryGarbageCollector { 260 261 /** 262 * Windows FAT filesystems have a time resolution of 2s. Other common filesystems have 1s. 263 */ 264 public static final int TIME_RESOLUTION = 2000; 265 266 protected final LocalBinaryManager binaryManager; 267 268 protected volatile long startTime; 269 270 protected BinaryManagerStatus status; 271 272 public DefaultBinaryGarbageCollector(LocalBinaryManager binaryManager) { 273 this.binaryManager = binaryManager; 274 } 275 276 @Override 277 public String getId() { 278 return binaryManager.getStorageDir().toURI().toString(); 279 } 280 281 @Override 282 public BinaryManagerStatus getStatus() { 283 return status; 284 } 285 286 @Override 287 public boolean isInProgress() { 288 // volatile as this is designed to be called from another thread 289 return startTime != 0; 290 } 291 292 @Override 293 public void start() { 294 if (startTime != 0) { 295 throw new RuntimeException("Alread started"); 296 } 297 startTime = System.currentTimeMillis(); 298 status = new BinaryManagerStatus(); 299 } 300 301 @Override 302 public void mark(String digest) { 303 File file = binaryManager.getFileForDigest(digest, false); 304 if (!file.exists()) { 305 log.error("Unknown file digest: " + digest); 306 return; 307 } 308 touch(file); 309 } 310 311 @Override 312 public void stop(boolean delete) { 313 if (startTime == 0) { 314 throw new RuntimeException("Not started"); 315 } 316 deleteOld(binaryManager.getStorageDir(), startTime - TIME_RESOLUTION, 0, delete); 317 status.gcDuration = System.currentTimeMillis() - startTime; 318 startTime = 0; 319 } 320 321 protected void deleteOld(File file, long minTime, int depth, boolean delete) { 322 if (file.isDirectory()) { 323 for (File f : file.listFiles()) { 324 deleteOld(f, minTime, depth + 1, delete); 325 } 326 if (depth > 0 && file.list().length == 0) { 327 // empty directory 328 file.delete(); 329 } 330 } else if (file.isFile() && file.canWrite()) { 331 long lastModified = file.lastModified(); 332 long length = file.length(); 333 if (lastModified == 0) { 334 log.error("Cannot read last modified for file: " + file); 335 } else if (lastModified < minTime) { 336 status.sizeBinariesGC += length; 337 status.numBinariesGC++; 338 if (delete && !file.delete()) { 339 log.warn("Cannot gc file: " + file); 340 } 341 } else { 342 status.sizeBinaries += length; 343 status.numBinaries++; 344 } 345 } 346 } 347 } 348 349 /** 350 * Sets the last modification date to now on a file 351 * 352 * @param file the file 353 */ 354 public static void touch(File file) { 355 long time = System.currentTimeMillis(); 356 if (file.setLastModified(time)) { 357 // ok 358 return; 359 } 360 if (!file.canWrite()) { 361 // cannot write -> stop won't be able to delete anyway 362 return; 363 } 364 try { 365 // Windows: the file may be open for reading 366 // workaround found by Thomas Mueller, see JCR-2872 367 try (RandomAccessFile r = new RandomAccessFile(file, "rw")) { 368 r.setLength(r.length()); 369 } 370 } catch (IOException e) { 371 log.error("Cannot set last modified for file: " + file, e); 372 } 373 } 374 375}