001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and contributors.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the GNU Lesser General Public License
006 * (LGPL) version 2.1 which accompanies this distribution, and is available at
007 * http://www.gnu.org/licenses/lgpl-2.1.html
008 *
009 * This library is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 * Lesser General Public License for more details.
013 *
014 * Contributors:
015 *     Thierry Delprat <tdelprat@nuxeo.com>
016 *     Antoine Taillefer <ataillefer@nuxeo.com>
017 */
018
019package org.nuxeo.ecm.core.transientstore;
020
021import java.io.File;
022import java.io.IOException;
023import java.io.Serializable;
024import java.nio.file.DirectoryStream;
025import java.nio.file.Files;
026import java.nio.file.Path;
027import java.nio.file.Paths;
028import java.util.ArrayList;
029import java.util.HashMap;
030import java.util.List;
031import java.util.Map;
032import java.util.UUID;
033
034import org.apache.commons.codec.binary.Base64;
035import org.apache.commons.io.FileUtils;
036import org.apache.commons.logging.Log;
037import org.apache.commons.logging.LogFactory;
038import org.nuxeo.common.Environment;
039import org.nuxeo.ecm.core.api.Blob;
040import org.nuxeo.ecm.core.api.NuxeoException;
041import org.nuxeo.ecm.core.api.impl.blob.FileBlob;
042import org.nuxeo.ecm.core.transientstore.api.MaximumTransientSpaceExceeded;
043import org.nuxeo.ecm.core.transientstore.api.TransientStore;
044import org.nuxeo.ecm.core.transientstore.api.TransientStoreConfig;
045
046/**
047 * Base class for a {@link TransientStore} implementation.
048 *
049 * @since 7.2
050 */
051public abstract class AbstractTransientStore implements TransientStore {
052
053    protected static final Log log = LogFactory.getLog(AbstractTransientStore.class);
054
055    protected TransientStoreConfig config;
056
057    protected File cacheDir;
058
059    @Override
060    public void init(TransientStoreConfig config) {
061        this.config = config;
062
063        // initialize caching directory
064        File transienStoreHome = new File(Environment.getDefault().getData(), "transientstores");
065        File data = new File(transienStoreHome, config.getName());
066        data.mkdirs();
067        cacheDir = data.getAbsoluteFile();
068    }
069
070    @Override
071    public abstract void shutdown();
072
073    @Override
074    public abstract boolean exists(String key);
075
076    @Override
077    public abstract void putParameter(String key, String parameter, Serializable value);
078
079    @Override
080    public abstract Serializable getParameter(String key, String parameter);
081
082    @Override
083    public abstract void putParameters(String key, Map<String, Serializable> parameters);
084
085    @Override
086    public abstract Map<String, Serializable> getParameters(String key);
087
088    @Override
089    public abstract List<Blob> getBlobs(String key);
090
091    @Override
092    public abstract long getSize(String key);
093
094    @Override
095    public abstract boolean isCompleted(String key);
096
097    @Override
098    public abstract void setCompleted(String key, boolean completed);
099
100    @Override
101    public abstract void remove(String key);
102
103    @Override
104    public abstract void release(String key);
105
106    /**
107     * Updates the total storage size and the storage size of the entry with the given {@code key} according to
108     * {@code sizeOfBlobs} and stores the blob information in this entry.
109     */
110    protected abstract void persistBlobs(String key, long sizeOfBlobs, List<Map<String, String>> blobInfos);
111
112    /**
113     * Returns the size of the disk storage in bytes.
114     */
115    public abstract long getStorageSize();
116
117    /**
118     * Sets the size of the disk storage in bytes.
119     */
120    protected abstract void setStorageSize(long newSize);
121
122    protected abstract long incrementStorageSize(long size);
123
124    protected abstract long decrementStorageSize(long size);
125
126    protected abstract void removeAllEntries();
127
128    @Override
129    public void putBlobs(String key, List<Blob> blobs) {
130        if (config.getAbsoluteMaxSizeMB() < 0 || getStorageSize() < config.getAbsoluteMaxSizeMB() * (1024 * 1024)) {
131            // Store blobs on the file system
132            List<Map<String, String>> blobInfos = storeBlobs(key, blobs);
133            // Persist blob information in the store
134            persistBlobs(key, getSizeOfBlobs(blobs), blobInfos);
135        } else {
136            throw new MaximumTransientSpaceExceeded();
137        }
138    }
139
140    protected List<Map<String, String>> storeBlobs(String key, List<Blob> blobs) {
141        if (blobs == null) {
142            return null;
143        }
144        // Store blobs on the file system and compute blob information
145        List<Map<String, String>> blobInfos = new ArrayList<>();
146        for (Blob blob : blobs) {
147            Map<String, String> blobInfo = new HashMap<>();
148            File cachingDir = getCachingDirectory(key);
149            String uuid = UUID.randomUUID().toString();
150            File cachedFile = new File(cachingDir, uuid);
151            try {
152                if (blob instanceof FileBlob && ((FileBlob) blob).isTemporary()) {
153                    ((FileBlob) blob).moveTo(cachedFile);
154                } else {
155                    blob.transferTo(cachedFile);
156                }
157            } catch (IOException e) {
158                throw new NuxeoException(e);
159            }
160            Path cachedFileRelativePath = Paths.get(cachingDir.getName(), uuid);
161            blobInfo.put("file", cachedFileRelativePath.toString());
162            // Redis doesn't support null values
163            if (blob.getFilename() != null) {
164                blobInfo.put("filename", blob.getFilename());
165            }
166            if (blob.getEncoding() != null) {
167                blobInfo.put("encoding", blob.getEncoding());
168            }
169            if (blob.getMimeType() != null) {
170                blobInfo.put("mimetype", blob.getMimeType());
171            }
172            if (blob.getDigest() != null) {
173                blobInfo.put("digest", blob.getDigest());
174            }
175            blobInfos.add(blobInfo);
176        }
177        log.debug("Stored blobs on the file system: " + blobInfos);
178        return blobInfos;
179    }
180
181    public File getCachingDirectory(String key) {
182        try {
183            File cachingDir = new File(cacheDir.getCanonicalFile(), getCachingDirName(key));
184
185            if (!cachingDir.getCanonicalPath().startsWith(cacheDir.getCanonicalPath())) {
186                throw new SecurityException("Trying to traverse illegal path");
187            }
188
189            if (!cachingDir.exists()) {
190                cachingDir.mkdir();
191            }
192            return cachingDir;
193        } catch (IOException e) {
194            throw new RuntimeException("Error when trying to access cache directory");
195        }
196    }
197
198    protected String getCachingDirName(String key) {
199        String dirName = Base64.encodeBase64String(key.getBytes());
200        dirName = dirName.replaceAll("/", "_");
201        return dirName;
202    }
203
204    protected long getSizeOfBlobs(List<Blob> blobs) {
205        int size = 0;
206        if (blobs != null) {
207            for (Blob blob : blobs) {
208                long blobLength = blob.getLength();
209                if (blobLength > -1) {
210                    size += blobLength;
211                }
212            }
213        }
214        return size;
215    }
216
217    protected List<Blob> loadBlobs(List<Map<String, String>> blobInfos) {
218        log.debug("Loading blobs from the file system: " + blobInfos);
219        List<Blob> blobs = new ArrayList<>();
220        for (Map<String, String> info : blobInfos) {
221            File blobFile = new File(cacheDir, info.get("file"));
222            Blob blob = new FileBlob(blobFile);
223            blob.setEncoding(info.get("encoding"));
224            blob.setMimeType(info.get("mimetype"));
225            blob.setFilename(info.get("filename"));
226            blob.setDigest(info.get("digest"));
227            blobs.add(blob);
228        }
229        return blobs;
230    }
231
232    @Override
233    public int getStorageSizeMB() {
234        return (int) getStorageSize() / (1024 * 1024);
235    }
236
237    @Override
238    public void doGC() {
239        log.debug(String.format("Performing GC for TransientStore %s", config.getName()));
240        long newSize = 0;
241        try {
242            try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(cacheDir.getAbsolutePath()))) {
243                for (Path entry : stream) {
244                    String key = getKeyCachingDirName(entry.getFileName().toString());
245                    try {
246                        if (exists(key)) {
247                            newSize += getFilePathSize(entry);
248                            continue;
249                        }
250                        FileUtils.deleteDirectory(entry.toFile());
251                    } catch (IOException e) {
252                        log.error("Error while performing GC", e);
253                    }
254                }
255            }
256        } catch (IOException e) {
257            log.error("Error while performing GC", e);
258        }
259        setStorageSize(newSize);
260    }
261
262    protected String getKeyCachingDirName(String dir) {
263        String key = dir.replaceAll("_", "/");
264        return new String(Base64.decodeBase64(key));
265    }
266
267    protected long getFilePathSize(Path entry) {
268        long size = 0;
269        for (File file : entry.toFile().listFiles()) {
270            size += file.length();
271        }
272        return size;
273    }
274
275    @Override
276    public void removeAll() {
277        log.debug("Removing all entries from TransientStore " + config.getName());
278        removeAllEntries();
279        doGC();
280    }
281
282}