001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Thierry Delprat <tdelprat@nuxeo.com>
018 *     Antoine Taillefer <ataillefer@nuxeo.com>
019 */
020
021package org.nuxeo.ecm.core.transientstore;
022
023import java.io.File;
024import java.io.IOException;
025import java.io.Serializable;
026import java.nio.file.DirectoryStream;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.nio.file.Paths;
030import java.util.ArrayList;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.UUID;
035
036import org.apache.commons.codec.binary.Base64;
037import org.apache.commons.io.FileUtils;
038import org.apache.commons.logging.Log;
039import org.apache.commons.logging.LogFactory;
040import org.nuxeo.common.Environment;
041import org.nuxeo.ecm.core.api.Blob;
042import org.nuxeo.ecm.core.api.NuxeoException;
043import org.nuxeo.ecm.core.api.impl.blob.FileBlob;
044import org.nuxeo.ecm.core.transientstore.api.MaximumTransientSpaceExceeded;
045import org.nuxeo.ecm.core.transientstore.api.TransientStore;
046import org.nuxeo.ecm.core.transientstore.api.TransientStoreConfig;
047
048/**
049 * Base class for a {@link TransientStore} implementation.
050 *
051 * @since 7.2
052 */
053public abstract class AbstractTransientStore implements TransientStore {
054
055    protected static final Log log = LogFactory.getLog(AbstractTransientStore.class);
056
057    protected TransientStoreConfig config;
058
059    protected File cacheDir;
060
061    @Override
062    public void init(TransientStoreConfig config) {
063        this.config = config;
064
065        // initialize caching directory
066        File transienStoreHome = new File(Environment.getDefault().getData(), "transientstores");
067        File data = new File(transienStoreHome, config.getName());
068        data.mkdirs();
069        cacheDir = data.getAbsoluteFile();
070    }
071
072    @Override
073    public abstract void shutdown();
074
075    @Override
076    public abstract boolean exists(String key);
077
078    @Override
079    public abstract void putParameter(String key, String parameter, Serializable value);
080
081    @Override
082    public abstract Serializable getParameter(String key, String parameter);
083
084    @Override
085    public abstract void putParameters(String key, Map<String, Serializable> parameters);
086
087    @Override
088    public abstract Map<String, Serializable> getParameters(String key);
089
090    @Override
091    public abstract List<Blob> getBlobs(String key);
092
093    @Override
094    public abstract long getSize(String key);
095
096    @Override
097    public abstract boolean isCompleted(String key);
098
099    @Override
100    public abstract void setCompleted(String key, boolean completed);
101
102    @Override
103    public abstract void remove(String key);
104
105    @Override
106    public abstract void release(String key);
107
108    /**
109     * Updates the total storage size and the storage size of the entry with the given {@code key} according to
110     * {@code sizeOfBlobs} and stores the blob information in this entry.
111     */
112    protected abstract void persistBlobs(String key, long sizeOfBlobs, List<Map<String, String>> blobInfos);
113
114    /**
115     * Returns the size of the disk storage in bytes.
116     */
117    public abstract long getStorageSize();
118
119    /**
120     * Sets the size of the disk storage in bytes.
121     */
122    protected abstract void setStorageSize(long newSize);
123
124    protected abstract long incrementStorageSize(long size);
125
126    protected abstract long decrementStorageSize(long size);
127
128    protected abstract void removeAllEntries();
129
130    @Override
131    public void putBlobs(String key, List<Blob> blobs) {
132        if (config.getAbsoluteMaxSizeMB() < 0 || getStorageSize() < config.getAbsoluteMaxSizeMB() * (1024 * 1024)) {
133            // Store blobs on the file system
134            List<Map<String, String>> blobInfos = storeBlobs(key, blobs);
135            // Persist blob information in the store
136            persistBlobs(key, getSizeOfBlobs(blobs), blobInfos);
137        } else {
138            throw new MaximumTransientSpaceExceeded();
139        }
140    }
141
142    protected List<Map<String, String>> storeBlobs(String key, List<Blob> blobs) {
143        if (blobs == null) {
144            return null;
145        }
146        // Store blobs on the file system and compute blob information
147        List<Map<String, String>> blobInfos = new ArrayList<>();
148        for (Blob blob : blobs) {
149            Map<String, String> blobInfo = new HashMap<>();
150            File cachingDir = getCachingDirectory(key);
151            String uuid = UUID.randomUUID().toString();
152            File cachedFile = new File(cachingDir, uuid);
153            try {
154                if (blob instanceof FileBlob && ((FileBlob) blob).isTemporary()) {
155                    ((FileBlob) blob).moveTo(cachedFile);
156                } else {
157                    blob.transferTo(cachedFile);
158                }
159            } catch (IOException e) {
160                throw new NuxeoException(e);
161            }
162            Path cachedFileRelativePath = Paths.get(cachingDir.getName(), uuid);
163            blobInfo.put("file", cachedFileRelativePath.toString());
164            // Redis doesn't support null values
165            if (blob.getFilename() != null) {
166                blobInfo.put("filename", blob.getFilename());
167            }
168            if (blob.getEncoding() != null) {
169                blobInfo.put("encoding", blob.getEncoding());
170            }
171            if (blob.getMimeType() != null) {
172                blobInfo.put("mimetype", blob.getMimeType());
173            }
174            if (blob.getDigest() != null) {
175                blobInfo.put("digest", blob.getDigest());
176            }
177            blobInfos.add(blobInfo);
178        }
179        log.debug("Stored blobs on the file system: " + blobInfos);
180        return blobInfos;
181    }
182
183    public File getCachingDirectory(String key) {
184        try {
185            File cachingDir = new File(cacheDir.getCanonicalFile(), getCachingDirName(key));
186
187            if (!cachingDir.getCanonicalPath().startsWith(cacheDir.getCanonicalPath())) {
188                throw new SecurityException("Trying to traverse illegal path");
189            }
190
191            if (!cachingDir.exists()) {
192                cachingDir.mkdir();
193            }
194            return cachingDir;
195        } catch (IOException e) {
196            throw new RuntimeException("Error when trying to access cache directory");
197        }
198    }
199
200    protected String getCachingDirName(String key) {
201        String dirName = Base64.encodeBase64String(key.getBytes());
202        dirName = dirName.replaceAll("/", "_");
203        return dirName;
204    }
205
206    protected long getSizeOfBlobs(List<Blob> blobs) {
207        int size = 0;
208        if (blobs != null) {
209            for (Blob blob : blobs) {
210                long blobLength = blob.getLength();
211                if (blobLength > -1) {
212                    size += blobLength;
213                }
214            }
215        }
216        return size;
217    }
218
219    protected List<Blob> loadBlobs(List<Map<String, String>> blobInfos) {
220        log.debug("Loading blobs from the file system: " + blobInfos);
221        List<Blob> blobs = new ArrayList<>();
222        for (Map<String, String> info : blobInfos) {
223            File blobFile = new File(cacheDir, info.get("file"));
224            Blob blob = new FileBlob(blobFile);
225            blob.setEncoding(info.get("encoding"));
226            blob.setMimeType(info.get("mimetype"));
227            blob.setFilename(info.get("filename"));
228            blob.setDigest(info.get("digest"));
229            blobs.add(blob);
230        }
231        return blobs;
232    }
233
234    @Override
235    public int getStorageSizeMB() {
236        return (int) getStorageSize() / (1024 * 1024);
237    }
238
239    @Override
240    public void doGC() {
241        log.debug(String.format("Performing GC for TransientStore %s", config.getName()));
242        long newSize = 0;
243        try {
244            try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(cacheDir.getAbsolutePath()))) {
245                for (Path entry : stream) {
246                    String key = getKeyCachingDirName(entry.getFileName().toString());
247                    try {
248                        if (exists(key)) {
249                            newSize += getFilePathSize(entry);
250                            continue;
251                        }
252                        FileUtils.deleteDirectory(entry.toFile());
253                    } catch (IOException e) {
254                        log.error("Error while performing GC", e);
255                    }
256                }
257            }
258        } catch (IOException e) {
259            log.error("Error while performing GC", e);
260        }
261        setStorageSize(newSize);
262    }
263
264    protected String getKeyCachingDirName(String dir) {
265        String key = dir.replaceAll("_", "/");
266        return new String(Base64.decodeBase64(key));
267    }
268
269    protected long getFilePathSize(Path entry) {
270        long size = 0;
271        for (File file : entry.toFile().listFiles()) {
272            size += file.length();
273        }
274        return size;
275    }
276
277    @Override
278    public void removeAll() {
279        log.debug("Removing all entries from TransientStore " + config.getName());
280        removeAllEntries();
281        doGC();
282    }
283
284}