001/*
002 * (C) Copyright 2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Thierry Delprat <tdelprat@nuxeo.com>
018 *     Antoine Taillefer <ataillefer@nuxeo.com>
019 */
020
021package org.nuxeo.ecm.core.transientstore;
022
023import java.io.File;
024import java.io.IOException;
025import java.io.Serializable;
026import java.nio.file.DirectoryStream;
027import java.nio.file.Files;
028import java.nio.file.Path;
029import java.nio.file.Paths;
030import java.util.ArrayList;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034import java.util.UUID;
035
036import org.apache.commons.codec.binary.Base64;
037import org.apache.commons.io.FileUtils;
038import org.apache.commons.lang3.StringUtils;
039import org.apache.commons.logging.Log;
040import org.apache.commons.logging.LogFactory;
041import org.nuxeo.common.Environment;
042import org.nuxeo.ecm.core.api.Blob;
043import org.nuxeo.ecm.core.api.NuxeoException;
044import org.nuxeo.ecm.core.api.impl.blob.FileBlob;
045import org.nuxeo.ecm.core.transientstore.api.MaximumTransientSpaceExceeded;
046import org.nuxeo.ecm.core.transientstore.api.TransientStore;
047import org.nuxeo.ecm.core.transientstore.api.TransientStoreConfig;
048import org.nuxeo.ecm.core.transientstore.api.TransientStoreProvider;
049
050/**
051 * Base class for a {@link TransientStore} implementation.
052 *
053 * @since 7.2
054 */
055public abstract class AbstractTransientStore implements TransientStoreProvider {
056
057    protected static final Log log = LogFactory.getLog(AbstractTransientStore.class);
058
059    protected TransientStoreConfig config;
060
061    protected File cacheDir;
062
063    @Override
064    public void init(TransientStoreConfig config) {
065        this.config = config;
066        File data = getDataDir(config);
067        data.mkdirs();
068        cacheDir = data.getAbsoluteFile();
069    }
070
071    private File getDataDir(TransientStoreConfig config) {
072        String dataDirPath = config.getDataDir();
073        if (StringUtils.isBlank(dataDirPath)) {
074            File transienStoreHome = new File(Environment.getDefault().getData(), "transientstores");
075            return new File(transienStoreHome, config.getName());
076        } else {
077            return new File(dataDirPath);
078        }
079    }
080
081    @Override
082    public abstract void shutdown();
083
084    @Override
085    public abstract boolean exists(String key);
086
087    @Override
088    public abstract void putParameter(String key, String parameter, Serializable value);
089
090    @Override
091    public abstract Serializable getParameter(String key, String parameter);
092
093    @Override
094    public abstract void putParameters(String key, Map<String, Serializable> parameters);
095
096    @Override
097    public abstract Map<String, Serializable> getParameters(String key);
098
099    @Override
100    public abstract List<Blob> getBlobs(String key);
101
102    @Override
103    public abstract long getSize(String key);
104
105    @Override
106    public abstract boolean isCompleted(String key);
107
108    @Override
109    public abstract void setCompleted(String key, boolean completed);
110
111    @Override
112    public abstract void release(String key);
113
114    /**
115     * Updates the total storage size and the storage size of the entry with the given {@code key} according to
116     * {@code sizeOfBlobs} and stores the blob information in this entry.
117     */
118    protected abstract void persistBlobs(String key, long sizeOfBlobs, List<Map<String, String>> blobInfos);
119
120    /**
121     * Sets the size of the disk storage in bytes.
122     */
123    protected abstract void setStorageSize(long newSize);
124
125    protected abstract long incrementStorageSize(long size);
126
127    protected abstract long decrementStorageSize(long size);
128
129    protected abstract void removeAllEntries();
130
131    protected abstract void removeEntry(String key);
132
133    @Override
134    public void putBlobs(String key, List<Blob> blobs) {
135        if (config.getAbsoluteMaxSizeMB() < 0 || getStorageSize() < config.getAbsoluteMaxSizeMB() * (1024 * 1024)) {
136            // Store blobs on the file system
137            List<Map<String, String>> blobInfos = storeBlobs(key, blobs);
138            // Persist blob information in the store
139            persistBlobs(key, getSizeOfBlobs(blobs), blobInfos);
140        } else {
141            throw new MaximumTransientSpaceExceeded();
142        }
143    }
144
145    protected List<Map<String, String>> storeBlobs(String key, List<Blob> blobs) {
146        if (blobs == null) {
147            return null;
148        }
149        // Store blobs on the file system and compute blob information
150        List<Map<String, String>> blobInfos = new ArrayList<>();
151        for (Blob blob : blobs) {
152            Map<String, String> blobInfo = new HashMap<>();
153            File cachingDir = getCachingDirectory(key);
154            String uuid = UUID.randomUUID().toString();
155            File cachedFile = new File(cachingDir, uuid);
156            try {
157                if (blob instanceof FileBlob && ((FileBlob) blob).isTemporary()) {
158                    ((FileBlob) blob).moveTo(cachedFile);
159                } else {
160                    blob.transferTo(cachedFile);
161                }
162            } catch (IOException e) {
163                throw new NuxeoException(e);
164            }
165            Path cachedFileRelativePath = Paths.get(cachingDir.getName(), uuid);
166            blobInfo.put("file", cachedFileRelativePath.toString());
167            // Redis doesn't support null values
168            if (blob.getFilename() != null) {
169                blobInfo.put("filename", blob.getFilename());
170            }
171            if (blob.getEncoding() != null) {
172                blobInfo.put("encoding", blob.getEncoding());
173            }
174            if (blob.getMimeType() != null) {
175                blobInfo.put("mimetype", blob.getMimeType());
176            }
177            if (blob.getDigest() != null) {
178                blobInfo.put("digest", blob.getDigest());
179            }
180            blobInfos.add(blobInfo);
181        }
182        log.debug("Stored blobs on the file system: " + blobInfos);
183        return blobInfos;
184    }
185
186    public File getCachingDirectory(String key) {
187        String cachingDirName = getCachingDirName(key);
188        try {
189            File cachingDir = new File(cacheDir.getCanonicalFile(), cachingDirName);
190            if (!cachingDir.getCanonicalPath().startsWith(cacheDir.getCanonicalPath())) {
191                throw new NuxeoException("Trying to traverse illegal path: " + cachingDir + " for key: " + key);
192            }
193            if (!cachingDir.exists()) {
194                cachingDir.mkdir();
195            }
196            return cachingDir;
197        } catch (IOException e) {
198            throw new NuxeoException("Error when trying to access cache directory: " + cacheDir + "/" + cachingDirName
199                    + " for key: " + key, e);
200        }
201    }
202
203    protected String getCachingDirName(String key) {
204        String dirName = Base64.encodeBase64String(key.getBytes());
205        dirName = dirName.replaceAll("/", "_");
206        return dirName;
207    }
208
209    protected long getSizeOfBlobs(List<Blob> blobs) {
210        int size = 0;
211        if (blobs != null) {
212            for (Blob blob : blobs) {
213                long blobLength = blob.getLength();
214                if (blobLength > -1) {
215                    size += blobLength;
216                }
217            }
218        }
219        return size;
220    }
221
222    protected List<Blob> loadBlobs(List<Map<String, String>> blobInfos) {
223        log.debug("Loading blobs from the file system: " + blobInfos);
224        List<Blob> blobs = new ArrayList<>();
225        for (Map<String, String> info : blobInfos) {
226            File blobFile = new File(cacheDir, info.get("file"));
227            Blob blob = new FileBlob(blobFile);
228            blob.setEncoding(info.get("encoding"));
229            blob.setMimeType(info.get("mimetype"));
230            blob.setFilename(info.get("filename"));
231            blob.setDigest(info.get("digest"));
232            blobs.add(blob);
233        }
234        return blobs;
235    }
236
237    @Override
238    public void doGC() {
239        log.debug(String.format("Performing GC for TransientStore %s", config.getName()));
240        long newSize = 0;
241        try {
242            try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(cacheDir.getAbsolutePath()))) {
243                for (Path entry : stream) {
244                    String key = getKeyCachingDirName(entry.getFileName().toString());
245                    if (exists(key)) {
246                        newSize += getFilePathSize(entry);
247                        continue;
248                    }
249                    FileUtils.deleteQuietly(entry.toFile());
250                }
251            }
252        } catch (IOException e) {
253            log.error("Error while performing GC", e);
254        }
255        setStorageSize(newSize);
256    }
257
258    protected String getKeyCachingDirName(String dir) {
259        String key = dir.replaceAll("_", "/");
260        return new String(Base64.decodeBase64(key));
261    }
262
263    protected long getFilePathSize(Path entry) {
264        long size = 0;
265        for (File file : entry.toFile().listFiles()) {
266            size += file.length();
267        }
268        return size;
269    }
270
271    @Override
272    public void remove(String key) {
273        removeEntry(key);
274        removeBlobs(key);
275    }
276
277    protected void removeBlobs(String key) {
278        FileUtils.deleteQuietly(getCachingDirectory(key));
279    }
280
281    @Override
282    public void removeAll() {
283        log.debug("Removing all entries from TransientStore " + config.getName());
284        removeAllEntries();
285        doGC();
286    }
287
288    public File getCacheDir() {
289        return cacheDir;
290    }
291
292}