001/*
002 * (C) Copyright 2011-2016 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     tdelprat
018 */
019package org.nuxeo.wizard.download;
020
021import java.io.File;
022import java.io.FileInputStream;
023import java.io.FileNotFoundException;
024import java.io.FileReader;
025import java.io.FileWriter;
026import java.io.IOException;
027import java.io.InputStream;
028import java.security.MessageDigest;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.List;
033import java.util.Properties;
034import java.util.concurrent.CopyOnWriteArrayList;
035import java.util.concurrent.LinkedBlockingQueue;
036import java.util.concurrent.ThreadFactory;
037import java.util.concurrent.ThreadPoolExecutor;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.atomic.AtomicInteger;
040import java.util.stream.Collectors;
041
042import org.apache.commons.io.FileUtils;
043import org.apache.commons.lang3.StringUtils;
044import org.apache.commons.logging.Log;
045import org.apache.commons.logging.LogFactory;
046import org.apache.http.Header;
047import org.apache.http.HttpHost;
048import org.apache.http.HttpResponse;
049import org.apache.http.auth.AuthScope;
050import org.apache.http.auth.NTCredentials;
051import org.apache.http.auth.UsernamePasswordCredentials;
052import org.apache.http.client.methods.HttpGet;
053import org.apache.http.conn.params.ConnRoutePNames;
054import org.apache.http.conn.scheme.PlainSocketFactory;
055import org.apache.http.conn.scheme.Scheme;
056import org.apache.http.conn.scheme.SchemeRegistry;
057import org.apache.http.conn.ssl.SSLSocketFactory;
058import org.apache.http.impl.client.DefaultHttpClient;
059import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
060import org.apache.http.params.BasicHttpParams;
061import org.apache.http.params.HttpParams;
062import org.apache.http.params.HttpProtocolParams;
063import org.apache.http.util.EntityUtils;
064import org.nuxeo.common.Environment;
065import org.nuxeo.launcher.config.ConfigurationGenerator;
066
067/**
068 * @author Tiry (tdelprat@nuxeo.com)
069 */
070public class PackageDownloader {
071
072    protected final static Log log = LogFactory.getLog(PackageDownloader.class);
073
074    public static final String PACKAGES_XML = "packages.xml";
075
076    public static final String PACKAGES_DEFAULT_SELECTION = "packages-default-selection.properties";
077
078    public static final String PACKAGES_DEFAULT_SELECTION_PRESETS = "preset";
079
080    public static final String PACKAGES_DEFAULT_SELECTION_PACKAGES = "packages";
081
082    protected static final int NB_DOWNLOAD_THREADS = 3;
083
084    protected static final int NB_CHECK_THREADS = 1;
085
086    protected static final int QUEUESIZE = 20;
087
088    public static final String BASE_URL_KEY = "nuxeo.wizard.packages.url";
089
090    public static final String DEFAULT_BASE_URL = "http://cdn.nuxeo.com/"; // nuxeo-XXX/mp
091
092    protected CopyOnWriteArrayList<PendingDownload> pendingDownloads = new CopyOnWriteArrayList<>();
093
094    protected static PackageDownloader instance;
095
096    protected DefaultHttpClient httpClient;
097
098    protected Boolean canReachServer = null;
099
100    protected DownloadablePackageOptions downloadOptions;
101
102    protected static final String DIGEST = "MD5";
103
104    protected static final int DIGEST_CHUNK = 1024 * 100;
105
106    boolean downloadStarted = false;
107
108    protected String lastSelectionDigest;
109
110    protected final AtomicInteger dwThreadCount = new AtomicInteger(0);
111
112    protected final AtomicInteger checkThreadCount = new AtomicInteger(0);
113
114    protected String baseUrl;
115
116    protected ConfigurationGenerator configurationGenerator = null;
117
118    protected ConfigurationGenerator getConfig() {
119        if (configurationGenerator == null) {
120            configurationGenerator = new ConfigurationGenerator();
121            configurationGenerator.init();
122        }
123        return configurationGenerator;
124    }
125
126    protected String getBaseUrl() {
127        if (baseUrl == null) {
128            String base = getConfig().getUserConfig().getProperty(BASE_URL_KEY, "");
129            if ("".equals(base)) {
130                base = DEFAULT_BASE_URL + "nuxeo-"
131                        + getConfig().getUserConfig().getProperty(Environment.DISTRIBUTION_VERSION) + "/mp/";
132            }
133            if (!base.endsWith("/")) {
134                base = base + "/";
135            }
136            baseUrl = base;
137        }
138        return baseUrl;
139    }
140
141    protected ThreadPoolExecutor download_tpe = new ThreadPoolExecutor(NB_DOWNLOAD_THREADS, NB_DOWNLOAD_THREADS, 10L,
142            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
143                @Override
144                public Thread newThread(Runnable r) {
145                    Thread t = new Thread(r);
146                    t.setDaemon(true);
147                    t.setName("DownloaderThread-" + dwThreadCount.incrementAndGet());
148                    return t;
149                }
150            });
151
152    protected ThreadPoolExecutor check_tpe = new ThreadPoolExecutor(NB_CHECK_THREADS, NB_CHECK_THREADS, 10L,
153            TimeUnit.SECONDS, new LinkedBlockingQueue<>(QUEUESIZE), r -> {
154                Thread t = new Thread(r);
155                t.setDaemon(true);
156                t.setName("MD5CheckThread-" + checkThreadCount.incrementAndGet());
157                return t;
158            });
159
160    protected PackageDownloader() {
161        SchemeRegistry registry = new SchemeRegistry();
162        registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
163        registry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
164        HttpParams httpParams = new BasicHttpParams();
165        HttpProtocolParams.setUseExpectContinue(httpParams, false);
166        ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(registry);
167        cm.setMaxTotal(NB_DOWNLOAD_THREADS);
168        cm.setDefaultMaxPerRoute(NB_DOWNLOAD_THREADS);
169        httpClient = new DefaultHttpClient(cm, httpParams);
170    }
171
172    public synchronized static PackageDownloader instance() {
173        if (instance == null) {
174            instance = new PackageDownloader();
175            instance.download_tpe.prestartAllCoreThreads();
176            instance.check_tpe.prestartAllCoreThreads();
177        }
178        return instance;
179    }
180
181    public static void reset() {
182        if (instance != null) {
183            instance.shutdown();
184            instance = null;
185        }
186    }
187
188    public void setProxy(String proxy, int port, String login, String password, String NTLMHost, String NTLMDomain) {
189        if (proxy != null) {
190            HttpHost proxyHost = new HttpHost(proxy, port);
191            httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
192            if (login != null) {
193                if (NTLMHost != null && !NTLMHost.trim().isEmpty()) {
194                    NTCredentials ntlmCredentials = new NTCredentials(login, password, NTLMHost, NTLMDomain);
195                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), ntlmCredentials);
196                } else {
197                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port),
198                            new UsernamePasswordCredentials(login, password));
199                }
200            } else {
201                httpClient.getCredentialsProvider().clear();
202            }
203        } else {
204            httpClient.getParams().removeParameter(ConnRoutePNames.DEFAULT_PROXY);
205            httpClient.getCredentialsProvider().clear();
206        }
207    }
208
209    protected String getSelectionDigest(List<String> ids) {
210        List<String> lst = new ArrayList<>(ids);
211        Collections.sort(lst);
212        StringBuilder sb = new StringBuilder();
213        for (String item : lst) {
214            sb.append(item);
215            sb.append(":");
216        }
217        return sb.toString();
218    }
219
220    public void selectOptions(List<String> ids) {
221        String newSelectionDigest = getSelectionDigest(ids);
222        if (lastSelectionDigest != null) {
223            if (lastSelectionDigest.equals(newSelectionDigest)) {
224                return;
225            }
226        }
227        getPackageOptions().select(ids);
228        downloadStarted = false;
229        lastSelectionDigest = newSelectionDigest;
230    }
231
232    protected File getDownloadDirectory() {
233        File mpDir = getConfig().getDistributionMPDir();
234        if (!mpDir.exists()) {
235            mpDir.mkdirs();
236        }
237        return mpDir;
238    }
239
240    public boolean canReachServer() {
241        if (canReachServer == null) {
242            HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
243            try {
244                HttpResponse response = httpClient.execute(ping);
245                if (response.getStatusLine().getStatusCode() == 200) {
246                    canReachServer = true;
247                } else {
248                    log.info("Unable to ping server -> status code :" + response.getStatusLine().getStatusCode() + " ("
249                            + response.getStatusLine().getReasonPhrase() + ")");
250                    canReachServer = false;
251                }
252            } catch (Exception e) {
253                log.info("Unable to ping remote server " + e.getMessage());
254                log.debug("Unable to ping remote server", e);
255                canReachServer = false;
256            }
257        }
258        return canReachServer;
259    }
260
261    public DownloadablePackageOptions getPackageOptions() {
262        if (downloadOptions == null) {
263            File packageFile = null;
264            if (canReachServer()) {
265                packageFile = getRemotePackagesDescriptor();
266            }
267            if (packageFile == null) {
268                packageFile = getLocalPackagesDescriptor();
269                if (packageFile == null) {
270                    log.warn("Unable to find local copy of packages.xml");
271                } else {
272                    log.info("Wizard will use the local copy of packages.xml.");
273                }
274            }
275            if (packageFile != null) {
276                try {
277                    downloadOptions = DownloadDescriptorParser.parsePackages(new FileInputStream(packageFile));
278
279                    // manage init from presets if available
280                    Properties defaultSelection = getDefaultPackageSelection();
281                    if (defaultSelection != null) {
282                        String presetId = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PRESETS, null);
283                        if (presetId != null && !presetId.isEmpty()) {
284                            for (Preset preset : downloadOptions.getPresets()) {
285                                if (preset.getId().equals(presetId)) {
286                                    List<String> pkgIds = Arrays.asList(preset.getPkgs());
287                                    downloadOptions.select(pkgIds);
288                                    break;
289                                }
290                            }
291                        } else {
292                            String pkgIdsList = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PACKAGES, null);
293                            if (pkgIdsList != null && !pkgIdsList.isEmpty()) {
294                                String[] ids = pkgIdsList.split(",");
295                                List<String> pkgIds = Arrays.asList(ids);
296                                downloadOptions.select(pkgIds);
297                            }
298                        }
299                    }
300                } catch (FileNotFoundException e) {
301                    log.error("Unable to read packages.xml", e);
302                }
303            }
304        }
305        return downloadOptions;
306    }
307
308    protected File getRemotePackagesDescriptor() {
309        File desc;
310        HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
311        try {
312            HttpResponse response = httpClient.execute(ping);
313            if (response.getStatusLine().getStatusCode() == 200) {
314                desc = new File(getDownloadDirectory(), PACKAGES_XML);
315                FileUtils.copyInputStreamToFile(response.getEntity().getContent(), desc);
316            } else {
317                log.warn("Unable to download remote packages.xml, status code :"
318                        + response.getStatusLine().getStatusCode() + " (" + response.getStatusLine().getReasonPhrase()
319                        + ")");
320                return null;
321            }
322        } catch (Exception e) {
323            log.warn("Unable to reach remote packages.xml", e);
324            return null;
325        }
326        return desc;
327    }
328
329    protected Properties getDefaultPackageSelection() {
330        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
331        if (desc.exists()) {
332            try {
333                Properties props = new Properties();
334                props.load(new FileReader(desc));
335                return props;
336            } catch (IOException e) {
337                log.warn("Unable to load presets", e);
338            }
339        }
340        return null;
341    }
342
343    protected void saveSelectedPackages(List<DownloadPackage> pkgs) {
344        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
345        String defaultSelPackages = pkgs.stream().map(DownloadPackage::getId).collect(Collectors.joining(","));
346        Properties props = new Properties();
347        props.put(PACKAGES_DEFAULT_SELECTION_PACKAGES, defaultSelPackages);
348        try {
349            props.store(new FileWriter(desc), "Saved from Nuxeo SetupWizard");
350        } catch (IOException e) {
351            log.error("Unable to save package selection", e);
352        }
353    }
354
355    protected File getLocalPackagesDescriptor() {
356        File desc = new File(getDownloadDirectory(), PACKAGES_XML);
357        if (desc.exists()) {
358            return desc;
359        }
360        return null;
361    }
362
363    public List<DownloadPackage> getSelectedPackages() {
364        List<DownloadPackage> pkgs = getPackageOptions().getPkg4Install();
365        File[] listFiles = getDownloadDirectory().listFiles();
366        for (DownloadPackage pkg : pkgs) {
367            for (File file : listFiles) {
368                if (file.getName().equals(pkg.getMd5())) {
369                    // recheck md5 ???
370                    pkg.setLocalFile(file);
371                }
372            }
373            needToDownload(pkg);
374        }
375        return pkgs;
376    }
377
378    public void scheduleDownloadedPackagesForInstallation(String installationFilePath) throws IOException {
379        List<String> fileEntries = new ArrayList<>();
380        fileEntries.add("init");
381
382        List<DownloadPackage> pkgs = downloadOptions.getPkg4Install();
383        List<String> pkgInstallIds = new ArrayList<>();
384        for (DownloadPackage pkg : pkgs) {
385            if (pkg.isVirtual()) {
386                log.debug("No install for virtual package: " + pkg.getId());
387            } else if (pkg.isAlreadyInLocal() || StringUtils.isBlank(pkg.getFilename())) {
388                // Blank filename means later downloaded
389                fileEntries.add("install " + pkg.getId());
390                pkgInstallIds.add(pkg.getId());
391            } else {
392                for (PendingDownload download : pendingDownloads) {
393                    if (download.getPkg().equals(pkg)) {
394                        if (download.getStatus() == PendingDownloadStatus.VERIFIED) {
395                            File file = download.getDowloadingFile();
396                            fileEntries.add("add file:" + file.getAbsolutePath());
397                            fileEntries.add("install " + pkg.getId());
398                            pkgInstallIds.add(pkg.getId());
399                        } else {
400                            log.error("One selected package has not been downloaded : " + pkg.getId());
401                        }
402                    }
403                }
404            }
405        }
406
407        File installLog = new File(installationFilePath);
408        if (fileEntries.size() > 0) {
409            if (!installLog.exists()) {
410                File parent = installLog.getParentFile();
411                if (!parent.exists()) {
412                    parent.mkdirs();
413                }
414                installLog.createNewFile();
415            }
416            FileUtils.writeLines(installLog, fileEntries);
417        } else {
418            // Should not happen as the file always has "init"
419            if (installLog.exists()) {
420                installLog.delete();
421            }
422        }
423
424        // Save presets
425        saveSelectedPackages(pkgs);
426    }
427
428    public List<PendingDownload> getPendingDownloads() {
429        return pendingDownloads;
430    }
431
432    public void reStartDownload(String id) {
433        for (PendingDownload pending : pendingDownloads) {
434            if (pending.getPkg().getId().equals(id)) {
435                if (Arrays.asList(PendingDownloadStatus.CORRUPTED, PendingDownloadStatus.ABORTED).contains(
436                        pending.getStatus())) {
437                    pendingDownloads.remove(pending);
438                    startDownloadPackage(pending.getPkg());
439                }
440                break;
441            }
442        }
443    }
444
445    public void startDownload() {
446        startDownload(downloadOptions.getPkg4Install());
447    }
448
449    public void startDownload(List<DownloadPackage> pkgs) {
450        downloadStarted = true;
451        for (DownloadPackage pkg : pkgs) {
452            if (needToDownload(pkg)) {
453                startDownloadPackage(pkg);
454            }
455        }
456    }
457
458    protected boolean needToDownload(DownloadPackage pkg) {
459        return !pkg.isVirtual() && !pkg.isLaterDownload() && !pkg.isAlreadyInLocal();
460    }
461
462    protected void startDownloadPackage(final DownloadPackage pkg) {
463        final PendingDownload download = new PendingDownload(pkg);
464        if (pendingDownloads.addIfAbsent(download)) {
465            Runnable downloadRunner = () -> {
466                log.info("Starting download on Thread " + Thread.currentThread().getName());
467                download.setStatus(PendingDownloadStatus.INPROGRESS);
468                String url = pkg.getDownloadUrl();
469                if (!url.startsWith("http")) {
470                    url = getBaseUrl() + url;
471                }
472                File filePkg;
473                HttpGet dw = new HttpGet(url);
474                try {
475                    HttpResponse response = httpClient.execute(dw);
476                    if (response.getStatusLine().getStatusCode() == 200) {
477                        filePkg = new File(getDownloadDirectory(), pkg.filename);
478                        Header clh = response.getFirstHeader("Content-Length");
479                        if (clh != null) {
480                            long filesize = Long.parseLong(clh.getValue());
481                            download.setFile(filesize, filePkg);
482                        }
483                        FileUtils.copyInputStreamToFile(response.getEntity().getContent(), filePkg);
484                        download.setStatus(PendingDownloadStatus.COMPLETED);
485                    } else if (response.getStatusLine().getStatusCode() == 404) {
486                        log.error("Package " + pkg.filename + " not found :" + url);
487                        download.setStatus(PendingDownloadStatus.MISSING);
488                        EntityUtils.consume(response.getEntity());
489                        dw.abort();
490                        return;
491                    } else {
492                        log.error("Received StatusCode " + response.getStatusLine().getStatusCode());
493                        download.setStatus(PendingDownloadStatus.ABORTED);
494                        EntityUtils.consume(response.getEntity());
495                        dw.abort();
496                        return;
497                    }
498                } catch (Exception e) {
499                    download.setStatus(PendingDownloadStatus.ABORTED);
500                    log.error("Error during download", e);
501                    return;
502                }
503                checkPackage(download);
504            };
505            download_tpe.execute(downloadRunner);
506        }
507    }
508
509    protected void checkPackage(final PendingDownload download) {
510        final File filePkg = download.getDowloadingFile();
511        Runnable checkRunner = () -> {
512            download.setStatus(PendingDownloadStatus.VERIFICATION);
513            String expectedDigest = download.getPkg().getMd5();
514            String digest = getDigest(filePkg);
515            if (digest == null || (expectedDigest != null && !expectedDigest.equals(digest))) {
516                download.setStatus(PendingDownloadStatus.CORRUPTED);
517                log.error("Digest check failed: expected=" + expectedDigest + " computed=" + digest);
518                return;
519            }
520            File newFile = new File(getDownloadDirectory(), digest);
521            filePkg.renameTo(newFile);
522            download.setStatus(PendingDownloadStatus.VERIFIED);
523            download.setFile(newFile.length(), newFile);
524        };
525        check_tpe.execute(checkRunner);
526    }
527
528    protected String getDigest(File file) {
529        try {
530            MessageDigest md = MessageDigest.getInstance(DIGEST);
531            byte[] buffer = new byte[DIGEST_CHUNK];
532            try (InputStream stream = new FileInputStream(file)) {
533                int bytesRead;
534                while ((bytesRead = stream.read(buffer)) >= 0) {
535                    md.update(buffer, 0, bytesRead);
536                }
537            }
538            byte[] b = md.digest();
539            return md5ToHex(b);
540        } catch (Exception e) {
541            log.error("Error while computing Digest ", e);
542            return null;
543        }
544    }
545
546    protected static String md5ToHex(byte[] hash) {
547        StringBuilder hexString = new StringBuilder();
548        for (byte b : hash) {
549            String hex = Integer.toHexString(0xFF & b);
550            if (hex.length() == 1) {
551                hexString.append('0');
552            }
553            hexString.append(hex);
554        }
555        return hexString.toString();
556    }
557
558    public boolean isDownloadStarted() {
559        return downloadStarted;
560    }
561
562    public boolean isDownloadCompleted() {
563        if (!isDownloadStarted()) {
564            return false;
565        }
566        for (PendingDownload download : pendingDownloads) {
567            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()) {
568                return false;
569            }
570        }
571        return true;
572    }
573
574    public boolean isDownloadInProgress() {
575        if (!isDownloadStarted()) {
576            return false;
577        }
578        if (isDownloadCompleted()) {
579            return false;
580        }
581        int nbInProgress = 0;
582        for (PendingDownload download : pendingDownloads) {
583            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()
584                    && download.getStatus().getValue() >= PendingDownloadStatus.PENDING.getValue()) {
585                nbInProgress++;
586            }
587        }
588        return nbInProgress > 0;
589    }
590
591    public void shutdown() {
592        if (httpClient != null) {
593            httpClient.getConnectionManager().shutdown();
594        }
595        download_tpe.shutdownNow();
596        check_tpe.shutdownNow();
597    }
598
599}