001/*
002 * (C) Copyright 2011-2015 Nuxeo SA (http://nuxeo.com/) and contributors.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the GNU Lesser General Public License
006 * (LGPL) version 2.1 which accompanies this distribution, and is available at
007 * http://www.gnu.org/licenses/lgpl-2.1.html
008 *
009 * This library is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 * Lesser General Public License for more details.
013 *
014 * Contributors:
015 *     tdelprat
016 *
017 */
018
019package org.nuxeo.wizard.download;
020
021import java.io.File;
022import java.io.FileInputStream;
023import java.io.FileNotFoundException;
024import java.io.FileReader;
025import java.io.FileWriter;
026import java.io.IOException;
027import java.io.InputStream;
028import java.security.MessageDigest;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.List;
033import java.util.Properties;
034import java.util.concurrent.CopyOnWriteArrayList;
035import java.util.concurrent.LinkedBlockingQueue;
036import java.util.concurrent.ThreadFactory;
037import java.util.concurrent.ThreadPoolExecutor;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.atomic.AtomicInteger;
040
041import org.apache.commons.lang.StringUtils;
042import org.apache.commons.logging.Log;
043import org.apache.commons.logging.LogFactory;
044import org.apache.http.Header;
045import org.apache.http.HttpHost;
046import org.apache.http.HttpResponse;
047import org.apache.http.auth.AuthScope;
048import org.apache.http.auth.NTCredentials;
049import org.apache.http.auth.UsernamePasswordCredentials;
050import org.apache.http.client.methods.HttpGet;
051import org.apache.http.conn.params.ConnRoutePNames;
052import org.apache.http.conn.scheme.PlainSocketFactory;
053import org.apache.http.conn.scheme.Scheme;
054import org.apache.http.conn.scheme.SchemeRegistry;
055import org.apache.http.conn.ssl.SSLSocketFactory;
056import org.apache.http.impl.client.DefaultHttpClient;
057import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
058import org.apache.http.params.BasicHttpParams;
059import org.apache.http.params.HttpParams;
060import org.apache.http.params.HttpProtocolParams;
061import org.apache.http.util.EntityUtils;
062
063import org.nuxeo.common.Environment;
064import org.nuxeo.common.utils.FileUtils;
065import org.nuxeo.launcher.config.ConfigurationGenerator;
066
067/**
068 * @author Tiry (tdelprat@nuxeo.com)
069 */
070public class PackageDownloader {
071
072    protected final static Log log = LogFactory.getLog(PackageDownloader.class);
073
074    public static final String PACKAGES_XML = "packages.xml";
075
076    public static final String PACKAGES_DEFAULT_SELECTION = "packages-default-selection.properties";
077
078    public static final String PACKAGES_DEFAULT_SELECTION_PRESETS = "preset";
079
080    public static final String PACKAGES_DEFAULT_SELECTION_PACKAGES = "packages";
081
082    protected static final int NB_DOWNLOAD_THREADS = 3;
083
084    protected static final int NB_CHECK_THREADS = 1;
085
086    protected static final int QUEUESIZE = 20;
087
088    public static final String BASE_URL_KEY = "nuxeo.wizard.packages.url";
089
090    public static final String DEFAULT_BASE_URL = "http://cdn.nuxeo.com/"; // nuxeo-XXX/mp
091
092    protected CopyOnWriteArrayList<PendingDownload> pendingDownloads = new CopyOnWriteArrayList<>();
093
094    protected static PackageDownloader instance;
095
096    protected DefaultHttpClient httpClient;
097
098    protected Boolean canReachServer = null;
099
100    protected DownloadablePackageOptions downloadOptions;
101
102    protected static final String DIGEST = "MD5";
103
104    protected static final int DIGEST_CHUNK = 1024 * 100;
105
106    boolean downloadStarted = false;
107
108    protected String lastSelectionDigest;
109
110    protected final AtomicInteger dwThreadCount = new AtomicInteger(0);
111
112    protected final AtomicInteger checkThreadCount = new AtomicInteger(0);
113
114    protected String baseUrl;
115
116    protected ConfigurationGenerator configurationGenerator = null;
117
118    protected ConfigurationGenerator getConfig() {
119        if (configurationGenerator == null) {
120            configurationGenerator = new ConfigurationGenerator();
121            configurationGenerator.init();
122        }
123        return configurationGenerator;
124    }
125
126    protected String getBaseUrl() {
127        if (baseUrl == null) {
128            String base = getConfig().getUserConfig().getProperty(BASE_URL_KEY, "");
129            if ("".equals(base)) {
130                base = DEFAULT_BASE_URL + "nuxeo-"
131                        + getConfig().getUserConfig().getProperty(Environment.DISTRIBUTION_VERSION) + "/mp/";
132            }
133            if (!base.endsWith("/")) {
134                base = base + "/";
135            }
136            baseUrl = base;
137        }
138        return baseUrl;
139    }
140
141    protected ThreadPoolExecutor download_tpe = new ThreadPoolExecutor(NB_DOWNLOAD_THREADS, NB_DOWNLOAD_THREADS, 10L,
142            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
143                @Override
144                public Thread newThread(Runnable r) {
145                    Thread t = new Thread(r);
146                    t.setDaemon(true);
147                    t.setName("DownloaderThread-" + dwThreadCount.incrementAndGet());
148                    return t;
149                }
150            });
151
152    protected ThreadPoolExecutor check_tpe = new ThreadPoolExecutor(NB_CHECK_THREADS, NB_CHECK_THREADS, 10L,
153            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
154                @Override
155                public Thread newThread(Runnable r) {
156                    Thread t = new Thread(r);
157                    t.setDaemon(true);
158                    t.setName("MD5CheckThread-" + checkThreadCount.incrementAndGet());
159                    return t;
160                }
161            });
162
163    protected PackageDownloader() {
164        SchemeRegistry registry = new SchemeRegistry();
165        registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
166        registry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
167        HttpParams httpParams = new BasicHttpParams();
168        HttpProtocolParams.setUseExpectContinue(httpParams, false);
169        ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(registry);
170        cm.setMaxTotal(NB_DOWNLOAD_THREADS);
171        cm.setDefaultMaxPerRoute(NB_DOWNLOAD_THREADS);
172        httpClient = new DefaultHttpClient(cm, httpParams);
173    }
174
175    public synchronized static PackageDownloader instance() {
176        if (instance == null) {
177            instance = new PackageDownloader();
178            instance.download_tpe.prestartAllCoreThreads();
179            instance.check_tpe.prestartAllCoreThreads();
180        }
181        return instance;
182    }
183
184    public static void reset() {
185        if (instance != null) {
186            instance.shutdown();
187            instance = null;
188        }
189    }
190
191    public void setProxy(String proxy, int port, String login, String password, String NTLMHost, String NTLMDomain) {
192        if (proxy != null) {
193            HttpHost proxyHost = new HttpHost(proxy, port);
194            httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
195            if (login != null) {
196                if (NTLMHost != null && !NTLMHost.trim().isEmpty()) {
197                    NTCredentials ntlmCredentials = new NTCredentials(login, password, NTLMHost, NTLMDomain);
198                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), ntlmCredentials);
199                } else {
200                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port),
201                            new UsernamePasswordCredentials(login, password));
202                }
203            } else {
204                httpClient.getCredentialsProvider().clear();
205            }
206        } else {
207            httpClient.getParams().removeParameter(ConnRoutePNames.DEFAULT_PROXY);
208            httpClient.getCredentialsProvider().clear();
209        }
210    }
211
212    protected String getSelectionDigest(List<String> ids) {
213        ArrayList<String> lst = new ArrayList<>(ids);
214        Collections.sort(lst);
215        StringBuffer sb = new StringBuffer();
216        for (String item : lst) {
217            sb.append(item);
218            sb.append(":");
219        }
220        return sb.toString();
221    }
222
223    public void selectOptions(List<String> ids) {
224        String newSelectionDigest = getSelectionDigest(ids);
225        if (lastSelectionDigest != null) {
226            if (lastSelectionDigest.equals(newSelectionDigest)) {
227                return;
228            }
229        }
230        getPackageOptions().select(ids);
231        downloadStarted = false;
232        lastSelectionDigest = newSelectionDigest;
233    }
234
235    protected File getDownloadDirectory() {
236        File mpDir = getConfig().getDistributionMPDir();
237        if (!mpDir.exists()) {
238            mpDir.mkdirs();
239        }
240        return mpDir;
241    }
242
243    public boolean canReachServer() {
244        if (canReachServer == null) {
245            HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
246            try {
247                HttpResponse response = httpClient.execute(ping);
248                if (response.getStatusLine().getStatusCode() == 200) {
249                    canReachServer = true;
250                } else {
251                    log.info("Unable to ping server -> status code :" + response.getStatusLine().getStatusCode() + " ("
252                            + response.getStatusLine().getReasonPhrase() + ")");
253                    canReachServer = false;
254                }
255            } catch (Exception e) {
256                log.info("Unable to ping remote server " + e.getMessage());
257                log.debug("Unable to ping remote server", e);
258                canReachServer = false;
259            }
260        }
261        return canReachServer;
262    }
263
264    public DownloadablePackageOptions getPackageOptions() {
265        if (downloadOptions == null) {
266            File packageFile = null;
267            if (canReachServer()) {
268                packageFile = getRemotePackagesDescriptor();
269            }
270            if (packageFile == null) {
271                packageFile = getLocalPackagesDescriptor();
272                if (packageFile == null) {
273                    log.warn("Unable to find local copy of packages.xml");
274                } else {
275                    log.info("Wizard will use the local copy of packages.xml.");
276                }
277            }
278            if (packageFile != null) {
279                try {
280                    downloadOptions = DownloadDescriptorParser.parsePackages(new FileInputStream(packageFile));
281
282                    // manage init from presets if available
283                    Properties defaultSelection = getDefaultPackageSelection();
284                    if (defaultSelection != null) {
285                        String presetId = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PRESETS, null);
286                        if (presetId != null && !presetId.isEmpty()) {
287                            for (Preset preset : downloadOptions.getPresets()) {
288                                if (preset.getId().equals(presetId)) {
289                                    List<String> pkgIds = Arrays.asList(preset.getPkgs());
290                                    downloadOptions.select(pkgIds);
291                                    break;
292                                }
293                            }
294                        } else {
295                            String pkgIdsList = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PACKAGES, null);
296                            if (pkgIdsList != null && !pkgIdsList.isEmpty()) {
297                                String[] ids = pkgIdsList.split(",");
298                                List<String> pkgIds = Arrays.asList(ids);
299                                downloadOptions.select(pkgIds);
300                            }
301                        }
302                    }
303                } catch (FileNotFoundException e) {
304                    log.error("Unable to read packages.xml", e);
305                }
306            }
307        }
308        return downloadOptions;
309    }
310
311    protected File getRemotePackagesDescriptor() {
312        File desc = null;
313        HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
314        try {
315            HttpResponse response = httpClient.execute(ping);
316            if (response.getStatusLine().getStatusCode() == 200) {
317                desc = new File(getDownloadDirectory(), PACKAGES_XML);
318                FileUtils.copyToFile(response.getEntity().getContent(), desc);
319            } else {
320                log.warn("Unable to download remote packages.xml, status code :"
321                        + response.getStatusLine().getStatusCode() + " (" + response.getStatusLine().getReasonPhrase()
322                        + ")");
323                return null;
324            }
325        } catch (Exception e) {
326            log.warn("Unable to reach remote packages.xml", e);
327            return null;
328        }
329        return desc;
330    }
331
332    protected Properties getDefaultPackageSelection() {
333        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
334        if (desc.exists()) {
335            try {
336                Properties props = new Properties();
337                props.load(new FileReader(desc));
338                return props;
339            } catch (IOException e) {
340                log.warn("Unable to load presets", e);
341            }
342        }
343        return null;
344    }
345
346    protected void saveSelectedPackages(List<DownloadPackage> pkgs) {
347        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
348        Properties props = new Properties();
349        StringBuffer sb = new StringBuffer();
350        for (int i = 0; i < pkgs.size(); i++) {
351            if (i > 0) {
352                sb.append(",");
353            }
354            sb.append(pkgs.get(i).getId());
355        }
356        props.put(PACKAGES_DEFAULT_SELECTION_PACKAGES, sb.toString());
357        try {
358            props.store(new FileWriter(desc), "Saved from Nuxeo SetupWizard");
359        } catch (IOException e) {
360            log.error("Unable to save package selection", e);
361        }
362    }
363
364    protected File getLocalPackagesDescriptor() {
365        File desc = new File(getDownloadDirectory(), PACKAGES_XML);
366        if (desc.exists()) {
367            return desc;
368        }
369        return null;
370    }
371
372    public List<DownloadPackage> getSelectedPackages() {
373        List<DownloadPackage> pkgs = getPackageOptions().getPkg4Install();
374        File[] listFiles = getDownloadDirectory().listFiles();
375        for (DownloadPackage pkg : pkgs) {
376            for (File file : listFiles) {
377                if (file.getName().equals(pkg.getMd5())) {
378                    // recheck md5 ???
379                    pkg.setLocalFile(file);
380                }
381            }
382            needToDownload(pkg);
383        }
384        return pkgs;
385    }
386
387    public void scheduleDownloadedPackagesForInstallation(String installationFilePath) throws IOException {
388        List<String> fileEntries = new ArrayList<>();
389        fileEntries.add("init");
390
391        List<DownloadPackage> pkgs = downloadOptions.getPkg4Install();
392        List<String> pkgInstallIds = new ArrayList<>();
393        for (DownloadPackage pkg : pkgs) {
394            if (pkg.isVirtual()) {
395                log.debug("No install for virtual package: " + pkg.getId());
396            } else if (pkg.isAlreadyInLocal() || StringUtils.isBlank(pkg.getFilename())) {
397                // Blank filename means later downloaded
398                fileEntries.add("install " + pkg.getId());
399                pkgInstallIds.add(pkg.getId());
400            } else {
401                for (PendingDownload download : pendingDownloads) {
402                    if (download.getPkg().equals(pkg)) {
403                        if (download.getStatus() == PendingDownloadStatus.VERIFIED) {
404                            File file = download.getDowloadingFile();
405                            fileEntries.add("add file:" + file.getAbsolutePath());
406                            fileEntries.add("install " + pkg.getId());
407                            pkgInstallIds.add(pkg.getId());
408                        } else {
409                            log.error("One selected package has not been downloaded : " + pkg.getId());
410                        }
411                    }
412                }
413            }
414        }
415
416        File installLog = new File(installationFilePath);
417        if (fileEntries.size() > 0) {
418            if (!installLog.exists()) {
419                File parent = installLog.getParentFile();
420                if (!parent.exists()) {
421                    parent.mkdirs();
422                }
423                installLog.createNewFile();
424            }
425            FileUtils.writeLines(installLog, fileEntries);
426        } else {
427            // Should not happen as the file always has "init"
428            if (installLog.exists()) {
429                installLog.delete();
430            }
431        }
432
433        // Save presets
434        saveSelectedPackages(pkgs);
435    }
436
437    public List<PendingDownload> getPendingDownloads() {
438        return pendingDownloads;
439    }
440
441    public void reStartDownload(String id) {
442        for (PendingDownload pending : pendingDownloads) {
443            if (pending.getPkg().getId().equals(id)) {
444                if (Arrays.asList(PendingDownloadStatus.CORRUPTED, PendingDownloadStatus.ABORTED).contains(
445                        pending.getStatus())) {
446                    pendingDownloads.remove(pending);
447                    startDownloadPackage(pending.getPkg());
448                }
449                break;
450            }
451        }
452    }
453
454    public void startDownload() {
455        startDownload(downloadOptions.getPkg4Install());
456    }
457
458    public void startDownload(List<DownloadPackage> pkgs) {
459        downloadStarted = true;
460        for (DownloadPackage pkg : pkgs) {
461            if (needToDownload(pkg)) {
462                startDownloadPackage(pkg);
463            }
464        }
465    }
466
467    protected boolean needToDownload(DownloadPackage pkg) {
468        return !pkg.isVirtual() && !pkg.isLaterDownload() && !pkg.isAlreadyInLocal();
469    }
470
471    protected void startDownloadPackage(final DownloadPackage pkg) {
472        final PendingDownload download = new PendingDownload(pkg);
473        if (pendingDownloads.addIfAbsent(download)) {
474            Runnable downloadRunner = new Runnable() {
475
476                @Override
477                public void run() {
478                    log.info("Starting download on Thread " + Thread.currentThread().getName());
479                    download.setStatus(PendingDownloadStatus.INPROGRESS);
480                    String url = pkg.getDownloadUrl();
481                    if (!url.startsWith("http")) {
482                        url = getBaseUrl() + url;
483                    }
484                    File filePkg = null;
485                    HttpGet dw = new HttpGet(url);
486                    try {
487                        HttpResponse response = httpClient.execute(dw);
488                        if (response.getStatusLine().getStatusCode() == 200) {
489                            filePkg = new File(getDownloadDirectory(), pkg.filename);
490                            Header clh = response.getFirstHeader("Content-Length");
491                            if (clh != null) {
492                                long filesize = Long.parseLong(clh.getValue());
493                                download.setFile(filesize, filePkg);
494                            }
495                            FileUtils.copyToFile(response.getEntity().getContent(), filePkg);
496                            download.setStatus(PendingDownloadStatus.COMPLETED);
497                        } else if (response.getStatusLine().getStatusCode() == 404) {
498                            log.error("Package " + pkg.filename + " not found :" + url);
499                            download.setStatus(PendingDownloadStatus.MISSING);
500                            EntityUtils.consume(response.getEntity());
501                            dw.abort();
502                            return;
503                        } else {
504                            log.error("Received StatusCode " + response.getStatusLine().getStatusCode());
505                            download.setStatus(PendingDownloadStatus.ABORTED);
506                            EntityUtils.consume(response.getEntity());
507                            dw.abort();
508                            return;
509                        }
510                    } catch (Exception e) {
511                        download.setStatus(PendingDownloadStatus.ABORTED);
512                        log.error("Error during download", e);
513                        return;
514                    }
515                    checkPackage(download);
516                }
517            };
518            download_tpe.execute(downloadRunner);
519        }
520    }
521
522    protected void checkPackage(final PendingDownload download) {
523        final File filePkg = download.getDowloadingFile();
524        Runnable checkRunner = new Runnable() {
525            @Override
526            public void run() {
527                download.setStatus(PendingDownloadStatus.VERIFICATION);
528                String expectedDigest = download.getPkg().getMd5();
529                String digest = getDigest(filePkg);
530                if (digest == null || (expectedDigest != null && !expectedDigest.equals(digest))) {
531                    download.setStatus(PendingDownloadStatus.CORRUPTED);
532                    log.error("Digest check failed: expected=" + expectedDigest + " computed=" + digest);
533                    return;
534                }
535                File newFile = new File(getDownloadDirectory(), digest);
536                filePkg.renameTo(newFile);
537                download.setStatus(PendingDownloadStatus.VERIFIED);
538                download.setFile(newFile.length(), newFile);
539            }
540        };
541        check_tpe.execute(checkRunner);
542    }
543
544    protected String getDigest(File file) {
545        try {
546            MessageDigest md = MessageDigest.getInstance(DIGEST);
547            byte[] buffer = new byte[DIGEST_CHUNK];
548            InputStream stream = new FileInputStream(file);
549            int bytesRead = -1;
550            while ((bytesRead = stream.read(buffer)) >= 0) {
551                md.update(buffer, 0, bytesRead);
552            }
553            stream.close();
554            byte[] b = md.digest();
555            return md5ToHex(b);
556        } catch (Exception e) {
557            log.error("Error while computing Digest ", e);
558            return null;
559        }
560    }
561
562    protected static String md5ToHex(byte[] hash) {
563        StringBuilder hexString = new StringBuilder();
564        for (byte b : hash) {
565            String hex = Integer.toHexString(0xFF & b);
566            if (hex.length() == 1) {
567                hexString.append('0');
568            }
569            hexString.append(hex);
570        }
571        return hexString.toString();
572    }
573
574    public boolean isDownloadStarted() {
575        return downloadStarted;
576    }
577
578    public boolean isDownloadCompleted() {
579        if (!isDownloadStarted()) {
580            return false;
581        }
582        for (PendingDownload download : pendingDownloads) {
583            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()) {
584                return false;
585            }
586        }
587        return true;
588    }
589
590    public boolean isDownloadInProgress() {
591        if (!isDownloadStarted()) {
592            return false;
593        }
594        if (isDownloadCompleted()) {
595            return false;
596        }
597        int nbInProgress = 0;
598        for (PendingDownload download : pendingDownloads) {
599            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()
600                    && download.getStatus().getValue() >= PendingDownloadStatus.PENDING.getValue()) {
601                nbInProgress++;
602            }
603        }
604        return nbInProgress > 0;
605    }
606
607    public void shutdown() {
608        if (httpClient != null) {
609            httpClient.getConnectionManager().shutdown();
610        }
611        download_tpe.shutdownNow();
612        check_tpe.shutdownNow();
613    }
614
615}