Source code

001/*
002 * (C) Copyright 2011-2017 Nuxeo SA (http://nuxeo.com/) and contributors.
003 *
004 * All rights reserved. This program and the accompanying materials
005 * are made available under the terms of the GNU Lesser General Public License
006 * (LGPL) version 2.1 which accompanies this distribution, and is available at
007 * http://www.gnu.org/licenses/lgpl-2.1.html
008 *
009 * This library is distributed in the hope that it will be useful,
010 * but WITHOUT ANY WARRANTY; without even the implied warranty of
011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
012 * Lesser General Public License for more details.
013 *
014 * Contributors:
015 *     tdelprat
016 *
017 */
018
019package org.nuxeo.wizard.download;
020
021import java.io.File;
022import java.io.FileInputStream;
023import java.io.FileNotFoundException;
024import java.io.FileReader;
025import java.io.FileWriter;
026import java.io.IOException;
027import java.io.InputStream;
028import java.security.MessageDigest;
029import java.util.ArrayList;
030import java.util.Arrays;
031import java.util.Collections;
032import java.util.List;
033import java.util.Properties;
034import java.util.concurrent.CopyOnWriteArrayList;
035import java.util.concurrent.LinkedBlockingQueue;
036import java.util.concurrent.ThreadFactory;
037import java.util.concurrent.ThreadPoolExecutor;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.atomic.AtomicInteger;
040
041import org.apache.commons.lang.StringUtils;
042import org.apache.commons.logging.Log;
043import org.apache.commons.logging.LogFactory;
044import org.apache.http.Header;
045import org.apache.http.HttpHost;
046import org.apache.http.HttpResponse;
047import org.apache.http.auth.AuthScope;
048import org.apache.http.auth.NTCredentials;
049import org.apache.http.auth.UsernamePasswordCredentials;
050import org.apache.http.client.methods.HttpGet;
051import org.apache.http.conn.params.ConnRoutePNames;
052import org.apache.http.conn.scheme.PlainSocketFactory;
053import org.apache.http.conn.scheme.Scheme;
054import org.apache.http.conn.scheme.SchemeRegistry;
055import org.apache.http.conn.ssl.SSLSocketFactory;
056import org.apache.http.impl.client.DefaultHttpClient;
057import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
058import org.apache.http.params.BasicHttpParams;
059import org.apache.http.params.HttpParams;
060import org.apache.http.params.HttpProtocolParams;
061import org.apache.http.util.EntityUtils;
062import org.nuxeo.common.utils.FileUtils;
063import org.nuxeo.launcher.config.ConfigurationGenerator;
064
065/**
066 * @author Tiry (tdelprat@nuxeo.com)
067 */
068public class PackageDownloader {
069
070    protected final static Log log = LogFactory.getLog(PackageDownloader.class);
071
072    public static final String PACKAGES_XML = "packages.xml";
073
074    public static final String PACKAGES_DEFAULT_SELECTION = "packages-default-selection.properties";
075
076    public static final String PACKAGES_DEFAULT_SELECTION_PRESETS = "preset";
077
078    public static final String PACKAGES_DEFAULT_SELECTION_PACKAGES = "packages";
079
080    protected static final int NB_DOWNLOAD_THREADS = 3;
081
082    protected static final int NB_CHECK_THREADS = 1;
083
084    protected static final int QUEUESIZE = 20;
085
086    public static final String BASE_URL_KEY = "nuxeo.wizard.packages.url";
087
088    public static final String DEFAULT_BASE_URL = "http://cdn.nuxeo.com/"; // nuxeo-XXX/mp
089
090    protected CopyOnWriteArrayList<PendingDownload> pendingDownloads = new CopyOnWriteArrayList<>();
091
092    protected static PackageDownloader instance;
093
094    protected DefaultHttpClient httpClient;
095
096    protected Boolean canReachServer = null;
097
098    protected DownloadablePackageOptions downloadOptions;
099
100    protected static final String DIGEST = "MD5";
101
102    protected static final int DIGEST_CHUNK = 1024 * 100;
103
104    boolean downloadStarted = false;
105
106    protected String lastSelectionDigest;
107
108    protected final AtomicInteger dwThreadCount = new AtomicInteger(0);
109
110    protected final AtomicInteger checkThreadCount = new AtomicInteger(0);
111
112    protected String baseUrl;
113
114    protected ConfigurationGenerator configurationGenerator = null;
115
116    protected ConfigurationGenerator getConfig() {
117        if (configurationGenerator == null) {
118            configurationGenerator = new ConfigurationGenerator();
119            configurationGenerator.init();
120        }
121        return configurationGenerator;
122    }
123
124    protected String getBaseUrl() {
125        if (baseUrl == null) {
126            String base = getConfig().getUserConfig().getProperty(BASE_URL_KEY, "");
127            if ("".equals(base)) {
128                base = DEFAULT_BASE_URL + "nuxeo-"
129                        + getConfig().getUserConfig().getProperty("org.nuxeo.ecm.product.version") + "/mp/";
130            }
131            if (!base.endsWith("/")) {
132                base = base + "/";
133            }
134            baseUrl = base;
135        }
136        return baseUrl;
137    }
138
139    protected ThreadPoolExecutor download_tpe = new ThreadPoolExecutor(NB_DOWNLOAD_THREADS, NB_DOWNLOAD_THREADS, 10L,
140            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
141                @Override
142                public Thread newThread(Runnable r) {
143                    Thread t = new Thread(r);
144                    t.setDaemon(true);
145                    t.setName("DownloaderThread-" + dwThreadCount.incrementAndGet());
146                    return t;
147                }
148            });
149
150    protected ThreadPoolExecutor check_tpe = new ThreadPoolExecutor(NB_CHECK_THREADS, NB_CHECK_THREADS, 10L,
151            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
152                @Override
153                public Thread newThread(Runnable r) {
154                    Thread t = new Thread(r);
155                    t.setDaemon(true);
156                    t.setName("MD5CheckThread-" + checkThreadCount.incrementAndGet());
157                    return t;
158                }
159            });
160
161    protected PackageDownloader() {
162        SchemeRegistry registry = new SchemeRegistry();
163        registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
164        registry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
165        HttpParams httpParams = new BasicHttpParams();
166        HttpProtocolParams.setUseExpectContinue(httpParams, false);
167        ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(registry);
168        cm.setMaxTotal(NB_DOWNLOAD_THREADS);
169        cm.setDefaultMaxPerRoute(NB_DOWNLOAD_THREADS);
170        httpClient = new DefaultHttpClient(cm, httpParams);
171    }
172
173    public synchronized static PackageDownloader instance() {
174        if (instance == null) {
175            instance = new PackageDownloader();
176            instance.download_tpe.prestartAllCoreThreads();
177            instance.check_tpe.prestartAllCoreThreads();
178        }
179        return instance;
180    }
181
182    public static void reset() {
183        if (instance != null) {
184            instance.shutdown();
185            instance = null;
186        }
187    }
188
189    public void setProxy(String proxy, int port, String login, String password, String NTLMHost, String NTLMDomain) {
190        if (proxy != null) {
191            HttpHost proxyHost = new HttpHost(proxy, port);
192            httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
193            if (login != null) {
194                if (NTLMHost != null && !NTLMHost.trim().isEmpty()) {
195                    NTCredentials ntlmCredentials = new NTCredentials(login, password, NTLMHost, NTLMDomain);
196                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), ntlmCredentials);
197                } else {
198                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port),
199                            new UsernamePasswordCredentials(login, password));
200                }
201            } else {
202                httpClient.getCredentialsProvider().clear();
203            }
204        } else {
205            httpClient.getParams().removeParameter(ConnRoutePNames.DEFAULT_PROXY);
206            httpClient.getCredentialsProvider().clear();
207        }
208    }
209
210    protected String getSelectionDigest(List<String> ids) {
211        ArrayList<String> lst = new ArrayList<>(ids);
212        Collections.sort(lst);
213        StringBuffer sb = new StringBuffer();
214        for (String item : lst) {
215            sb.append(item);
216            sb.append(":");
217        }
218        return sb.toString();
219    }
220
221    public void selectOptions(List<String> ids) {
222        String newSelectionDigest = getSelectionDigest(ids);
223        if (lastSelectionDigest != null) {
224            if (lastSelectionDigest.equals(newSelectionDigest)) {
225                return;
226            }
227        }
228        getPackageOptions().select(ids);
229        downloadStarted = false;
230        lastSelectionDigest = newSelectionDigest;
231    }
232
233    protected File getDownloadDirectory() {
234        File mpDir = getConfig().getDistributionMPDir();
235        if (!mpDir.exists()) {
236            mpDir.mkdirs();
237        }
238        return mpDir;
239    }
240
241    public boolean canReachServer() {
242        if (canReachServer == null) {
243            HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
244            try {
245                HttpResponse response = httpClient.execute(ping);
246                if (response.getStatusLine().getStatusCode() == 200) {
247                    canReachServer = true;
248                } else {
249                    log.info("Unable to ping server -> status code :" + response.getStatusLine().getStatusCode() + " ("
250                            + response.getStatusLine().getReasonPhrase() + ")");
251                    canReachServer = false;
252                }
253            } catch (Exception e) {
254                log.info("Unable to ping remote server " + e.getMessage());
255                log.debug("Unable to ping remote server", e);
256                canReachServer = false;
257            }
258        }
259        return canReachServer;
260    }
261
262    public DownloadablePackageOptions getPackageOptions() {
263        if (downloadOptions == null) {
264            File packageFile = null;
265            if (canReachServer()) {
266                packageFile = getRemotePackagesDescriptor();
267            }
268            if (packageFile == null) {
269                packageFile = getLocalPackagesDescriptor();
270                if (packageFile == null) {
271                    log.warn("Unable to find local copy of packages.xml");
272                } else {
273                    log.info("Wizard will use the local copy of packages.xml.");
274                }
275            }
276            if (packageFile != null) {
277                try {
278                    downloadOptions = DownloadDescriptorParser.parsePackages(new FileInputStream(packageFile));
279
280                    // manage init from presets if available
281                    Properties defaultSelection = getDefaultPackageSelection();
282                    if (defaultSelection != null) {
283                        String presetId = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PRESETS, null);
284                        if (presetId != null && !presetId.isEmpty()) {
285                            for (Preset preset : downloadOptions.getPresets()) {
286                                if (preset.getId().equals(presetId)) {
287                                    List<String> pkgIds = Arrays.asList(preset.getPkgs());
288                                    downloadOptions.select(pkgIds);
289                                    break;
290                                }
291                            }
292                        } else {
293                            String pkgIdsList = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PACKAGES, null);
294                            if (pkgIdsList != null && !pkgIdsList.isEmpty()) {
295                                String[] ids = pkgIdsList.split(",");
296                                List<String> pkgIds = Arrays.asList(ids);
297                                downloadOptions.select(pkgIds);
298                            }
299                        }
300                    }
301                } catch (FileNotFoundException e) {
302                    log.error("Unable to read packages.xml", e);
303                }
304            }
305        }
306        return downloadOptions;
307    }
308
309    protected File getRemotePackagesDescriptor() {
310        File desc = null;
311        HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
312        try {
313            HttpResponse response = httpClient.execute(ping);
314            if (response.getStatusLine().getStatusCode() == 200) {
315                desc = new File(getDownloadDirectory(), PACKAGES_XML);
316                FileUtils.copyToFile(response.getEntity().getContent(), desc);
317            } else {
318                log.warn("Unable to download remote packages.xml, status code :"
319                        + response.getStatusLine().getStatusCode() + " (" + response.getStatusLine().getReasonPhrase()
320                        + ")");
321                return null;
322            }
323        } catch (Exception e) {
324            log.warn("Unable to reach remote packages.xml", e);
325            return null;
326        }
327        return desc;
328    }
329
330    protected Properties getDefaultPackageSelection() {
331        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
332        if (desc.exists()) {
333            try {
334                Properties props = new Properties();
335                props.load(new FileReader(desc));
336                return props;
337            } catch (IOException e) {
338                log.warn("Unable to load presets", e);
339            }
340        }
341        return null;
342    }
343
344    protected void saveSelectedPackages(List<DownloadPackage> pkgs) {
345        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
346        Properties props = new Properties();
347        StringBuffer sb = new StringBuffer();
348        for (int i = 0; i < pkgs.size(); i++) {
349            if (i > 0) {
350                sb.append(",");
351            }
352            sb.append(pkgs.get(i).getId());
353        }
354        props.put(PACKAGES_DEFAULT_SELECTION_PACKAGES, sb.toString());
355        try {
356            props.store(new FileWriter(desc), "Saved from Nuxeo SetupWizard");
357        } catch (IOException e) {
358            log.error("Unable to save package selection", e);
359        }
360    }
361
362    protected File getLocalPackagesDescriptor() {
363        File desc = new File(getDownloadDirectory(), PACKAGES_XML);
364        if (desc.exists()) {
365            return desc;
366        }
367        return null;
368    }
369
370    public List<DownloadPackage> getSelectedPackages() {
371        List<DownloadPackage> pkgs = getPackageOptions().getPkg4Install();
372        File[] listFiles = getDownloadDirectory().listFiles();
373        for (DownloadPackage pkg : pkgs) {
374            for (File file : listFiles) {
375                if (file.getName().equals(pkg.getMd5())) {
376                    // recheck md5 ???
377                    pkg.setLocalFile(file);
378                }
379            }
380            needToDownload(pkg);
381        }
382        return pkgs;
383    }
384
385    public void scheduleDownloadedPackagesForInstallation(String installationFilePath) throws IOException {
386        List<String> fileEntries = new ArrayList<>();
387        fileEntries.add("init");
388
389        List<DownloadPackage> pkgs = downloadOptions.getPkg4Install();
390        List<String> pkgInstallIds = new ArrayList<>();
391        for (DownloadPackage pkg : pkgs) {
392            if (pkg.isVirtual()) {
393                log.debug("No install for virtual package: " + pkg.getId());
394            } else if (pkg.isAlreadyInLocal() || StringUtils.isBlank(pkg.getFilename())) {
395                // Blank filename means later downloaded
396                fileEntries.add("install " + pkg.getId());
397                pkgInstallIds.add(pkg.getId());
398            } else {
399                for (PendingDownload download : pendingDownloads) {
400                    if (download.getPkg().equals(pkg)) {
401                        if (download.getStatus() == PendingDownloadStatus.VERIFIED) {
402                            File file = download.getDowloadingFile();
403                            fileEntries.add("add file:" + file.getAbsolutePath());
404                            fileEntries.add("install " + pkg.getId());
405                            pkgInstallIds.add(pkg.getId());
406                        } else {
407                            log.error("One selected package has not been downloaded : " + pkg.getId());
408                        }
409                    }
410                }
411            }
412        }
413
414        File installLog = new File(installationFilePath);
415        if (fileEntries.size() > 0) {
416            if (!installLog.exists()) {
417                File parent = installLog.getParentFile();
418                if (!parent.exists()) {
419                    parent.mkdirs();
420                }
421                installLog.createNewFile();
422            }
423            FileUtils.writeLines(installLog, fileEntries);
424        } else {
425            // Should not happen as the file always has "init"
426            if (installLog.exists()) {
427                installLog.delete();
428            }
429        }
430
431        // Save presets
432        saveSelectedPackages(pkgs);
433    }
434
435    public List<PendingDownload> getPendingDownloads() {
436        return pendingDownloads;
437    }
438
439    public void reStartDownload(String id) {
440        for (PendingDownload pending : pendingDownloads) {
441            if (pending.getPkg().getId().equals(id)) {
442                if (Arrays.asList(PendingDownloadStatus.CORRUPTED, PendingDownloadStatus.ABORTED).contains(
443                        pending.getStatus())) {
444                    pendingDownloads.remove(pending);
445                    startDownloadPackage(pending.getPkg());
446                }
447                break;
448            }
449        }
450    }
451
452    public void startDownload() {
453        startDownload(downloadOptions.getPkg4Install());
454    }
455
456    public void startDownload(List<DownloadPackage> pkgs) {
457        downloadStarted = true;
458        for (DownloadPackage pkg : pkgs) {
459            if (needToDownload(pkg)) {
460                startDownloadPackage(pkg);
461            }
462        }
463    }
464
465    protected boolean needToDownload(DownloadPackage pkg) {
466        return !pkg.isVirtual() && !pkg.isLaterDownload() && !pkg.isAlreadyInLocal();
467    }
468
469    protected void startDownloadPackage(final DownloadPackage pkg) {
470        final PendingDownload download = new PendingDownload(pkg);
471        if (pendingDownloads.addIfAbsent(download)) {
472            Runnable downloadRunner = new Runnable() {
473
474                @Override
475                public void run() {
476                    log.info("Starting download on Thread " + Thread.currentThread().getName());
477                    download.setStatus(PendingDownloadStatus.INPROGRESS);
478                    String url = pkg.getDownloadUrl();
479                    if (!url.startsWith("http")) {
480                        url = getBaseUrl() + url;
481                    }
482                    File filePkg = null;
483                    HttpGet dw = new HttpGet(url);
484                    try {
485                        HttpResponse response = httpClient.execute(dw);
486                        if (response.getStatusLine().getStatusCode() == 200) {
487                            filePkg = new File(getDownloadDirectory(), pkg.filename);
488                            Header clh = response.getFirstHeader("Content-Length");
489                            if (clh != null) {
490                                long filesize = Long.parseLong(clh.getValue());
491                                download.setFile(filesize, filePkg);
492                            }
493                            FileUtils.copyToFile(response.getEntity().getContent(), filePkg);
494                            download.setStatus(PendingDownloadStatus.COMPLETED);
495                        } else if (response.getStatusLine().getStatusCode() == 404) {
496                            log.error("Package " + pkg.filename + " not found :" + url);
497                            download.setStatus(PendingDownloadStatus.MISSING);
498                            EntityUtils.consume(response.getEntity());
499                            dw.abort();
500                            return;
501                        } else {
502                            log.error("Received StatusCode " + response.getStatusLine().getStatusCode());
503                            download.setStatus(PendingDownloadStatus.ABORTED);
504                            EntityUtils.consume(response.getEntity());
505                            dw.abort();
506                            return;
507                        }
508                    } catch (Exception e) {
509                        download.setStatus(PendingDownloadStatus.ABORTED);
510                        log.error("Error during download", e);
511                        return;
512                    }
513                    checkPackage(download);
514                }
515            };
516            download_tpe.execute(downloadRunner);
517        }
518    }
519
520    protected void checkPackage(final PendingDownload download) {
521        final File filePkg = download.getDowloadingFile();
522        Runnable checkRunner = new Runnable() {
523            @Override
524            public void run() {
525                download.setStatus(PendingDownloadStatus.VERIFICATION);
526                String expectedDigest = download.getPkg().getMd5();
527                String digest = getDigest(filePkg);
528                if (digest == null || (expectedDigest != null && !expectedDigest.equals(digest))) {
529                    download.setStatus(PendingDownloadStatus.CORRUPTED);
530                    log.error("Digest check failed: expected=" + expectedDigest + " computed=" + digest);
531                    return;
532                }
533                File newFile = new File(getDownloadDirectory(), digest);
534                filePkg.renameTo(newFile);
535                download.setStatus(PendingDownloadStatus.VERIFIED);
536                download.setFile(newFile.length(), newFile);
537            }
538        };
539        check_tpe.execute(checkRunner);
540    }
541
542    protected String getDigest(File file) {
543        try {
544            MessageDigest md = MessageDigest.getInstance(DIGEST);
545            byte[] buffer = new byte[DIGEST_CHUNK];
546            InputStream stream = new FileInputStream(file);
547            int bytesRead = -1;
548            while ((bytesRead = stream.read(buffer)) >= 0) {
549                md.update(buffer, 0, bytesRead);
550            }
551            stream.close();
552            byte[] b = md.digest();
553            return md5ToHex(b);
554        } catch (Exception e) {
555            log.error("Error while computing Digest ", e);
556            return null;
557        }
558    }
559
560    protected static String md5ToHex(byte[] hash) {
561        StringBuilder hexString = new StringBuilder();
562        for (byte b : hash) {
563            String hex = Integer.toHexString(0xFF & b);
564            if (hex.length() == 1) {
565                hexString.append('0');
566            }
567            hexString.append(hex);
568        }
569        return hexString.toString();
570    }
571
572    public boolean isDownloadStarted() {
573        return downloadStarted;
574    }
575
576    public boolean isDownloadCompleted() {
577        if (!isDownloadStarted()) {
578            return false;
579        }
580        for (PendingDownload download : pendingDownloads) {
581            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()) {
582                return false;
583            }
584        }
585        return true;
586    }
587
588    public boolean isDownloadInProgress() {
589        if (!isDownloadStarted()) {
590            return false;
591        }
592        if (isDownloadCompleted()) {
593            return false;
594        }
595        int nbInProgress = 0;
596        for (PendingDownload download : pendingDownloads) {
597            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()
598                    && download.getStatus().getValue() >= PendingDownloadStatus.PENDING.getValue()) {
599                nbInProgress++;
600            }
601        }
602        return nbInProgress > 0;
603    }
604
605    public void shutdown() {
606        if (httpClient != null) {
607            httpClient.getConnectionManager().shutdown();
608        }
609        download_tpe.shutdownNow();
610        check_tpe.shutdownNow();
611    }
612
613}