001/*
002 * (C) Copyright 2011-2015 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     tdelprat
018 *
019 */
020
021package org.nuxeo.wizard.download;
022
023import java.io.File;
024import java.io.FileInputStream;
025import java.io.FileNotFoundException;
026import java.io.FileReader;
027import java.io.FileWriter;
028import java.io.IOException;
029import java.io.InputStream;
030import java.security.MessageDigest;
031import java.util.ArrayList;
032import java.util.Arrays;
033import java.util.Collections;
034import java.util.List;
035import java.util.Properties;
036import java.util.concurrent.CopyOnWriteArrayList;
037import java.util.concurrent.LinkedBlockingQueue;
038import java.util.concurrent.ThreadFactory;
039import java.util.concurrent.ThreadPoolExecutor;
040import java.util.concurrent.TimeUnit;
041import java.util.concurrent.atomic.AtomicInteger;
042
043import org.apache.commons.lang.StringUtils;
044import org.apache.commons.logging.Log;
045import org.apache.commons.logging.LogFactory;
046import org.apache.http.Header;
047import org.apache.http.HttpHost;
048import org.apache.http.HttpResponse;
049import org.apache.http.auth.AuthScope;
050import org.apache.http.auth.NTCredentials;
051import org.apache.http.auth.UsernamePasswordCredentials;
052import org.apache.http.client.methods.HttpGet;
053import org.apache.http.conn.params.ConnRoutePNames;
054import org.apache.http.conn.scheme.PlainSocketFactory;
055import org.apache.http.conn.scheme.Scheme;
056import org.apache.http.conn.scheme.SchemeRegistry;
057import org.apache.http.conn.ssl.SSLSocketFactory;
058import org.apache.http.impl.client.DefaultHttpClient;
059import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
060import org.apache.http.params.BasicHttpParams;
061import org.apache.http.params.HttpParams;
062import org.apache.http.params.HttpProtocolParams;
063import org.apache.http.util.EntityUtils;
064
065import org.nuxeo.common.Environment;
066import org.nuxeo.common.utils.FileUtils;
067import org.nuxeo.launcher.config.ConfigurationGenerator;
068
069/**
070 * @author Tiry (tdelprat@nuxeo.com)
071 */
072public class PackageDownloader {
073
074    protected final static Log log = LogFactory.getLog(PackageDownloader.class);
075
076    public static final String PACKAGES_XML = "packages.xml";
077
078    public static final String PACKAGES_DEFAULT_SELECTION = "packages-default-selection.properties";
079
080    public static final String PACKAGES_DEFAULT_SELECTION_PRESETS = "preset";
081
082    public static final String PACKAGES_DEFAULT_SELECTION_PACKAGES = "packages";
083
084    protected static final int NB_DOWNLOAD_THREADS = 3;
085
086    protected static final int NB_CHECK_THREADS = 1;
087
088    protected static final int QUEUESIZE = 20;
089
090    public static final String BASE_URL_KEY = "nuxeo.wizard.packages.url";
091
092    public static final String DEFAULT_BASE_URL = "http://cdn.nuxeo.com/"; // nuxeo-XXX/mp
093
094    protected CopyOnWriteArrayList<PendingDownload> pendingDownloads = new CopyOnWriteArrayList<>();
095
096    protected static PackageDownloader instance;
097
098    protected DefaultHttpClient httpClient;
099
100    protected Boolean canReachServer = null;
101
102    protected DownloadablePackageOptions downloadOptions;
103
104    protected static final String DIGEST = "MD5";
105
106    protected static final int DIGEST_CHUNK = 1024 * 100;
107
108    boolean downloadStarted = false;
109
110    protected String lastSelectionDigest;
111
112    protected final AtomicInteger dwThreadCount = new AtomicInteger(0);
113
114    protected final AtomicInteger checkThreadCount = new AtomicInteger(0);
115
116    protected String baseUrl;
117
118    protected ConfigurationGenerator configurationGenerator = null;
119
120    protected ConfigurationGenerator getConfig() {
121        if (configurationGenerator == null) {
122            configurationGenerator = new ConfigurationGenerator();
123            configurationGenerator.init();
124        }
125        return configurationGenerator;
126    }
127
128    protected String getBaseUrl() {
129        if (baseUrl == null) {
130            String base = getConfig().getUserConfig().getProperty(BASE_URL_KEY, "");
131            if ("".equals(base)) {
132                base = DEFAULT_BASE_URL + "nuxeo-"
133                        + getConfig().getUserConfig().getProperty(Environment.DISTRIBUTION_VERSION) + "/mp/";
134            }
135            if (!base.endsWith("/")) {
136                base = base + "/";
137            }
138            baseUrl = base;
139        }
140        return baseUrl;
141    }
142
143    protected ThreadPoolExecutor download_tpe = new ThreadPoolExecutor(NB_DOWNLOAD_THREADS, NB_DOWNLOAD_THREADS, 10L,
144            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
145                @Override
146                public Thread newThread(Runnable r) {
147                    Thread t = new Thread(r);
148                    t.setDaemon(true);
149                    t.setName("DownloaderThread-" + dwThreadCount.incrementAndGet());
150                    return t;
151                }
152            });
153
154    protected ThreadPoolExecutor check_tpe = new ThreadPoolExecutor(NB_CHECK_THREADS, NB_CHECK_THREADS, 10L,
155            TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() {
156                @Override
157                public Thread newThread(Runnable r) {
158                    Thread t = new Thread(r);
159                    t.setDaemon(true);
160                    t.setName("MD5CheckThread-" + checkThreadCount.incrementAndGet());
161                    return t;
162                }
163            });
164
165    protected PackageDownloader() {
166        SchemeRegistry registry = new SchemeRegistry();
167        registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
168        registry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
169        HttpParams httpParams = new BasicHttpParams();
170        HttpProtocolParams.setUseExpectContinue(httpParams, false);
171        ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(registry);
172        cm.setMaxTotal(NB_DOWNLOAD_THREADS);
173        cm.setDefaultMaxPerRoute(NB_DOWNLOAD_THREADS);
174        httpClient = new DefaultHttpClient(cm, httpParams);
175    }
176
177    public synchronized static PackageDownloader instance() {
178        if (instance == null) {
179            instance = new PackageDownloader();
180            instance.download_tpe.prestartAllCoreThreads();
181            instance.check_tpe.prestartAllCoreThreads();
182        }
183        return instance;
184    }
185
186    public static void reset() {
187        if (instance != null) {
188            instance.shutdown();
189            instance = null;
190        }
191    }
192
193    public void setProxy(String proxy, int port, String login, String password, String NTLMHost, String NTLMDomain) {
194        if (proxy != null) {
195            HttpHost proxyHost = new HttpHost(proxy, port);
196            httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost);
197            if (login != null) {
198                if (NTLMHost != null && !NTLMHost.trim().isEmpty()) {
199                    NTCredentials ntlmCredentials = new NTCredentials(login, password, NTLMHost, NTLMDomain);
200                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), ntlmCredentials);
201                } else {
202                    httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port),
203                            new UsernamePasswordCredentials(login, password));
204                }
205            } else {
206                httpClient.getCredentialsProvider().clear();
207            }
208        } else {
209            httpClient.getParams().removeParameter(ConnRoutePNames.DEFAULT_PROXY);
210            httpClient.getCredentialsProvider().clear();
211        }
212    }
213
214    protected String getSelectionDigest(List<String> ids) {
215        ArrayList<String> lst = new ArrayList<>(ids);
216        Collections.sort(lst);
217        StringBuffer sb = new StringBuffer();
218        for (String item : lst) {
219            sb.append(item);
220            sb.append(":");
221        }
222        return sb.toString();
223    }
224
225    public void selectOptions(List<String> ids) {
226        String newSelectionDigest = getSelectionDigest(ids);
227        if (lastSelectionDigest != null) {
228            if (lastSelectionDigest.equals(newSelectionDigest)) {
229                return;
230            }
231        }
232        getPackageOptions().select(ids);
233        downloadStarted = false;
234        lastSelectionDigest = newSelectionDigest;
235    }
236
237    protected File getDownloadDirectory() {
238        File mpDir = getConfig().getDistributionMPDir();
239        if (!mpDir.exists()) {
240            mpDir.mkdirs();
241        }
242        return mpDir;
243    }
244
245    public boolean canReachServer() {
246        if (canReachServer == null) {
247            HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
248            try {
249                HttpResponse response = httpClient.execute(ping);
250                if (response.getStatusLine().getStatusCode() == 200) {
251                    canReachServer = true;
252                } else {
253                    log.info("Unable to ping server -> status code :" + response.getStatusLine().getStatusCode() + " ("
254                            + response.getStatusLine().getReasonPhrase() + ")");
255                    canReachServer = false;
256                }
257            } catch (Exception e) {
258                log.info("Unable to ping remote server " + e.getMessage());
259                log.debug("Unable to ping remote server", e);
260                canReachServer = false;
261            }
262        }
263        return canReachServer;
264    }
265
266    public DownloadablePackageOptions getPackageOptions() {
267        if (downloadOptions == null) {
268            File packageFile = null;
269            if (canReachServer()) {
270                packageFile = getRemotePackagesDescriptor();
271            }
272            if (packageFile == null) {
273                packageFile = getLocalPackagesDescriptor();
274                if (packageFile == null) {
275                    log.warn("Unable to find local copy of packages.xml");
276                } else {
277                    log.info("Wizard will use the local copy of packages.xml.");
278                }
279            }
280            if (packageFile != null) {
281                try {
282                    downloadOptions = DownloadDescriptorParser.parsePackages(new FileInputStream(packageFile));
283
284                    // manage init from presets if available
285                    Properties defaultSelection = getDefaultPackageSelection();
286                    if (defaultSelection != null) {
287                        String presetId = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PRESETS, null);
288                        if (presetId != null && !presetId.isEmpty()) {
289                            for (Preset preset : downloadOptions.getPresets()) {
290                                if (preset.getId().equals(presetId)) {
291                                    List<String> pkgIds = Arrays.asList(preset.getPkgs());
292                                    downloadOptions.select(pkgIds);
293                                    break;
294                                }
295                            }
296                        } else {
297                            String pkgIdsList = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PACKAGES, null);
298                            if (pkgIdsList != null && !pkgIdsList.isEmpty()) {
299                                String[] ids = pkgIdsList.split(",");
300                                List<String> pkgIds = Arrays.asList(ids);
301                                downloadOptions.select(pkgIds);
302                            }
303                        }
304                    }
305                } catch (FileNotFoundException e) {
306                    log.error("Unable to read packages.xml", e);
307                }
308            }
309        }
310        return downloadOptions;
311    }
312
313    protected File getRemotePackagesDescriptor() {
314        File desc = null;
315        HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML);
316        try {
317            HttpResponse response = httpClient.execute(ping);
318            if (response.getStatusLine().getStatusCode() == 200) {
319                desc = new File(getDownloadDirectory(), PACKAGES_XML);
320                FileUtils.copyToFile(response.getEntity().getContent(), desc);
321            } else {
322                log.warn("Unable to download remote packages.xml, status code :"
323                        + response.getStatusLine().getStatusCode() + " (" + response.getStatusLine().getReasonPhrase()
324                        + ")");
325                return null;
326            }
327        } catch (Exception e) {
328            log.warn("Unable to reach remote packages.xml", e);
329            return null;
330        }
331        return desc;
332    }
333
334    protected Properties getDefaultPackageSelection() {
335        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
336        if (desc.exists()) {
337            try {
338                Properties props = new Properties();
339                props.load(new FileReader(desc));
340                return props;
341            } catch (IOException e) {
342                log.warn("Unable to load presets", e);
343            }
344        }
345        return null;
346    }
347
348    protected void saveSelectedPackages(List<DownloadPackage> pkgs) {
349        File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION);
350        Properties props = new Properties();
351        StringBuffer sb = new StringBuffer();
352        for (int i = 0; i < pkgs.size(); i++) {
353            if (i > 0) {
354                sb.append(",");
355            }
356            sb.append(pkgs.get(i).getId());
357        }
358        props.put(PACKAGES_DEFAULT_SELECTION_PACKAGES, sb.toString());
359        try {
360            props.store(new FileWriter(desc), "Saved from Nuxeo SetupWizard");
361        } catch (IOException e) {
362            log.error("Unable to save package selection", e);
363        }
364    }
365
366    protected File getLocalPackagesDescriptor() {
367        File desc = new File(getDownloadDirectory(), PACKAGES_XML);
368        if (desc.exists()) {
369            return desc;
370        }
371        return null;
372    }
373
374    public List<DownloadPackage> getSelectedPackages() {
375        List<DownloadPackage> pkgs = getPackageOptions().getPkg4Install();
376        File[] listFiles = getDownloadDirectory().listFiles();
377        for (DownloadPackage pkg : pkgs) {
378            for (File file : listFiles) {
379                if (file.getName().equals(pkg.getMd5())) {
380                    // recheck md5 ???
381                    pkg.setLocalFile(file);
382                }
383            }
384            needToDownload(pkg);
385        }
386        return pkgs;
387    }
388
389    public void scheduleDownloadedPackagesForInstallation(String installationFilePath) throws IOException {
390        List<String> fileEntries = new ArrayList<>();
391        fileEntries.add("init");
392
393        List<DownloadPackage> pkgs = downloadOptions.getPkg4Install();
394        List<String> pkgInstallIds = new ArrayList<>();
395        for (DownloadPackage pkg : pkgs) {
396            if (pkg.isVirtual()) {
397                log.debug("No install for virtual package: " + pkg.getId());
398            } else if (pkg.isAlreadyInLocal() || StringUtils.isBlank(pkg.getFilename())) {
399                // Blank filename means later downloaded
400                fileEntries.add("install " + pkg.getId());
401                pkgInstallIds.add(pkg.getId());
402            } else {
403                for (PendingDownload download : pendingDownloads) {
404                    if (download.getPkg().equals(pkg)) {
405                        if (download.getStatus() == PendingDownloadStatus.VERIFIED) {
406                            File file = download.getDowloadingFile();
407                            fileEntries.add("add file:" + file.getAbsolutePath());
408                            fileEntries.add("install " + pkg.getId());
409                            pkgInstallIds.add(pkg.getId());
410                        } else {
411                            log.error("One selected package has not been downloaded : " + pkg.getId());
412                        }
413                    }
414                }
415            }
416        }
417
418        File installLog = new File(installationFilePath);
419        if (fileEntries.size() > 0) {
420            if (!installLog.exists()) {
421                File parent = installLog.getParentFile();
422                if (!parent.exists()) {
423                    parent.mkdirs();
424                }
425                installLog.createNewFile();
426            }
427            FileUtils.writeLines(installLog, fileEntries);
428        } else {
429            // Should not happen as the file always has "init"
430            if (installLog.exists()) {
431                installLog.delete();
432            }
433        }
434
435        // Save presets
436        saveSelectedPackages(pkgs);
437    }
438
439    public List<PendingDownload> getPendingDownloads() {
440        return pendingDownloads;
441    }
442
443    public void reStartDownload(String id) {
444        for (PendingDownload pending : pendingDownloads) {
445            if (pending.getPkg().getId().equals(id)) {
446                if (Arrays.asList(PendingDownloadStatus.CORRUPTED, PendingDownloadStatus.ABORTED).contains(
447                        pending.getStatus())) {
448                    pendingDownloads.remove(pending);
449                    startDownloadPackage(pending.getPkg());
450                }
451                break;
452            }
453        }
454    }
455
456    public void startDownload() {
457        startDownload(downloadOptions.getPkg4Install());
458    }
459
460    public void startDownload(List<DownloadPackage> pkgs) {
461        downloadStarted = true;
462        for (DownloadPackage pkg : pkgs) {
463            if (needToDownload(pkg)) {
464                startDownloadPackage(pkg);
465            }
466        }
467    }
468
469    protected boolean needToDownload(DownloadPackage pkg) {
470        return !pkg.isVirtual() && !pkg.isLaterDownload() && !pkg.isAlreadyInLocal();
471    }
472
473    protected void startDownloadPackage(final DownloadPackage pkg) {
474        final PendingDownload download = new PendingDownload(pkg);
475        if (pendingDownloads.addIfAbsent(download)) {
476            Runnable downloadRunner = new Runnable() {
477
478                @Override
479                public void run() {
480                    log.info("Starting download on Thread " + Thread.currentThread().getName());
481                    download.setStatus(PendingDownloadStatus.INPROGRESS);
482                    String url = pkg.getDownloadUrl();
483                    if (!url.startsWith("http")) {
484                        url = getBaseUrl() + url;
485                    }
486                    File filePkg = null;
487                    HttpGet dw = new HttpGet(url);
488                    try {
489                        HttpResponse response = httpClient.execute(dw);
490                        if (response.getStatusLine().getStatusCode() == 200) {
491                            filePkg = new File(getDownloadDirectory(), pkg.filename);
492                            Header clh = response.getFirstHeader("Content-Length");
493                            if (clh != null) {
494                                long filesize = Long.parseLong(clh.getValue());
495                                download.setFile(filesize, filePkg);
496                            }
497                            FileUtils.copyToFile(response.getEntity().getContent(), filePkg);
498                            download.setStatus(PendingDownloadStatus.COMPLETED);
499                        } else if (response.getStatusLine().getStatusCode() == 404) {
500                            log.error("Package " + pkg.filename + " not found :" + url);
501                            download.setStatus(PendingDownloadStatus.MISSING);
502                            EntityUtils.consume(response.getEntity());
503                            dw.abort();
504                            return;
505                        } else {
506                            log.error("Received StatusCode " + response.getStatusLine().getStatusCode());
507                            download.setStatus(PendingDownloadStatus.ABORTED);
508                            EntityUtils.consume(response.getEntity());
509                            dw.abort();
510                            return;
511                        }
512                    } catch (Exception e) {
513                        download.setStatus(PendingDownloadStatus.ABORTED);
514                        log.error("Error during download", e);
515                        return;
516                    }
517                    checkPackage(download);
518                }
519            };
520            download_tpe.execute(downloadRunner);
521        }
522    }
523
524    protected void checkPackage(final PendingDownload download) {
525        final File filePkg = download.getDowloadingFile();
526        Runnable checkRunner = new Runnable() {
527            @Override
528            public void run() {
529                download.setStatus(PendingDownloadStatus.VERIFICATION);
530                String expectedDigest = download.getPkg().getMd5();
531                String digest = getDigest(filePkg);
532                if (digest == null || (expectedDigest != null && !expectedDigest.equals(digest))) {
533                    download.setStatus(PendingDownloadStatus.CORRUPTED);
534                    log.error("Digest check failed: expected=" + expectedDigest + " computed=" + digest);
535                    return;
536                }
537                File newFile = new File(getDownloadDirectory(), digest);
538                filePkg.renameTo(newFile);
539                download.setStatus(PendingDownloadStatus.VERIFIED);
540                download.setFile(newFile.length(), newFile);
541            }
542        };
543        check_tpe.execute(checkRunner);
544    }
545
546    protected String getDigest(File file) {
547        try {
548            MessageDigest md = MessageDigest.getInstance(DIGEST);
549            byte[] buffer = new byte[DIGEST_CHUNK];
550            InputStream stream = new FileInputStream(file);
551            int bytesRead = -1;
552            while ((bytesRead = stream.read(buffer)) >= 0) {
553                md.update(buffer, 0, bytesRead);
554            }
555            stream.close();
556            byte[] b = md.digest();
557            return md5ToHex(b);
558        } catch (Exception e) {
559            log.error("Error while computing Digest ", e);
560            return null;
561        }
562    }
563
564    protected static String md5ToHex(byte[] hash) {
565        StringBuilder hexString = new StringBuilder();
566        for (byte b : hash) {
567            String hex = Integer.toHexString(0xFF & b);
568            if (hex.length() == 1) {
569                hexString.append('0');
570            }
571            hexString.append(hex);
572        }
573        return hexString.toString();
574    }
575
576    public boolean isDownloadStarted() {
577        return downloadStarted;
578    }
579
580    public boolean isDownloadCompleted() {
581        if (!isDownloadStarted()) {
582            return false;
583        }
584        for (PendingDownload download : pendingDownloads) {
585            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()) {
586                return false;
587            }
588        }
589        return true;
590    }
591
592    public boolean isDownloadInProgress() {
593        if (!isDownloadStarted()) {
594            return false;
595        }
596        if (isDownloadCompleted()) {
597            return false;
598        }
599        int nbInProgress = 0;
600        for (PendingDownload download : pendingDownloads) {
601            if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()
602                    && download.getStatus().getValue() >= PendingDownloadStatus.PENDING.getValue()) {
603                nbInProgress++;
604            }
605        }
606        return nbInProgress > 0;
607    }
608
609    public void shutdown() {
610        if (httpClient != null) {
611            httpClient.getConnectionManager().shutdown();
612        }
613        download_tpe.shutdownNow();
614        check_tpe.shutdownNow();
615    }
616
617}