001/* 002 * (C) Copyright 2011-2015 Nuxeo SA (http://nuxeo.com/) and contributors. 003 * 004 * All rights reserved. This program and the accompanying materials 005 * are made available under the terms of the GNU Lesser General Public License 006 * (LGPL) version 2.1 which accompanies this distribution, and is available at 007 * http://www.gnu.org/licenses/lgpl-2.1.html 008 * 009 * This library is distributed in the hope that it will be useful, 010 * but WITHOUT ANY WARRANTY; without even the implied warranty of 011 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 012 * Lesser General Public License for more details. 013 * 014 * Contributors: 015 * tdelprat 016 * 017 */ 018 019package org.nuxeo.wizard.download; 020 021import java.io.File; 022import java.io.FileInputStream; 023import java.io.FileNotFoundException; 024import java.io.FileReader; 025import java.io.FileWriter; 026import java.io.IOException; 027import java.io.InputStream; 028import java.security.MessageDigest; 029import java.util.ArrayList; 030import java.util.Arrays; 031import java.util.Collections; 032import java.util.List; 033import java.util.Properties; 034import java.util.concurrent.CopyOnWriteArrayList; 035import java.util.concurrent.LinkedBlockingQueue; 036import java.util.concurrent.ThreadFactory; 037import java.util.concurrent.ThreadPoolExecutor; 038import java.util.concurrent.TimeUnit; 039import java.util.concurrent.atomic.AtomicInteger; 040 041import org.apache.commons.lang.StringUtils; 042import org.apache.commons.logging.Log; 043import org.apache.commons.logging.LogFactory; 044import org.apache.http.Header; 045import org.apache.http.HttpHost; 046import org.apache.http.HttpResponse; 047import org.apache.http.auth.AuthScope; 048import org.apache.http.auth.NTCredentials; 049import org.apache.http.auth.UsernamePasswordCredentials; 050import org.apache.http.client.methods.HttpGet; 051import org.apache.http.conn.params.ConnRoutePNames; 052import org.apache.http.conn.scheme.PlainSocketFactory; 053import org.apache.http.conn.scheme.Scheme; 054import org.apache.http.conn.scheme.SchemeRegistry; 055import org.apache.http.conn.ssl.SSLSocketFactory; 056import org.apache.http.impl.client.DefaultHttpClient; 057import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; 058import org.apache.http.params.BasicHttpParams; 059import org.apache.http.params.HttpParams; 060import org.apache.http.params.HttpProtocolParams; 061import org.apache.http.util.EntityUtils; 062 063import org.nuxeo.common.Environment; 064import org.nuxeo.common.utils.FileUtils; 065import org.nuxeo.launcher.config.ConfigurationGenerator; 066 067/** 068 * @author Tiry (tdelprat@nuxeo.com) 069 */ 070public class PackageDownloader { 071 072 protected final static Log log = LogFactory.getLog(PackageDownloader.class); 073 074 public static final String PACKAGES_XML = "packages.xml"; 075 076 public static final String PACKAGES_DEFAULT_SELECTION = "packages-default-selection.properties"; 077 078 public static final String PACKAGES_DEFAULT_SELECTION_PRESETS = "preset"; 079 080 public static final String PACKAGES_DEFAULT_SELECTION_PACKAGES = "packages"; 081 082 protected static final int NB_DOWNLOAD_THREADS = 3; 083 084 protected static final int NB_CHECK_THREADS = 1; 085 086 protected static final int QUEUESIZE = 20; 087 088 public static final String BASE_URL_KEY = "nuxeo.wizard.packages.url"; 089 090 public static final String DEFAULT_BASE_URL = "http://cdn.nuxeo.com/"; // nuxeo-XXX/mp 091 092 protected CopyOnWriteArrayList<PendingDownload> pendingDownloads = new CopyOnWriteArrayList<>(); 093 094 protected static PackageDownloader instance; 095 096 protected DefaultHttpClient httpClient; 097 098 protected Boolean canReachServer = null; 099 100 protected DownloadablePackageOptions downloadOptions; 101 102 protected static final String DIGEST = "MD5"; 103 104 protected static final int DIGEST_CHUNK = 1024 * 100; 105 106 boolean downloadStarted = false; 107 108 protected String lastSelectionDigest; 109 110 protected final AtomicInteger dwThreadCount = new AtomicInteger(0); 111 112 protected final AtomicInteger checkThreadCount = new AtomicInteger(0); 113 114 protected String baseUrl; 115 116 protected ConfigurationGenerator configurationGenerator = null; 117 118 protected ConfigurationGenerator getConfig() { 119 if (configurationGenerator == null) { 120 configurationGenerator = new ConfigurationGenerator(); 121 configurationGenerator.init(); 122 } 123 return configurationGenerator; 124 } 125 126 protected String getBaseUrl() { 127 if (baseUrl == null) { 128 String base = getConfig().getUserConfig().getProperty(BASE_URL_KEY, ""); 129 if ("".equals(base)) { 130 base = DEFAULT_BASE_URL + "nuxeo-" 131 + getConfig().getUserConfig().getProperty(Environment.DISTRIBUTION_VERSION) + "/mp/"; 132 } 133 if (!base.endsWith("/")) { 134 base = base + "/"; 135 } 136 baseUrl = base; 137 } 138 return baseUrl; 139 } 140 141 protected ThreadPoolExecutor download_tpe = new ThreadPoolExecutor(NB_DOWNLOAD_THREADS, NB_DOWNLOAD_THREADS, 10L, 142 TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() { 143 @Override 144 public Thread newThread(Runnable r) { 145 Thread t = new Thread(r); 146 t.setDaemon(true); 147 t.setName("DownloaderThread-" + dwThreadCount.incrementAndGet()); 148 return t; 149 } 150 }); 151 152 protected ThreadPoolExecutor check_tpe = new ThreadPoolExecutor(NB_CHECK_THREADS, NB_CHECK_THREADS, 10L, 153 TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() { 154 @Override 155 public Thread newThread(Runnable r) { 156 Thread t = new Thread(r); 157 t.setDaemon(true); 158 t.setName("MD5CheckThread-" + checkThreadCount.incrementAndGet()); 159 return t; 160 } 161 }); 162 163 protected PackageDownloader() { 164 SchemeRegistry registry = new SchemeRegistry(); 165 registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); 166 registry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); 167 HttpParams httpParams = new BasicHttpParams(); 168 HttpProtocolParams.setUseExpectContinue(httpParams, false); 169 ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(registry); 170 cm.setMaxTotal(NB_DOWNLOAD_THREADS); 171 cm.setDefaultMaxPerRoute(NB_DOWNLOAD_THREADS); 172 httpClient = new DefaultHttpClient(cm, httpParams); 173 } 174 175 public synchronized static PackageDownloader instance() { 176 if (instance == null) { 177 instance = new PackageDownloader(); 178 instance.download_tpe.prestartAllCoreThreads(); 179 instance.check_tpe.prestartAllCoreThreads(); 180 } 181 return instance; 182 } 183 184 public static void reset() { 185 if (instance != null) { 186 instance.shutdown(); 187 instance = null; 188 } 189 } 190 191 public void setProxy(String proxy, int port, String login, String password, String NTLMHost, String NTLMDomain) { 192 if (proxy != null) { 193 HttpHost proxyHost = new HttpHost(proxy, port); 194 httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost); 195 if (login != null) { 196 if (NTLMHost != null && !NTLMHost.trim().isEmpty()) { 197 NTCredentials ntlmCredentials = new NTCredentials(login, password, NTLMHost, NTLMDomain); 198 httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), ntlmCredentials); 199 } else { 200 httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), 201 new UsernamePasswordCredentials(login, password)); 202 } 203 } else { 204 httpClient.getCredentialsProvider().clear(); 205 } 206 } else { 207 httpClient.getParams().removeParameter(ConnRoutePNames.DEFAULT_PROXY); 208 httpClient.getCredentialsProvider().clear(); 209 } 210 } 211 212 protected String getSelectionDigest(List<String> ids) { 213 ArrayList<String> lst = new ArrayList<>(ids); 214 Collections.sort(lst); 215 StringBuffer sb = new StringBuffer(); 216 for (String item : lst) { 217 sb.append(item); 218 sb.append(":"); 219 } 220 return sb.toString(); 221 } 222 223 public void selectOptions(List<String> ids) { 224 String newSelectionDigest = getSelectionDigest(ids); 225 if (lastSelectionDigest != null) { 226 if (lastSelectionDigest.equals(newSelectionDigest)) { 227 return; 228 } 229 } 230 getPackageOptions().select(ids); 231 downloadStarted = false; 232 lastSelectionDigest = newSelectionDigest; 233 } 234 235 protected File getDownloadDirectory() { 236 File mpDir = getConfig().getDistributionMPDir(); 237 if (!mpDir.exists()) { 238 mpDir.mkdirs(); 239 } 240 return mpDir; 241 } 242 243 public boolean canReachServer() { 244 if (canReachServer == null) { 245 HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML); 246 try { 247 HttpResponse response = httpClient.execute(ping); 248 if (response.getStatusLine().getStatusCode() == 200) { 249 canReachServer = true; 250 } else { 251 log.info("Unable to ping server -> status code :" + response.getStatusLine().getStatusCode() + " (" 252 + response.getStatusLine().getReasonPhrase() + ")"); 253 canReachServer = false; 254 } 255 } catch (Exception e) { 256 log.info("Unable to ping remote server " + e.getMessage()); 257 log.debug("Unable to ping remote server", e); 258 canReachServer = false; 259 } 260 } 261 return canReachServer; 262 } 263 264 public DownloadablePackageOptions getPackageOptions() { 265 if (downloadOptions == null) { 266 File packageFile = null; 267 if (canReachServer()) { 268 packageFile = getRemotePackagesDescriptor(); 269 } 270 if (packageFile == null) { 271 packageFile = getLocalPackagesDescriptor(); 272 if (packageFile == null) { 273 log.warn("Unable to find local copy of packages.xml"); 274 } else { 275 log.info("Wizard will use the local copy of packages.xml."); 276 } 277 } 278 if (packageFile != null) { 279 try { 280 downloadOptions = DownloadDescriptorParser.parsePackages(new FileInputStream(packageFile)); 281 282 // manage init from presets if available 283 Properties defaultSelection = getDefaultPackageSelection(); 284 if (defaultSelection != null) { 285 String presetId = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PRESETS, null); 286 if (presetId != null && !presetId.isEmpty()) { 287 for (Preset preset : downloadOptions.getPresets()) { 288 if (preset.getId().equals(presetId)) { 289 List<String> pkgIds = Arrays.asList(preset.getPkgs()); 290 downloadOptions.select(pkgIds); 291 break; 292 } 293 } 294 } else { 295 String pkgIdsList = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PACKAGES, null); 296 if (pkgIdsList != null && !pkgIdsList.isEmpty()) { 297 String[] ids = pkgIdsList.split(","); 298 List<String> pkgIds = Arrays.asList(ids); 299 downloadOptions.select(pkgIds); 300 } 301 } 302 } 303 } catch (FileNotFoundException e) { 304 log.error("Unable to read packages.xml", e); 305 } 306 } 307 } 308 return downloadOptions; 309 } 310 311 protected File getRemotePackagesDescriptor() { 312 File desc = null; 313 HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML); 314 try { 315 HttpResponse response = httpClient.execute(ping); 316 if (response.getStatusLine().getStatusCode() == 200) { 317 desc = new File(getDownloadDirectory(), PACKAGES_XML); 318 FileUtils.copyToFile(response.getEntity().getContent(), desc); 319 } else { 320 log.warn("Unable to download remote packages.xml, status code :" 321 + response.getStatusLine().getStatusCode() + " (" + response.getStatusLine().getReasonPhrase() 322 + ")"); 323 return null; 324 } 325 } catch (Exception e) { 326 log.warn("Unable to reach remote packages.xml", e); 327 return null; 328 } 329 return desc; 330 } 331 332 protected Properties getDefaultPackageSelection() { 333 File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION); 334 if (desc.exists()) { 335 try { 336 Properties props = new Properties(); 337 props.load(new FileReader(desc)); 338 return props; 339 } catch (IOException e) { 340 log.warn("Unable to load presets", e); 341 } 342 } 343 return null; 344 } 345 346 protected void saveSelectedPackages(List<DownloadPackage> pkgs) { 347 File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION); 348 Properties props = new Properties(); 349 StringBuffer sb = new StringBuffer(); 350 for (int i = 0; i < pkgs.size(); i++) { 351 if (i > 0) { 352 sb.append(","); 353 } 354 sb.append(pkgs.get(i).getId()); 355 } 356 props.put(PACKAGES_DEFAULT_SELECTION_PACKAGES, sb.toString()); 357 try { 358 props.store(new FileWriter(desc), "Saved from Nuxeo SetupWizard"); 359 } catch (IOException e) { 360 log.error("Unable to save package selection", e); 361 } 362 } 363 364 protected File getLocalPackagesDescriptor() { 365 File desc = new File(getDownloadDirectory(), PACKAGES_XML); 366 if (desc.exists()) { 367 return desc; 368 } 369 return null; 370 } 371 372 public List<DownloadPackage> getSelectedPackages() { 373 List<DownloadPackage> pkgs = getPackageOptions().getPkg4Install(); 374 File[] listFiles = getDownloadDirectory().listFiles(); 375 for (DownloadPackage pkg : pkgs) { 376 for (File file : listFiles) { 377 if (file.getName().equals(pkg.getMd5())) { 378 // recheck md5 ??? 379 pkg.setLocalFile(file); 380 } 381 } 382 needToDownload(pkg); 383 } 384 return pkgs; 385 } 386 387 public void scheduleDownloadedPackagesForInstallation(String installationFilePath) throws IOException { 388 List<String> fileEntries = new ArrayList<>(); 389 fileEntries.add("init"); 390 391 List<DownloadPackage> pkgs = downloadOptions.getPkg4Install(); 392 List<String> pkgInstallIds = new ArrayList<>(); 393 for (DownloadPackage pkg : pkgs) { 394 if (pkg.isVirtual()) { 395 log.debug("No install for virtual package: " + pkg.getId()); 396 } else if (pkg.isAlreadyInLocal() || StringUtils.isBlank(pkg.getFilename())) { 397 // Blank filename means later downloaded 398 fileEntries.add("install " + pkg.getId()); 399 pkgInstallIds.add(pkg.getId()); 400 } else { 401 for (PendingDownload download : pendingDownloads) { 402 if (download.getPkg().equals(pkg)) { 403 if (download.getStatus() == PendingDownloadStatus.VERIFIED) { 404 File file = download.getDowloadingFile(); 405 fileEntries.add("add file:" + file.getAbsolutePath()); 406 fileEntries.add("install " + pkg.getId()); 407 pkgInstallIds.add(pkg.getId()); 408 } else { 409 log.error("One selected package has not been downloaded : " + pkg.getId()); 410 } 411 } 412 } 413 } 414 } 415 416 File installLog = new File(installationFilePath); 417 if (fileEntries.size() > 0) { 418 if (!installLog.exists()) { 419 File parent = installLog.getParentFile(); 420 if (!parent.exists()) { 421 parent.mkdirs(); 422 } 423 installLog.createNewFile(); 424 } 425 FileUtils.writeLines(installLog, fileEntries); 426 } else { 427 // Should not happen as the file always has "init" 428 if (installLog.exists()) { 429 installLog.delete(); 430 } 431 } 432 433 // Save presets 434 saveSelectedPackages(pkgs); 435 } 436 437 public List<PendingDownload> getPendingDownloads() { 438 return pendingDownloads; 439 } 440 441 public void reStartDownload(String id) { 442 for (PendingDownload pending : pendingDownloads) { 443 if (pending.getPkg().getId().equals(id)) { 444 if (Arrays.asList(PendingDownloadStatus.CORRUPTED, PendingDownloadStatus.ABORTED).contains( 445 pending.getStatus())) { 446 pendingDownloads.remove(pending); 447 startDownloadPackage(pending.getPkg()); 448 } 449 break; 450 } 451 } 452 } 453 454 public void startDownload() { 455 startDownload(downloadOptions.getPkg4Install()); 456 } 457 458 public void startDownload(List<DownloadPackage> pkgs) { 459 downloadStarted = true; 460 for (DownloadPackage pkg : pkgs) { 461 if (needToDownload(pkg)) { 462 startDownloadPackage(pkg); 463 } 464 } 465 } 466 467 protected boolean needToDownload(DownloadPackage pkg) { 468 return !pkg.isVirtual() && !pkg.isLaterDownload() && !pkg.isAlreadyInLocal(); 469 } 470 471 protected void startDownloadPackage(final DownloadPackage pkg) { 472 final PendingDownload download = new PendingDownload(pkg); 473 if (pendingDownloads.addIfAbsent(download)) { 474 Runnable downloadRunner = new Runnable() { 475 476 @Override 477 public void run() { 478 log.info("Starting download on Thread " + Thread.currentThread().getName()); 479 download.setStatus(PendingDownloadStatus.INPROGRESS); 480 String url = pkg.getDownloadUrl(); 481 if (!url.startsWith("http")) { 482 url = getBaseUrl() + url; 483 } 484 File filePkg = null; 485 HttpGet dw = new HttpGet(url); 486 try { 487 HttpResponse response = httpClient.execute(dw); 488 if (response.getStatusLine().getStatusCode() == 200) { 489 filePkg = new File(getDownloadDirectory(), pkg.filename); 490 Header clh = response.getFirstHeader("Content-Length"); 491 if (clh != null) { 492 long filesize = Long.parseLong(clh.getValue()); 493 download.setFile(filesize, filePkg); 494 } 495 FileUtils.copyToFile(response.getEntity().getContent(), filePkg); 496 download.setStatus(PendingDownloadStatus.COMPLETED); 497 } else if (response.getStatusLine().getStatusCode() == 404) { 498 log.error("Package " + pkg.filename + " not found :" + url); 499 download.setStatus(PendingDownloadStatus.MISSING); 500 EntityUtils.consume(response.getEntity()); 501 dw.abort(); 502 return; 503 } else { 504 log.error("Received StatusCode " + response.getStatusLine().getStatusCode()); 505 download.setStatus(PendingDownloadStatus.ABORTED); 506 EntityUtils.consume(response.getEntity()); 507 dw.abort(); 508 return; 509 } 510 } catch (Exception e) { 511 download.setStatus(PendingDownloadStatus.ABORTED); 512 log.error("Error during download", e); 513 return; 514 } 515 checkPackage(download); 516 } 517 }; 518 download_tpe.execute(downloadRunner); 519 } 520 } 521 522 protected void checkPackage(final PendingDownload download) { 523 final File filePkg = download.getDowloadingFile(); 524 Runnable checkRunner = new Runnable() { 525 @Override 526 public void run() { 527 download.setStatus(PendingDownloadStatus.VERIFICATION); 528 String expectedDigest = download.getPkg().getMd5(); 529 String digest = getDigest(filePkg); 530 if (digest == null || (expectedDigest != null && !expectedDigest.equals(digest))) { 531 download.setStatus(PendingDownloadStatus.CORRUPTED); 532 log.error("Digest check failed: expected=" + expectedDigest + " computed=" + digest); 533 return; 534 } 535 File newFile = new File(getDownloadDirectory(), digest); 536 filePkg.renameTo(newFile); 537 download.setStatus(PendingDownloadStatus.VERIFIED); 538 download.setFile(newFile.length(), newFile); 539 } 540 }; 541 check_tpe.execute(checkRunner); 542 } 543 544 protected String getDigest(File file) { 545 try { 546 MessageDigest md = MessageDigest.getInstance(DIGEST); 547 byte[] buffer = new byte[DIGEST_CHUNK]; 548 InputStream stream = new FileInputStream(file); 549 int bytesRead = -1; 550 while ((bytesRead = stream.read(buffer)) >= 0) { 551 md.update(buffer, 0, bytesRead); 552 } 553 stream.close(); 554 byte[] b = md.digest(); 555 return md5ToHex(b); 556 } catch (Exception e) { 557 log.error("Error while computing Digest ", e); 558 return null; 559 } 560 } 561 562 protected static String md5ToHex(byte[] hash) { 563 StringBuilder hexString = new StringBuilder(); 564 for (byte b : hash) { 565 String hex = Integer.toHexString(0xFF & b); 566 if (hex.length() == 1) { 567 hexString.append('0'); 568 } 569 hexString.append(hex); 570 } 571 return hexString.toString(); 572 } 573 574 public boolean isDownloadStarted() { 575 return downloadStarted; 576 } 577 578 public boolean isDownloadCompleted() { 579 if (!isDownloadStarted()) { 580 return false; 581 } 582 for (PendingDownload download : pendingDownloads) { 583 if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()) { 584 return false; 585 } 586 } 587 return true; 588 } 589 590 public boolean isDownloadInProgress() { 591 if (!isDownloadStarted()) { 592 return false; 593 } 594 if (isDownloadCompleted()) { 595 return false; 596 } 597 int nbInProgress = 0; 598 for (PendingDownload download : pendingDownloads) { 599 if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue() 600 && download.getStatus().getValue() >= PendingDownloadStatus.PENDING.getValue()) { 601 nbInProgress++; 602 } 603 } 604 return nbInProgress > 0; 605 } 606 607 public void shutdown() { 608 if (httpClient != null) { 609 httpClient.getConnectionManager().shutdown(); 610 } 611 download_tpe.shutdownNow(); 612 check_tpe.shutdownNow(); 613 } 614 615}