001/* 002 * (C) Copyright 2011-2015 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * tdelprat 018 * 019 */ 020 021package org.nuxeo.wizard.download; 022 023import java.io.File; 024import java.io.FileInputStream; 025import java.io.FileNotFoundException; 026import java.io.FileReader; 027import java.io.FileWriter; 028import java.io.IOException; 029import java.io.InputStream; 030import java.security.MessageDigest; 031import java.util.ArrayList; 032import java.util.Arrays; 033import java.util.Collections; 034import java.util.List; 035import java.util.Properties; 036import java.util.concurrent.CopyOnWriteArrayList; 037import java.util.concurrent.LinkedBlockingQueue; 038import java.util.concurrent.ThreadFactory; 039import java.util.concurrent.ThreadPoolExecutor; 040import java.util.concurrent.TimeUnit; 041import java.util.concurrent.atomic.AtomicInteger; 042 043import org.apache.commons.lang.StringUtils; 044import org.apache.commons.logging.Log; 045import org.apache.commons.logging.LogFactory; 046import org.apache.http.Header; 047import org.apache.http.HttpHost; 048import org.apache.http.HttpResponse; 049import org.apache.http.auth.AuthScope; 050import org.apache.http.auth.NTCredentials; 051import org.apache.http.auth.UsernamePasswordCredentials; 052import org.apache.http.client.methods.HttpGet; 053import org.apache.http.conn.params.ConnRoutePNames; 054import org.apache.http.conn.scheme.PlainSocketFactory; 055import org.apache.http.conn.scheme.Scheme; 056import org.apache.http.conn.scheme.SchemeRegistry; 057import org.apache.http.conn.ssl.SSLSocketFactory; 058import org.apache.http.impl.client.DefaultHttpClient; 059import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; 060import org.apache.http.params.BasicHttpParams; 061import org.apache.http.params.HttpParams; 062import org.apache.http.params.HttpProtocolParams; 063import org.apache.http.util.EntityUtils; 064 065import org.nuxeo.common.Environment; 066import org.nuxeo.common.utils.FileUtils; 067import org.nuxeo.launcher.config.ConfigurationGenerator; 068 069/** 070 * @author Tiry (tdelprat@nuxeo.com) 071 */ 072public class PackageDownloader { 073 074 protected final static Log log = LogFactory.getLog(PackageDownloader.class); 075 076 public static final String PACKAGES_XML = "packages.xml"; 077 078 public static final String PACKAGES_DEFAULT_SELECTION = "packages-default-selection.properties"; 079 080 public static final String PACKAGES_DEFAULT_SELECTION_PRESETS = "preset"; 081 082 public static final String PACKAGES_DEFAULT_SELECTION_PACKAGES = "packages"; 083 084 protected static final int NB_DOWNLOAD_THREADS = 3; 085 086 protected static final int NB_CHECK_THREADS = 1; 087 088 protected static final int QUEUESIZE = 20; 089 090 public static final String BASE_URL_KEY = "nuxeo.wizard.packages.url"; 091 092 public static final String DEFAULT_BASE_URL = "http://cdn.nuxeo.com/"; // nuxeo-XXX/mp 093 094 protected CopyOnWriteArrayList<PendingDownload> pendingDownloads = new CopyOnWriteArrayList<>(); 095 096 protected static PackageDownloader instance; 097 098 protected DefaultHttpClient httpClient; 099 100 protected Boolean canReachServer = null; 101 102 protected DownloadablePackageOptions downloadOptions; 103 104 protected static final String DIGEST = "MD5"; 105 106 protected static final int DIGEST_CHUNK = 1024 * 100; 107 108 boolean downloadStarted = false; 109 110 protected String lastSelectionDigest; 111 112 protected final AtomicInteger dwThreadCount = new AtomicInteger(0); 113 114 protected final AtomicInteger checkThreadCount = new AtomicInteger(0); 115 116 protected String baseUrl; 117 118 protected ConfigurationGenerator configurationGenerator = null; 119 120 protected ConfigurationGenerator getConfig() { 121 if (configurationGenerator == null) { 122 configurationGenerator = new ConfigurationGenerator(); 123 configurationGenerator.init(); 124 } 125 return configurationGenerator; 126 } 127 128 protected String getBaseUrl() { 129 if (baseUrl == null) { 130 String base = getConfig().getUserConfig().getProperty(BASE_URL_KEY, ""); 131 if ("".equals(base)) { 132 base = DEFAULT_BASE_URL + "nuxeo-" 133 + getConfig().getUserConfig().getProperty(Environment.DISTRIBUTION_VERSION) + "/mp/"; 134 } 135 if (!base.endsWith("/")) { 136 base = base + "/"; 137 } 138 baseUrl = base; 139 } 140 return baseUrl; 141 } 142 143 protected ThreadPoolExecutor download_tpe = new ThreadPoolExecutor(NB_DOWNLOAD_THREADS, NB_DOWNLOAD_THREADS, 10L, 144 TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() { 145 @Override 146 public Thread newThread(Runnable r) { 147 Thread t = new Thread(r); 148 t.setDaemon(true); 149 t.setName("DownloaderThread-" + dwThreadCount.incrementAndGet()); 150 return t; 151 } 152 }); 153 154 protected ThreadPoolExecutor check_tpe = new ThreadPoolExecutor(NB_CHECK_THREADS, NB_CHECK_THREADS, 10L, 155 TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(QUEUESIZE), new ThreadFactory() { 156 @Override 157 public Thread newThread(Runnable r) { 158 Thread t = new Thread(r); 159 t.setDaemon(true); 160 t.setName("MD5CheckThread-" + checkThreadCount.incrementAndGet()); 161 return t; 162 } 163 }); 164 165 protected PackageDownloader() { 166 SchemeRegistry registry = new SchemeRegistry(); 167 registry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); 168 registry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); 169 HttpParams httpParams = new BasicHttpParams(); 170 HttpProtocolParams.setUseExpectContinue(httpParams, false); 171 ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(registry); 172 cm.setMaxTotal(NB_DOWNLOAD_THREADS); 173 cm.setDefaultMaxPerRoute(NB_DOWNLOAD_THREADS); 174 httpClient = new DefaultHttpClient(cm, httpParams); 175 } 176 177 public synchronized static PackageDownloader instance() { 178 if (instance == null) { 179 instance = new PackageDownloader(); 180 instance.download_tpe.prestartAllCoreThreads(); 181 instance.check_tpe.prestartAllCoreThreads(); 182 } 183 return instance; 184 } 185 186 public static void reset() { 187 if (instance != null) { 188 instance.shutdown(); 189 instance = null; 190 } 191 } 192 193 public void setProxy(String proxy, int port, String login, String password, String NTLMHost, String NTLMDomain) { 194 if (proxy != null) { 195 HttpHost proxyHost = new HttpHost(proxy, port); 196 httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxyHost); 197 if (login != null) { 198 if (NTLMHost != null && !NTLMHost.trim().isEmpty()) { 199 NTCredentials ntlmCredentials = new NTCredentials(login, password, NTLMHost, NTLMDomain); 200 httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), ntlmCredentials); 201 } else { 202 httpClient.getCredentialsProvider().setCredentials(new AuthScope(proxy, port), 203 new UsernamePasswordCredentials(login, password)); 204 } 205 } else { 206 httpClient.getCredentialsProvider().clear(); 207 } 208 } else { 209 httpClient.getParams().removeParameter(ConnRoutePNames.DEFAULT_PROXY); 210 httpClient.getCredentialsProvider().clear(); 211 } 212 } 213 214 protected String getSelectionDigest(List<String> ids) { 215 ArrayList<String> lst = new ArrayList<>(ids); 216 Collections.sort(lst); 217 StringBuffer sb = new StringBuffer(); 218 for (String item : lst) { 219 sb.append(item); 220 sb.append(":"); 221 } 222 return sb.toString(); 223 } 224 225 public void selectOptions(List<String> ids) { 226 String newSelectionDigest = getSelectionDigest(ids); 227 if (lastSelectionDigest != null) { 228 if (lastSelectionDigest.equals(newSelectionDigest)) { 229 return; 230 } 231 } 232 getPackageOptions().select(ids); 233 downloadStarted = false; 234 lastSelectionDigest = newSelectionDigest; 235 } 236 237 protected File getDownloadDirectory() { 238 File mpDir = getConfig().getDistributionMPDir(); 239 if (!mpDir.exists()) { 240 mpDir.mkdirs(); 241 } 242 return mpDir; 243 } 244 245 public boolean canReachServer() { 246 if (canReachServer == null) { 247 HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML); 248 try { 249 HttpResponse response = httpClient.execute(ping); 250 if (response.getStatusLine().getStatusCode() == 200) { 251 canReachServer = true; 252 } else { 253 log.info("Unable to ping server -> status code :" + response.getStatusLine().getStatusCode() + " (" 254 + response.getStatusLine().getReasonPhrase() + ")"); 255 canReachServer = false; 256 } 257 } catch (Exception e) { 258 log.info("Unable to ping remote server " + e.getMessage()); 259 log.debug("Unable to ping remote server", e); 260 canReachServer = false; 261 } 262 } 263 return canReachServer; 264 } 265 266 public DownloadablePackageOptions getPackageOptions() { 267 if (downloadOptions == null) { 268 File packageFile = null; 269 if (canReachServer()) { 270 packageFile = getRemotePackagesDescriptor(); 271 } 272 if (packageFile == null) { 273 packageFile = getLocalPackagesDescriptor(); 274 if (packageFile == null) { 275 log.warn("Unable to find local copy of packages.xml"); 276 } else { 277 log.info("Wizard will use the local copy of packages.xml."); 278 } 279 } 280 if (packageFile != null) { 281 try { 282 downloadOptions = DownloadDescriptorParser.parsePackages(new FileInputStream(packageFile)); 283 284 // manage init from presets if available 285 Properties defaultSelection = getDefaultPackageSelection(); 286 if (defaultSelection != null) { 287 String presetId = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PRESETS, null); 288 if (presetId != null && !presetId.isEmpty()) { 289 for (Preset preset : downloadOptions.getPresets()) { 290 if (preset.getId().equals(presetId)) { 291 List<String> pkgIds = Arrays.asList(preset.getPkgs()); 292 downloadOptions.select(pkgIds); 293 break; 294 } 295 } 296 } else { 297 String pkgIdsList = defaultSelection.getProperty(PACKAGES_DEFAULT_SELECTION_PACKAGES, null); 298 if (pkgIdsList != null && !pkgIdsList.isEmpty()) { 299 String[] ids = pkgIdsList.split(","); 300 List<String> pkgIds = Arrays.asList(ids); 301 downloadOptions.select(pkgIds); 302 } 303 } 304 } 305 } catch (FileNotFoundException e) { 306 log.error("Unable to read packages.xml", e); 307 } 308 } 309 } 310 return downloadOptions; 311 } 312 313 protected File getRemotePackagesDescriptor() { 314 File desc = null; 315 HttpGet ping = new HttpGet(getBaseUrl() + PACKAGES_XML); 316 try { 317 HttpResponse response = httpClient.execute(ping); 318 if (response.getStatusLine().getStatusCode() == 200) { 319 desc = new File(getDownloadDirectory(), PACKAGES_XML); 320 FileUtils.copyToFile(response.getEntity().getContent(), desc); 321 } else { 322 log.warn("Unable to download remote packages.xml, status code :" 323 + response.getStatusLine().getStatusCode() + " (" + response.getStatusLine().getReasonPhrase() 324 + ")"); 325 return null; 326 } 327 } catch (Exception e) { 328 log.warn("Unable to reach remote packages.xml", e); 329 return null; 330 } 331 return desc; 332 } 333 334 protected Properties getDefaultPackageSelection() { 335 File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION); 336 if (desc.exists()) { 337 try { 338 Properties props = new Properties(); 339 props.load(new FileReader(desc)); 340 return props; 341 } catch (IOException e) { 342 log.warn("Unable to load presets", e); 343 } 344 } 345 return null; 346 } 347 348 protected void saveSelectedPackages(List<DownloadPackage> pkgs) { 349 File desc = new File(getDownloadDirectory(), PACKAGES_DEFAULT_SELECTION); 350 Properties props = new Properties(); 351 StringBuffer sb = new StringBuffer(); 352 for (int i = 0; i < pkgs.size(); i++) { 353 if (i > 0) { 354 sb.append(","); 355 } 356 sb.append(pkgs.get(i).getId()); 357 } 358 props.put(PACKAGES_DEFAULT_SELECTION_PACKAGES, sb.toString()); 359 try { 360 props.store(new FileWriter(desc), "Saved from Nuxeo SetupWizard"); 361 } catch (IOException e) { 362 log.error("Unable to save package selection", e); 363 } 364 } 365 366 protected File getLocalPackagesDescriptor() { 367 File desc = new File(getDownloadDirectory(), PACKAGES_XML); 368 if (desc.exists()) { 369 return desc; 370 } 371 return null; 372 } 373 374 public List<DownloadPackage> getSelectedPackages() { 375 List<DownloadPackage> pkgs = getPackageOptions().getPkg4Install(); 376 File[] listFiles = getDownloadDirectory().listFiles(); 377 for (DownloadPackage pkg : pkgs) { 378 for (File file : listFiles) { 379 if (file.getName().equals(pkg.getMd5())) { 380 // recheck md5 ??? 381 pkg.setLocalFile(file); 382 } 383 } 384 needToDownload(pkg); 385 } 386 return pkgs; 387 } 388 389 public void scheduleDownloadedPackagesForInstallation(String installationFilePath) throws IOException { 390 List<String> fileEntries = new ArrayList<>(); 391 fileEntries.add("init"); 392 393 List<DownloadPackage> pkgs = downloadOptions.getPkg4Install(); 394 List<String> pkgInstallIds = new ArrayList<>(); 395 for (DownloadPackage pkg : pkgs) { 396 if (pkg.isVirtual()) { 397 log.debug("No install for virtual package: " + pkg.getId()); 398 } else if (pkg.isAlreadyInLocal() || StringUtils.isBlank(pkg.getFilename())) { 399 // Blank filename means later downloaded 400 fileEntries.add("install " + pkg.getId()); 401 pkgInstallIds.add(pkg.getId()); 402 } else { 403 for (PendingDownload download : pendingDownloads) { 404 if (download.getPkg().equals(pkg)) { 405 if (download.getStatus() == PendingDownloadStatus.VERIFIED) { 406 File file = download.getDowloadingFile(); 407 fileEntries.add("add file:" + file.getAbsolutePath()); 408 fileEntries.add("install " + pkg.getId()); 409 pkgInstallIds.add(pkg.getId()); 410 } else { 411 log.error("One selected package has not been downloaded : " + pkg.getId()); 412 } 413 } 414 } 415 } 416 } 417 418 File installLog = new File(installationFilePath); 419 if (fileEntries.size() > 0) { 420 if (!installLog.exists()) { 421 File parent = installLog.getParentFile(); 422 if (!parent.exists()) { 423 parent.mkdirs(); 424 } 425 installLog.createNewFile(); 426 } 427 FileUtils.writeLines(installLog, fileEntries); 428 } else { 429 // Should not happen as the file always has "init" 430 if (installLog.exists()) { 431 installLog.delete(); 432 } 433 } 434 435 // Save presets 436 saveSelectedPackages(pkgs); 437 } 438 439 public List<PendingDownload> getPendingDownloads() { 440 return pendingDownloads; 441 } 442 443 public void reStartDownload(String id) { 444 for (PendingDownload pending : pendingDownloads) { 445 if (pending.getPkg().getId().equals(id)) { 446 if (Arrays.asList(PendingDownloadStatus.CORRUPTED, PendingDownloadStatus.ABORTED).contains( 447 pending.getStatus())) { 448 pendingDownloads.remove(pending); 449 startDownloadPackage(pending.getPkg()); 450 } 451 break; 452 } 453 } 454 } 455 456 public void startDownload() { 457 startDownload(downloadOptions.getPkg4Install()); 458 } 459 460 public void startDownload(List<DownloadPackage> pkgs) { 461 downloadStarted = true; 462 for (DownloadPackage pkg : pkgs) { 463 if (needToDownload(pkg)) { 464 startDownloadPackage(pkg); 465 } 466 } 467 } 468 469 protected boolean needToDownload(DownloadPackage pkg) { 470 return !pkg.isVirtual() && !pkg.isLaterDownload() && !pkg.isAlreadyInLocal(); 471 } 472 473 protected void startDownloadPackage(final DownloadPackage pkg) { 474 final PendingDownload download = new PendingDownload(pkg); 475 if (pendingDownloads.addIfAbsent(download)) { 476 Runnable downloadRunner = new Runnable() { 477 478 @Override 479 public void run() { 480 log.info("Starting download on Thread " + Thread.currentThread().getName()); 481 download.setStatus(PendingDownloadStatus.INPROGRESS); 482 String url = pkg.getDownloadUrl(); 483 if (!url.startsWith("http")) { 484 url = getBaseUrl() + url; 485 } 486 File filePkg = null; 487 HttpGet dw = new HttpGet(url); 488 try { 489 HttpResponse response = httpClient.execute(dw); 490 if (response.getStatusLine().getStatusCode() == 200) { 491 filePkg = new File(getDownloadDirectory(), pkg.filename); 492 Header clh = response.getFirstHeader("Content-Length"); 493 if (clh != null) { 494 long filesize = Long.parseLong(clh.getValue()); 495 download.setFile(filesize, filePkg); 496 } 497 FileUtils.copyToFile(response.getEntity().getContent(), filePkg); 498 download.setStatus(PendingDownloadStatus.COMPLETED); 499 } else if (response.getStatusLine().getStatusCode() == 404) { 500 log.error("Package " + pkg.filename + " not found :" + url); 501 download.setStatus(PendingDownloadStatus.MISSING); 502 EntityUtils.consume(response.getEntity()); 503 dw.abort(); 504 return; 505 } else { 506 log.error("Received StatusCode " + response.getStatusLine().getStatusCode()); 507 download.setStatus(PendingDownloadStatus.ABORTED); 508 EntityUtils.consume(response.getEntity()); 509 dw.abort(); 510 return; 511 } 512 } catch (Exception e) { 513 download.setStatus(PendingDownloadStatus.ABORTED); 514 log.error("Error during download", e); 515 return; 516 } 517 checkPackage(download); 518 } 519 }; 520 download_tpe.execute(downloadRunner); 521 } 522 } 523 524 protected void checkPackage(final PendingDownload download) { 525 final File filePkg = download.getDowloadingFile(); 526 Runnable checkRunner = new Runnable() { 527 @Override 528 public void run() { 529 download.setStatus(PendingDownloadStatus.VERIFICATION); 530 String expectedDigest = download.getPkg().getMd5(); 531 String digest = getDigest(filePkg); 532 if (digest == null || (expectedDigest != null && !expectedDigest.equals(digest))) { 533 download.setStatus(PendingDownloadStatus.CORRUPTED); 534 log.error("Digest check failed: expected=" + expectedDigest + " computed=" + digest); 535 return; 536 } 537 File newFile = new File(getDownloadDirectory(), digest); 538 filePkg.renameTo(newFile); 539 download.setStatus(PendingDownloadStatus.VERIFIED); 540 download.setFile(newFile.length(), newFile); 541 } 542 }; 543 check_tpe.execute(checkRunner); 544 } 545 546 protected String getDigest(File file) { 547 try { 548 MessageDigest md = MessageDigest.getInstance(DIGEST); 549 byte[] buffer = new byte[DIGEST_CHUNK]; 550 InputStream stream = new FileInputStream(file); 551 int bytesRead = -1; 552 while ((bytesRead = stream.read(buffer)) >= 0) { 553 md.update(buffer, 0, bytesRead); 554 } 555 stream.close(); 556 byte[] b = md.digest(); 557 return md5ToHex(b); 558 } catch (Exception e) { 559 log.error("Error while computing Digest ", e); 560 return null; 561 } 562 } 563 564 protected static String md5ToHex(byte[] hash) { 565 StringBuilder hexString = new StringBuilder(); 566 for (byte b : hash) { 567 String hex = Integer.toHexString(0xFF & b); 568 if (hex.length() == 1) { 569 hexString.append('0'); 570 } 571 hexString.append(hex); 572 } 573 return hexString.toString(); 574 } 575 576 public boolean isDownloadStarted() { 577 return downloadStarted; 578 } 579 580 public boolean isDownloadCompleted() { 581 if (!isDownloadStarted()) { 582 return false; 583 } 584 for (PendingDownload download : pendingDownloads) { 585 if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue()) { 586 return false; 587 } 588 } 589 return true; 590 } 591 592 public boolean isDownloadInProgress() { 593 if (!isDownloadStarted()) { 594 return false; 595 } 596 if (isDownloadCompleted()) { 597 return false; 598 } 599 int nbInProgress = 0; 600 for (PendingDownload download : pendingDownloads) { 601 if (download.getStatus().getValue() < PendingDownloadStatus.VERIFIED.getValue() 602 && download.getStatus().getValue() >= PendingDownloadStatus.PENDING.getValue()) { 603 nbInProgress++; 604 } 605 } 606 return nbInProgress > 0; 607 } 608 609 public void shutdown() { 610 if (httpClient != null) { 611 httpClient.getConnectionManager().shutdown(); 612 } 613 download_tpe.shutdownNow(); 614 check_tpe.shutdownNow(); 615 } 616 617}