001/* 002 * (C) Copyright 2010-2020 Nuxeo (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Julien Carsique 018 * Kevin Leturc <kleturc@nuxeo.com> 019 */ 020package org.nuxeo.launcher.config; 021 022import static java.nio.charset.StandardCharsets.UTF_8; 023import static org.apache.commons.lang3.StringUtils.isNotBlank; 024import static org.nuxeo.launcher.config.ConfigurationGenerator.NUXEO_PROFILES; 025import static org.nuxeo.launcher.config.ConfigurationGenerator.TEMPLATE_SEPARATOR; 026 027import java.io.BufferedReader; 028import java.io.BufferedWriter; 029import java.io.File; 030import java.io.FileNotFoundException; 031import java.io.FileOutputStream; 032import java.io.FileReader; 033import java.io.FilenameFilter; 034import java.io.IOException; 035import java.io.OutputStreamWriter; 036import java.net.InetAddress; 037import java.nio.file.Files; 038import java.nio.file.Path; 039import java.util.ArrayList; 040import java.util.Arrays; 041import java.util.Enumeration; 042import java.util.HashSet; 043import java.util.List; 044import java.util.Properties; 045import java.util.Set; 046import java.util.StringTokenizer; 047import java.util.TreeSet; 048 049import org.apache.commons.io.FileUtils; 050import org.apache.logging.log4j.LogManager; 051import org.apache.logging.log4j.Logger; 052import org.apache.logging.log4j.core.config.Configurator; 053import org.apache.logging.log4j.core.config.DefaultConfiguration; 054import org.nuxeo.common.Environment; 055import org.nuxeo.common.codec.Crypto; 056import org.nuxeo.common.codec.CryptoProperties; 057import org.nuxeo.common.utils.TextTemplate; 058import org.nuxeo.connect.update.LocalPackage; 059import org.nuxeo.launcher.info.ConfigurationInfo; 060import org.nuxeo.launcher.info.DistributionInfo; 061import org.nuxeo.launcher.info.InstanceInfo; 062import org.nuxeo.launcher.info.KeyValueInfo; 063import org.nuxeo.launcher.info.PackageInfo; 064import org.nuxeo.log4j.Log4JHelper; 065 066import freemarker.template.TemplateException; 067 068/** 069 * @author jcarsique 070 * @implNote since 11.1, configurator only handles Tomcat and is no more abstract 071 */ 072public class ServerConfigurator { 073 074 private static final Logger log = LogManager.getLogger(ServerConfigurator.class); 075 076 /** @since 5.4.2 */ 077 public static final String TOMCAT_STARTUP_CLASS = "org.apache.catalina.startup.Bootstrap"; 078 079 /** @since 5.6 */ 080 public static final String TOMCAT_HOME = "tomcat.home"; 081 082 /** @since 5.7 */ 083 public static final String PARAM_HTTP_TOMCAT_ADMIN_PORT = "nuxeo.server.tomcat_admin.port"; 084 085 /** 086 * @since 5.4.2 087 */ 088 public static final List<String> NUXEO_SYSTEM_PROPERTIES = List.of("nuxeo.conf", "nuxeo.home", "log.id"); 089 090 protected static final String DEFAULT_CONTEXT_NAME = "/nuxeo"; 091 092 /** @since 9.3 */ 093 public static final String JAVA_OPTS = "JAVA_OPTS"; 094 095 private static final String NEW_FILES = ConfigurationGenerator.TEMPLATES + File.separator + "files.list"; 096 097 protected final ConfigurationGenerator generator; 098 099 protected File dataDir = null; 100 101 protected File logDir = null; 102 103 protected File pidDir = null; 104 105 protected File tmpDir = null; 106 107 protected File packagesDir = null; 108 109 private String contextName = null; 110 111 public ServerConfigurator(ConfigurationGenerator configurationGenerator) { 112 generator = configurationGenerator; 113 } 114 115 /** 116 * @return true if server configuration files already exist 117 */ 118 protected boolean isConfigured() { 119 Path nuxeoContext = Path.of("conf", "Catalina", "localhost", getContextName() + ".xml"); 120 return Files.exists(generator.getNuxeoHome().toPath().resolve(nuxeoContext)); 121 } 122 123 /** 124 * @return Configured context name 125 * @since 5.4.2 126 */ 127 public String getContextName() { 128 if (contextName == null) { 129 Properties userConfig = generator.getUserConfig(); 130 if (userConfig != null) { 131 contextName = userConfig.getProperty(ConfigurationGenerator.PARAM_CONTEXT_PATH, DEFAULT_CONTEXT_NAME); 132 } else { 133 contextName = DEFAULT_CONTEXT_NAME; 134 } 135 contextName = contextName.substring(1); 136 } 137 return contextName; 138 } 139 140 /** 141 * Generate configuration files from templates and given configuration parameters 142 * 143 * @param config Properties with configuration parameters for template replacement 144 */ 145 protected void parseAndCopy(Properties config) throws IOException, TemplateException, ConfigurationException { 146 // FilenameFilter for excluding "nuxeo.defaults" files from copy 147 String nuxeoEnvironmentConf = generator.getNuxeoEnvironmentConfName(); 148 final FilenameFilter filter = (dir, name) -> !ConfigurationGenerator.NUXEO_DEFAULT_CONF.equals(name) 149 && !nuxeoEnvironmentConf.equals(name); 150 final TextTemplate templateParser = new TextTemplate(config); 151 templateParser.setKeepEncryptedAsVar(true); 152 templateParser.setTrim(true); 153 templateParser.setTextParsingExtensions( 154 config.getProperty(ConfigurationGenerator.PARAM_TEMPLATES_PARSING_EXTENSIONS, "xml,properties,nx")); 155 templateParser.setFreemarkerParsingExtensions( 156 config.getProperty(ConfigurationGenerator.PARAM_TEMPLATES_FREEMARKER_EXTENSIONS, "nxftl")); 157 158 deleteTemplateFiles(); 159 // add included templates directories 160 List<String> newFilesList = new ArrayList<>(); 161 for (File includedTemplate : generator.getIncludedTemplates()) { 162 File[] listFiles = includedTemplate.listFiles(filter); 163 if (listFiles != null) { 164 String templateName = includedTemplate.getName(); 165 log.debug("Parsing {}... {}", () -> templateName, () -> Arrays.toString(listFiles)); 166 // Check for deprecation 167 boolean isDeprecated = Boolean.parseBoolean(config.getProperty(templateName + ".deprecated")); 168 if (isDeprecated) { 169 log.warn("WARNING: Template {} is deprecated.", templateName); 170 String deprecationMessage = config.getProperty(templateName + ".deprecation"); 171 if (deprecationMessage != null) { 172 log.warn(deprecationMessage); 173 } 174 } 175 // Retrieve optional target directory if defined 176 String outputDirectoryStr = config.getProperty(templateName + ".target"); 177 File outputDirectory = (outputDirectoryStr != null) 178 ? new File(generator.getNuxeoHome(), outputDirectoryStr) 179 : getOutputDirectory(); 180 for (File in : listFiles) { 181 // copy template(s) directories parsing properties 182 newFilesList.addAll(templateParser.processDirectory(in, new File(outputDirectory, in.getName()))); 183 } 184 } 185 } 186 storeNewFilesList(newFilesList); 187 } 188 189 /** 190 * Delete files previously deployed by templates. If a file had been overwritten by a template, it will be restored. 191 * Helps the server returning to the state before any template was applied. 192 */ 193 private void deleteTemplateFiles() throws IOException, ConfigurationException { 194 File newFiles = new File(generator.getNuxeoHome(), NEW_FILES); 195 if (!newFiles.exists()) { 196 return; 197 } 198 try (BufferedReader reader = new BufferedReader(new FileReader(newFiles))) { 199 String line; 200 while ((line = reader.readLine()) != null) { 201 if (line.endsWith(".bak")) { 202 log.debug("Restore {}", line); 203 String originalName = line.substring(0, line.length() - 4); 204 try { 205 File backup = new File(generator.getNuxeoHome(), line); 206 File original = new File(generator.getNuxeoHome(), originalName); 207 FileUtils.copyFile(backup, original); 208 backup.delete(); 209 } catch (IOException e) { 210 throw new ConfigurationException( 211 String.format("Failed to restore %s from %s\nEdit or delete %s to bypass that error.", 212 originalName, line, newFiles), 213 e); 214 } 215 } else { 216 log.debug("Remove {}", line); 217 new File(generator.getNuxeoHome(), line).delete(); 218 } 219 } 220 } 221 newFiles.delete(); 222 } 223 224 /** 225 * Store into {@link #NEW_FILES} the list of new files deployed by the templates. For later use by 226 * {@link #deleteTemplateFiles()} 227 */ 228 private void storeNewFilesList(List<String> newFilesList) throws IOException { 229 File newFiles = new File(generator.getNuxeoHome(), NEW_FILES); 230 try (BufferedWriter writer = new BufferedWriter( 231 new OutputStreamWriter(new FileOutputStream(newFiles, false), UTF_8))) { 232 // Store new files listing 233 int index = generator.getNuxeoHome().getCanonicalPath().length() + 1; 234 for (String filepath : newFilesList) { 235 writer.write(new File(filepath).getCanonicalPath().substring(index)); 236 writer.newLine(); 237 } 238 } 239 } 240 241 /** 242 * @return output directory for files generation 243 */ 244 protected File getOutputDirectory() { 245 return getRuntimeHome(); 246 } 247 248 /** 249 * @return Default data directory path relative to Nuxeo Home 250 * @since 5.4.2 251 */ 252 protected String getDefaultDataDir() { 253 return "nxserver" + File.separator + Environment.DEFAULT_DATA_DIR; 254 } 255 256 /** 257 * Returns the Home of NuxeoRuntime (same as Framework.getRuntime().getHome().getAbsolutePath()) 258 */ 259 protected File getRuntimeHome() { 260 return new File(generator.getNuxeoHome(), "nxserver"); 261 } 262 263 /** 264 * @return Data directory 265 * @since 5.4.2 266 */ 267 public File getDataDir() { 268 if (dataDir == null) { 269 dataDir = new File(generator.getNuxeoHome(), getDefaultDataDir()); 270 } 271 return dataDir; 272 } 273 274 /** 275 * @return Log directory 276 * @since 5.4.2 277 */ 278 public File getLogDir() { 279 if (logDir == null) { 280 logDir = new File(generator.getNuxeoHome(), Environment.DEFAULT_LOG_DIR); 281 } 282 return logDir; 283 } 284 285 /** 286 * @param dataDirStr Data directory path to set 287 * @since 5.4.2 288 */ 289 public void setDataDir(String dataDirStr) { 290 dataDir = new File(dataDirStr); 291 dataDir.mkdirs(); 292 } 293 294 /** 295 * @param logDirStr Log directory path to set 296 * @since 5.4.2 297 */ 298 public void setLogDir(String logDirStr) { 299 logDir = new File(logDirStr); 300 logDir.mkdirs(); 301 } 302 303 /** 304 * Initialize logs. This is called before {@link ConfigurationGenerator#init()} so the {@code logDir} field is not 305 * yet initialized 306 * 307 * @since 5.4.2 308 */ 309 public void initLogs() { 310 File logFile = getLogConfFile(); 311 String logDirectory = System.getProperty(Environment.NUXEO_LOG_DIR); 312 if (logDirectory == null) { 313 System.setProperty(Environment.NUXEO_LOG_DIR, getLogDir().getPath()); 314 } 315 if (logFile == null || !logFile.exists()) { 316 System.out.println("No logs configuration, will setup a basic one."); 317 Configurator.initialize(new DefaultConfiguration()); 318 } else { 319 System.out.println("Try to configure logs with " + logFile); 320 Configurator.initialize(Log4JHelper.newConfiguration(logFile)); 321 } 322 log.info("Logs successfully configured."); 323 } 324 325 /** 326 * @return Pid directory (usually known as "run directory"); Returns log directory if not set by configuration. 327 * @since 5.4.2 328 */ 329 public File getPidDir() { 330 if (pidDir == null) { 331 pidDir = getLogDir(); 332 } 333 return pidDir; 334 } 335 336 /** 337 * @param pidDirStr Pid directory path to set 338 * @since 5.4.2 339 */ 340 public void setPidDir(String pidDirStr) { 341 pidDir = new File(pidDirStr); 342 pidDir.mkdirs(); 343 } 344 345 /** 346 * Check server paths; warn if existing deprecated paths. Override this method to perform server specific checks. 347 * 348 * @throws ConfigurationException If deprecated paths have been detected 349 * @since 5.4.2 350 */ 351 public void checkPaths() throws ConfigurationException { 352 File badInstanceClid = new File(generator.getNuxeoHome(), 353 getDefaultDataDir() + File.separator + "instance.clid"); 354 if (badInstanceClid.exists() && !getDataDir().equals(badInstanceClid.getParentFile())) { 355 log.warn("Moving {} to {}.", () -> badInstanceClid, this::getDataDir); 356 try { 357 FileUtils.moveFileToDirectory(badInstanceClid, getDataDir(), true); 358 } catch (IOException e) { 359 throw new ConfigurationException("NXP-6722 move failed: " + e.getMessage(), e); 360 } 361 } 362 363 File oldPackagesPath = new File(getDataDir(), getDefaultPackagesDir()); 364 if (oldPackagesPath.exists() && !oldPackagesPath.equals(getPackagesDir())) { 365 log.warn("NXP-8014 Packages cache location changed. You can safely delete {} or move its content to {}", 366 () -> oldPackagesPath, this::getPackagesDir); 367 } 368 } 369 370 /** 371 * @return Temporary directory 372 * @since 5.4.2 373 */ 374 public File getTmpDir() { 375 if (tmpDir == null) { 376 tmpDir = new File(generator.getNuxeoHome(), getDefaultTmpDir()); 377 } 378 return tmpDir; 379 } 380 381 /** 382 * @return Default temporary directory path relative to Nuxeo Home 383 * @since 5.4.2 384 */ 385 public String getDefaultTmpDir() { 386 return Environment.DEFAULT_TMP_DIR; 387 } 388 389 /** 390 * @param tmpDirStr Temporary directory path to set 391 * @since 5.4.2 392 */ 393 public void setTmpDir(String tmpDirStr) { 394 tmpDir = new File(tmpDirStr); 395 tmpDir.mkdirs(); 396 } 397 398 /** 399 * @see Environment 400 * @param key directory system key 401 * @param directory absolute or relative directory path 402 * @since 5.4.2 403 */ 404 public void setDirectory(String key, String directory) { 405 String absoluteDirectory = setAbsolutePath(key, directory); 406 if (Environment.NUXEO_DATA_DIR.equals(key)) { 407 setDataDir(absoluteDirectory); 408 } else if (Environment.NUXEO_LOG_DIR.equals(key)) { 409 setLogDir(absoluteDirectory); 410 } else if (Environment.NUXEO_PID_DIR.equals(key)) { 411 setPidDir(absoluteDirectory); 412 } else if (Environment.NUXEO_TMP_DIR.equals(key)) { 413 setTmpDir(absoluteDirectory); 414 } else if (Environment.NUXEO_MP_DIR.equals(key)) { 415 setPackagesDir(absoluteDirectory); 416 } else { 417 log.error("Unknown directory key: {}", key); 418 } 419 } 420 421 /** 422 * @since 5.9.4 423 */ 424 private void setPackagesDir(String packagesDirStr) { 425 packagesDir = new File(packagesDirStr); 426 packagesDir.mkdirs(); 427 } 428 429 /** 430 * Make absolute the directory passed in parameter. If it was relative, then store absolute path in user config 431 * instead of relative and return value 432 * 433 * @param key Directory system key 434 * @param directory absolute or relative directory path 435 * @return absolute directory path 436 * @since 5.4.2 437 */ 438 private String setAbsolutePath(String key, String directory) { 439 if (!new File(directory).isAbsolute()) { 440 directory = new File(generator.getNuxeoHome(), directory).getPath(); 441 generator.getUserConfig().setProperty(key, directory); 442 } 443 return directory; 444 } 445 446 /** 447 * @see Environment 448 * @param key directory system key 449 * @return Directory denoted by key 450 * @since 5.4.2 451 */ 452 public File getDirectory(String key) { 453 if (Environment.NUXEO_DATA_DIR.equals(key)) { 454 return getDataDir(); 455 } else if (Environment.NUXEO_LOG_DIR.equals(key)) { 456 return getLogDir(); 457 } else if (Environment.NUXEO_PID_DIR.equals(key)) { 458 return getPidDir(); 459 } else if (Environment.NUXEO_TMP_DIR.equals(key)) { 460 return getTmpDir(); 461 } else if (Environment.NUXEO_MP_DIR.equals(key)) { 462 return getPackagesDir(); 463 } else { 464 log.error("Unknown directory key: {}", key); 465 return null; 466 } 467 } 468 469 /** 470 * Check if oldPath exist; if so, then raise a ConfigurationException with information for fixing issue 471 * 472 * @param oldPath Path that must NOT exist 473 * @param message Error message thrown with exception 474 * @throws ConfigurationException If an old path has been discovered 475 */ 476 protected void checkPath(File oldPath, String message) throws ConfigurationException { 477 if (oldPath.exists()) { 478 log.error("Deprecated paths used."); 479 throw new ConfigurationException(message); 480 } 481 } 482 483 /** 484 * @return Log4J configuration file 485 * @since 5.4.2 486 */ 487 public File getLogConfFile() { 488 return new File(getServerLibDir(), "log4j2.xml"); 489 } 490 491 /** 492 * @return Nuxeo config directory 493 * @since 5.4.2 494 */ 495 public File getConfigDir() { 496 return new File(getRuntimeHome(), Environment.DEFAULT_CONFIG_DIR); 497 } 498 499 /** 500 * @param userConfig Properties to dump into config directory 501 * @since 5.4.2 502 */ 503 public void dumpProperties(CryptoProperties userConfig) { 504 Properties dumpedProperties = filterSystemProperties(userConfig); 505 File dumpedFile = generator.getDumpedConfig(); 506 try (OutputStreamWriter os = new OutputStreamWriter(new FileOutputStream(dumpedFile, false), UTF_8)) { 507 dumpedProperties.store(os, "Generated by " + getClass()); 508 } catch (FileNotFoundException e) { 509 log.error(e); 510 } catch (IOException e) { 511 log.error("Could not dump properties to {}", dumpedFile, e); 512 } 513 } 514 515 /** 516 * Extract Nuxeo properties from given Properties (System properties are removed, except those set by Nuxeo) 517 * 518 * @param properties Properties to be filtered 519 * @return copy of given properties filtered out of System properties 520 * @since 5.4.2 521 */ 522 public Properties filterSystemProperties(CryptoProperties properties) { 523 Properties dumpedProperties = new Properties(); 524 for (@SuppressWarnings("unchecked") 525 Enumeration<String> propertyNames = (Enumeration<String>) properties.propertyNames(); propertyNames.hasMoreElements();) { 526 String key = propertyNames.nextElement(); 527 // Exclude System properties except Nuxeo's System properties 528 if (!System.getProperties().containsKey(key) || NUXEO_SYSTEM_PROPERTIES.contains(key)) { 529 dumpedProperties.setProperty(key, properties.getRawProperty(key)); 530 } 531 } 532 return dumpedProperties; 533 } 534 535 /** 536 * @return Nuxeo's third party libraries directory 537 * @since 5.4.1 538 */ 539 public File getNuxeoLibDir() { 540 return new File(getRuntimeHome(), "lib"); 541 } 542 543 /** 544 * @return Server's third party libraries directory 545 * @since 5.4.1 546 */ 547 public File getServerLibDir() { 548 return new File(generator.getNuxeoHome(), "lib"); 549 } 550 551 /** 552 * @since 5.7 553 */ 554 public void verifyInstallation() throws ConfigurationException { 555 checkPaths(); 556 checkNetwork(); 557 } 558 559 /** 560 * Perform server specific checks, not already done by {@link ConfigurationGenerator#checkAddressesAndPorts()} 561 * 562 * @since 5.7 563 * @see ConfigurationGenerator#checkAddressesAndPorts() 564 */ 565 protected void checkNetwork() throws ConfigurationException { 566 InetAddress bindAddress = generator.getBindAddress(); 567 ConfigurationGenerator.checkPortAvailable(bindAddress, 568 Integer.parseInt(generator.getUserConfig().getProperty(PARAM_HTTP_TOMCAT_ADMIN_PORT))); 569 } 570 571 /** 572 * @return Marketplace Packages directory 573 * @since 5.9.4 574 */ 575 public File getPackagesDir() { 576 if (packagesDir == null) { 577 packagesDir = new File(generator.getNuxeoHome(), getDefaultPackagesDir()); 578 } 579 return packagesDir; 580 } 581 582 /** 583 * @return Default MP directory path relative to Nuxeo Home 584 * @since 5.9.4 585 */ 586 public String getDefaultPackagesDir() { 587 return Environment.DEFAULT_MP_DIR; 588 } 589 590 /** 591 * Introspect the server and builds the instance info 592 * 593 * @since 8.3 594 */ 595 public InstanceInfo getInfo(String clid, List<LocalPackage> pkgs) { 596 InstanceInfo nxInstance = new InstanceInfo(); 597 nxInstance.NUXEO_CONF = generator.getNuxeoConf().getPath(); 598 nxInstance.NUXEO_HOME = generator.getNuxeoHome().getPath(); 599 // distribution 600 File distFile = new File(generator.getConfigDir(), "distribution.properties"); 601 if (!distFile.exists()) { 602 // fallback in the file in templates 603 distFile = new File(generator.getNuxeoHome(), "templates"); 604 distFile = new File(distFile, "common"); 605 distFile = new File(distFile, "config"); 606 distFile = new File(distFile, "distribution.properties"); 607 } 608 try { 609 nxInstance.distribution = new DistributionInfo(distFile); 610 } catch (IOException e) { 611 nxInstance.distribution = new DistributionInfo(); 612 } 613 // packages 614 nxInstance.clid = clid; 615 Set<String> pkgTemplates = new HashSet<>(); 616 for (LocalPackage pkg : pkgs) { 617 final PackageInfo info = new PackageInfo(pkg); 618 nxInstance.packages.add(info); 619 pkgTemplates.addAll(info.templates); 620 } 621 nxInstance.config = new ConfigurationInfo(); 622 // profiles 623 String profiles = generator.getEnvironment(NUXEO_PROFILES); 624 if (isNotBlank(profiles)) { 625 nxInstance.config.profiles.addAll(Arrays.asList(profiles.split(TEMPLATE_SEPARATOR))); 626 } 627 // templates 628 nxInstance.config.dbtemplate = generator.extractDatabaseTemplateName(); 629 String userTemplates = generator.getUserTemplates(); 630 StringTokenizer st = new StringTokenizer(userTemplates, ","); 631 while (st.hasMoreTokens()) { 632 String template = st.nextToken(); 633 if (template.equals(nxInstance.config.dbtemplate)) { 634 continue; 635 } 636 if (pkgTemplates.contains(template)) { 637 nxInstance.config.pkgtemplates.add(template); 638 } else { 639 File testBase = new File(generator.getNuxeoHome(), 640 ConfigurationGenerator.TEMPLATES + File.separator + template); 641 if (testBase.exists()) { 642 nxInstance.config.basetemplates.add(template); 643 } else { 644 nxInstance.config.usertemplates.add(template); 645 } 646 } 647 } 648 CryptoProperties userConfig = generator.getUserConfig(); 649 // Settings from nuxeo.conf 650 computeKeyVals(nxInstance.config.keyvals, userConfig, userConfig.keySet()); 651 // Effective configuration for environment and profiles 652 computeKeyVals(nxInstance.config.allkeyvals, userConfig, userConfig.stringPropertyNames()); 653 return nxInstance; 654 } 655 656 protected void computeKeyVals(List<KeyValueInfo> keyVals, CryptoProperties userConfig, Set<?> keys) { 657 for (Object item : new TreeSet<>(keys)) { 658 String key = (String) item; 659 String value = userConfig.getRawProperty(key); 660 if (JAVA_OPTS.equals(key)) { 661 value = generator.getJavaOptsString(); 662 } 663 if (ConfigurationGenerator.SECRET_KEYS.contains(key) || key.contains("password") 664 || key.equals(Environment.SERVER_STATUS_KEY) || Crypto.isEncrypted(value)) { 665 value = "********"; 666 } 667 keyVals.add(new KeyValueInfo(key, value)); 668 } 669 } 670}