001/* 002 * (C) Copyright 2013-2014 Nuxeo SA (http://nuxeo.com/) and others. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 * 016 * Contributors: 017 * Delbosc Benoit 018 */ 019package org.nuxeo.runtime.metrics; 020 021import static org.apache.logging.log4j.Level.INFO; 022import static org.apache.logging.log4j.LogManager.ROOT_LOGGER_NAME; 023 024import java.io.File; 025import java.io.Serializable; 026import java.net.InetAddress; 027import java.net.InetSocketAddress; 028import java.net.UnknownHostException; 029import java.text.DateFormat; 030import java.text.SimpleDateFormat; 031import java.util.ArrayList; 032import java.util.Arrays; 033import java.util.Calendar; 034import java.util.Date; 035import java.util.List; 036import java.util.concurrent.TimeUnit; 037 038import javax.management.MalformedObjectNameException; 039import javax.management.ObjectName; 040 041import org.apache.commons.logging.LogFactory; 042import org.apache.logging.log4j.LogManager; 043import org.apache.logging.log4j.core.LoggerContext; 044import org.apache.logging.log4j.core.config.Configuration; 045import org.nuxeo.common.Environment; 046import org.nuxeo.common.xmap.annotation.XNode; 047import org.nuxeo.common.xmap.annotation.XNodeList; 048import org.nuxeo.common.xmap.annotation.XObject; 049import org.nuxeo.runtime.api.Framework; 050import org.nuxeo.runtime.management.ServerLocator; 051 052import com.codahale.metrics.MetricRegistry; 053import com.codahale.metrics.graphite.Graphite; 054import com.codahale.metrics.graphite.GraphiteReporter; 055import com.codahale.metrics.jmx.JmxReporter; 056import com.codahale.metrics.jvm.BufferPoolMetricSet; 057import com.codahale.metrics.jvm.FileDescriptorRatioGauge; 058import com.codahale.metrics.jvm.GarbageCollectorMetricSet; 059import com.codahale.metrics.jvm.JmxAttributeGauge; 060import com.codahale.metrics.jvm.MemoryUsageGaugeSet; 061import com.codahale.metrics.jvm.ThreadStatesGaugeSet; 062import com.codahale.metrics.log4j2.InstrumentedAppender; 063import com.readytalk.metrics.StatsDReporter; 064 065@XObject("metrics") 066public class MetricsDescriptor implements Serializable { 067 068 private static final long serialVersionUID = 7833869486922092460L; 069 070 public MetricsDescriptor() { 071 super(); 072 graphiteReporter = new GraphiteDescriptor(); 073 csvReporter = new CsvDescriptor(); 074 tomcatInstrumentation = new TomcatInstrumentationDescriptor(); 075 log4jInstrumentation = new Log4jInstrumentationDescriptor(); 076 } 077 078 @XObject(value = "graphiteReporter") 079 public static class GraphiteDescriptor { 080 081 public static final String ENABLED_PROPERTY = "metrics.graphite.enabled"; 082 083 public static final String HOST_PROPERTY = "metrics.graphite.host"; 084 085 public static final String PORT_PROPERTY = "metrics.graphite.port"; 086 087 public static final String PERIOD_PROPERTY = "metrics.graphite.period"; 088 089 public static final String PREFIX_PROPERTY = "metrics.graphite.prefix"; 090 091 /** 092 * A list of metric prefixes that if defined should be kept reported, separated by commas 093 * 094 * @since 9.3 095 */ 096 public static final String ALLOWED_METRICS_PROPERTY = "metrics.graphite.allowedMetrics"; 097 098 /** 099 * A list of metric prefixes that if defined should not be reported, separated by commas 100 * 101 * @since 9.3 102 */ 103 public static final String DENIED_METRICS_PROPERTY = "metrics.graphite.deniedMetrics"; 104 105 /** 106 * @since 9.3 107 */ 108 public static final String DEFAULT_ALLOWED_METRICS = "nuxeo.cache.user-entry-cache,nuxeo.cache.group-entry-cache,nuxeo.directories.userDirectory,nuxeo.directories.groupDirectory"; 109 110 /** 111 * @since 9.3 112 */ 113 public static final String DEFAULT_DENIED_METRICS = "nuxeo.cache,nuxeo.directories"; 114 115 /** 116 * @since 9.3 117 */ 118 public static final String ALL_METRICS = "ALL"; 119 120 @XNode("@enabled") 121 protected Boolean enabled = Boolean.valueOf(Framework.getProperty(ENABLED_PROPERTY, "false")); 122 123 @XNode("@host") 124 public String host = Framework.getProperty(HOST_PROPERTY, "0.0.0.0"); 125 126 @XNode("@port") 127 public Integer port = Integer.valueOf(Framework.getProperty(PORT_PROPERTY, "2030")); 128 129 @XNode("@periodInSecond") 130 public Integer period = Integer.valueOf(Framework.getProperty(PERIOD_PROPERTY, "10")); 131 132 @XNode("@prefix") 133 public String prefix = getPrefix(); 134 135 /** 136 * A list of metric prefixes that if defined should be kept reported 137 * 138 * @since 9.3 139 */ 140 @XNodeList(value = "allowedMetrics/metric", type = ArrayList.class, componentType = String.class) 141 public List<String> allowedMetrics = Arrays.asList( 142 Framework.getProperty(ALLOWED_METRICS_PROPERTY, DEFAULT_ALLOWED_METRICS).split(",")); 143 144 /** 145 * A list of metric prefixes that if defined should not be reported 146 * 147 * @since 9.3 148 */ 149 @XNodeList(value = "deniedMetrics/metric", type = ArrayList.class, componentType = String.class) 150 public List<String> deniedMetrics = Arrays.asList( 151 Framework.getProperty(DENIED_METRICS_PROPERTY, DEFAULT_DENIED_METRICS).split(",")); 152 153 public String getPrefix() { 154 if (prefix == null) { 155 prefix = Framework.getProperty(PREFIX_PROPERTY, "servers.${hostname}.nuxeo"); 156 } 157 String hostname; 158 try { 159 hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0]; 160 } catch (UnknownHostException e) { 161 hostname = "unknown"; 162 } 163 return prefix.replace("${hostname}", hostname); 164 } 165 166 public boolean filter(String name) { 167 return allowedMetrics.stream().anyMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f)) 168 || deniedMetrics.stream().noneMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f)); 169 } 170 171 @Override 172 public String toString() { 173 return String.format("graphiteReporter %s prefix: %s, host: %s, port: %d, period: %d", 174 enabled ? "enabled" : "disabled", prefix, host, port, period); 175 } 176 177 protected GraphiteReporter reporter; 178 179 public void enable(MetricRegistry registry) { 180 if (!enabled) { 181 return; 182 } 183 184 InetSocketAddress address = new InetSocketAddress(host, port); 185 Graphite graphite = new Graphite(address); 186 reporter = GraphiteReporter.forRegistry(registry) 187 .convertRatesTo(TimeUnit.SECONDS) 188 .convertDurationsTo(TimeUnit.MICROSECONDS) 189 .prefixedWith(getPrefix()) 190 .filter((name, metric) -> filter(name)) 191 .build(graphite); 192 reporter.start(period, TimeUnit.SECONDS); 193 } 194 195 public void disable(MetricRegistry registry) { 196 if (reporter == null) { 197 return; 198 } 199 try { 200 reporter.stop(); 201 } finally { 202 reporter = null; 203 } 204 } 205 } 206 207 @XObject(value = "csvReporter") 208 public static class CsvDescriptor { 209 210 public static final String ENABLED_PROPERTY = "metrics.csv.enabled"; 211 212 public static final String PERIOD_PROPERTY = "metrics.csv.period"; 213 214 public static final String OUTPUT_PROPERTY = "metrics.csv.output"; 215 216 @XNode("@output") 217 public File outputDir = outputDir(); 218 219 @XNode("@periodInSecond") 220 public Integer period = 10; 221 222 @XNode("@enabled") 223 public boolean enabled = Framework.isBooleanPropertyTrue(ENABLED_PROPERTY); 224 225 public int getPeriod() { 226 if (period == null) { 227 period = Integer.valueOf(Framework.getProperty(PERIOD_PROPERTY, "10")); 228 } 229 return period; 230 } 231 232 protected File outputDir() { 233 String path = Framework.getProperty(OUTPUT_PROPERTY, Framework.getProperty(Environment.NUXEO_LOG_DIR)); 234 DateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); 235 Date today = Calendar.getInstance().getTime(); 236 outputDir = new File(path, "metrics-" + df.format(today)); 237 return outputDir; 238 } 239 240 @Override 241 public String toString() { 242 return String.format("csvReporter %s, outputDir: %s, period: %d", enabled ? "enabled" : "disabled", 243 outputDir().toString(), getPeriod()); 244 } 245 246 protected CsvReporter reporter; 247 248 public void enable(MetricRegistry registry) { 249 if (!enabled) { 250 return; 251 } 252 File parentDir = outputDir.getParentFile(); 253 if (parentDir.exists() && parentDir.isDirectory()) { 254 outputDir.mkdir(); 255 reporter = CsvReporter.forRegistry(registry).build(outputDir); 256 reporter.start(period, TimeUnit.SECONDS); 257 } else { 258 enabled = false; 259 LogFactory.getLog(MetricsServiceImpl.class).error("Invalid output directory, disabling: " + this); 260 } 261 } 262 263 public void disable(MetricRegistry registry) { 264 if (reporter == null) { 265 return; 266 } 267 try { 268 reporter.stop(); 269 } finally { 270 reporter = null; 271 } 272 } 273 274 } 275 276 /** 277 * @since 10.3 278 */ 279 @XObject(value = "statsDReporter") 280 public static class StatsDDescriptor { 281 282 public static final String ENABLED_PROPERTY = "metrics.statsd.enabled"; 283 284 public static final String HOST_PROPERTY = "metrics.statsd.host"; 285 286 public static final String PORT_PROPERTY = "metrics.statsd.port"; 287 288 public static final String PERIOD_PROPERTY = "metrics.statsd.period"; 289 290 public static final String PREFIX_PROPERTY = "metrics.statsd.prefix"; 291 292 /** 293 * A list of metric prefixes that if defined should be kept reported, separated by commas 294 */ 295 public static final String ALLOWED_METRICS_PROPERTY = "metrics.statsd.allowedMetrics"; 296 297 /** 298 * A list of metric prefixes that if defined should not be reported, separated by commas 299 */ 300 public static final String DENIED_METRICS_PROPERTY = "metrics.statsd.deniedMetrics"; 301 302 public static final String DEFAULT_ALLOWED_METRICS = GraphiteDescriptor.DEFAULT_ALLOWED_METRICS; 303 304 public static final String DEFAULT_DENIED_METRICS = GraphiteDescriptor.DEFAULT_DENIED_METRICS; 305 306 public static final String ALL_METRICS = "ALL"; 307 308 @XNode("@enabled") 309 protected Boolean enabled = Boolean.valueOf(Framework.getProperty(ENABLED_PROPERTY, "false")); 310 311 @XNode("@host") 312 public String host = Framework.getProperty(HOST_PROPERTY, "127.0.0.1"); 313 314 @XNode("@port") 315 public Integer port = Integer.valueOf(Framework.getProperty(PORT_PROPERTY, "8125")); 316 317 @XNode("@periodInSecond") 318 public Integer period = Integer.valueOf(Framework.getProperty(PERIOD_PROPERTY, "10")); 319 320 @XNode("@prefix") 321 public String prefix = getPrefix(); 322 323 /** 324 * A list of metric prefixes that if defined should be kept reported 325 */ 326 @XNodeList(value = "allowedMetrics/metric", type = ArrayList.class, componentType = String.class) 327 public List<String> allowedMetrics = Arrays.asList( 328 Framework.getProperty(ALLOWED_METRICS_PROPERTY, DEFAULT_ALLOWED_METRICS).split(",")); 329 330 /** 331 * A list of metric prefixes that if defined should not be reported 332 */ 333 @XNodeList(value = "deniedMetrics/metric", type = ArrayList.class, componentType = String.class) 334 public List<String> deniedMetrics = Arrays.asList( 335 Framework.getProperty(DENIED_METRICS_PROPERTY, DEFAULT_DENIED_METRICS).split(",")); 336 337 public String getPrefix() { 338 if (prefix == null) { 339 prefix = Framework.getProperty(PREFIX_PROPERTY, "servers.${hostname}.nuxeo"); 340 } 341 String hostname; 342 try { 343 hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0]; 344 } catch (UnknownHostException e) { 345 hostname = "unknown"; 346 } 347 return prefix.replace("${hostname}", hostname); 348 } 349 350 public boolean filter(String name) { 351 return allowedMetrics.stream().anyMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f)) 352 || deniedMetrics.stream().noneMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f)); 353 } 354 355 @Override 356 public String toString() { 357 return String.format("statdReporter %s prefix: %s, host: %s, port: %d, period: %d", 358 enabled ? "enabled" : "disabled", prefix, host, port, period); 359 } 360 361 protected StatsDReporter reporter; 362 363 public void enable(MetricRegistry registry) { 364 if (!enabled) { 365 return; 366 } 367 reporter = StatsDReporter.forRegistry(registry).build(host, port); 368 reporter.start(period, TimeUnit.SECONDS); 369 } 370 371 public void disable(MetricRegistry registry) { 372 if (reporter == null) { 373 return; 374 } 375 try { 376 reporter.stop(); 377 } finally { 378 reporter = null; 379 } 380 } 381 } 382 383 @XObject(value = "log4jInstrumentation") 384 public static class Log4jInstrumentationDescriptor { 385 386 public static final String ENABLED_PROPERTY = "metrics.log4j.enabled"; 387 388 @XNode("@enabled") 389 protected boolean enabled = Boolean.parseBoolean(Framework.getProperty(ENABLED_PROPERTY, "false")); 390 391 private InstrumentedAppender appender; 392 393 @Override 394 public String toString() { 395 return String.format("log4jInstrumentation %s", enabled ? "enabled" : "disabled"); 396 } 397 398 public void enable(MetricRegistry registry) { 399 if (!enabled) { 400 return; 401 } 402 LogFactory.getLog(MetricsServiceImpl.class).info(this); 403 404 InstrumentedAppender appender = new InstrumentedAppender(registry, null, null, false); 405 appender.start(); 406 407 LoggerContext context = (LoggerContext) LogManager.getContext(false); 408 Configuration config = context.getConfiguration(); 409 config.getLoggerConfig(ROOT_LOGGER_NAME).addAppender(appender, INFO, null); 410 context.updateLoggers(config); 411 412 } 413 414 public void disable(MetricRegistry registry) { 415 if (appender == null) { 416 return; 417 } 418 try { 419 LoggerContext context = (LoggerContext) LogManager.getContext(false); 420 Configuration config = context.getConfiguration(); 421 config.getLoggerConfig(ROOT_LOGGER_NAME).removeAppender(appender.getName()); 422 context.updateLoggers(config); 423 } finally { 424 appender = null; 425 } 426 } 427 428 } 429 430 @XObject(value = "tomcatInstrumentation") 431 public static class TomcatInstrumentationDescriptor { 432 433 public static final String ENABLED_PROPERTY = "metrics.tomcat.enabled"; 434 435 @XNode("@enabled") 436 protected boolean enabled = Boolean.parseBoolean(Framework.getProperty(ENABLED_PROPERTY, "false")); 437 438 @Override 439 public String toString() { 440 return String.format("tomcatInstrumentation %s", enabled ? "enabled" : "disabled"); 441 } 442 443 protected void registerTomcatGauge(String mbean, String attribute, MetricRegistry registry, String name) { 444 try { 445 registry.register(MetricRegistry.name("tomcat", name), 446 new JmxAttributeGauge(new ObjectName(mbean), attribute)); 447 } catch (MalformedObjectNameException | IllegalArgumentException e) { 448 throw new UnsupportedOperationException("Cannot compute object name of " + mbean, e); 449 } 450 } 451 452 public void enable(MetricRegistry registry) { 453 if (!enabled) { 454 return; 455 } 456 LogFactory.getLog(MetricsServiceImpl.class).info(this); 457 // TODO: do not hard code the common datasource 458 // nameenable(registry) 459 String pool = "org.nuxeo.ecm.core.management.jtajca:type=ConnectionPoolMonitor,name=jdbc/nuxeo"; 460 String connector = String.format("Catalina:type=ThreadPool,name=\"http-nio-%s-%s\"", 461 Framework.getProperty("nuxeo.bind.address", "0.0.0.0"), 462 Framework.getProperty("nuxeo.bind.port", "8080")); 463 String requestProcessor = String.format("Catalina:type=GlobalRequestProcessor,name=\"http-nio-%s-%s\"", 464 Framework.getProperty("nuxeo.bind.address", "0.0.0.0"), 465 Framework.getProperty("nuxeo.bind.port", "8080")); 466 String manager = "Catalina:type=Manager,host=localhost,context=/nuxeo"; 467 registerTomcatGauge(pool, "ConnectionCount", registry, "jdbc-numActive"); 468 registerTomcatGauge(pool, "IdleConnectionCount", registry, "jdbc-numIdle"); 469 registerTomcatGauge(connector, "currentThreadCount", registry, "currentThreadCount"); 470 registerTomcatGauge(connector, "currentThreadsBusy", registry, "currentThreadBusy"); 471 registerTomcatGauge(requestProcessor, "errorCount", registry, "errorCount"); 472 registerTomcatGauge(requestProcessor, "requestCount", registry, "requestCount"); 473 registerTomcatGauge(requestProcessor, "processingTime", registry, "processingTime"); 474 registerTomcatGauge(requestProcessor, "bytesReceived", registry, "bytesReceived"); 475 registerTomcatGauge(requestProcessor, "bytesSent", registry, "bytesSent"); 476 registerTomcatGauge(manager, "activeSessions", registry, "activeSessions"); 477 } 478 479 public void disable(MetricRegistry registry) { 480 registry.remove("tomcat.jdbc-numActive"); 481 registry.remove("tomcat.jdbc-numIdle"); 482 registry.remove("tomcat.currentThreadCount"); 483 registry.remove("tomcat.currentThreadBusy"); 484 registry.remove("tomcat.errorCount"); 485 registry.remove("tomcat.requestCount"); 486 registry.remove("tomcat.processingTime"); 487 registry.remove("tomcat.activeSessions"); 488 } 489 } 490 491 @XObject(value = "jvmInstrumentation") 492 public static class JvmInstrumentationDescriptor { 493 494 public static final String ENABLED_PROPERTY = "metrics.jvm.enabled"; 495 496 @XNode("@enabled") 497 protected boolean enabled = Boolean.parseBoolean(Framework.getProperty(ENABLED_PROPERTY, "true")); 498 499 public void enable(MetricRegistry registry) { 500 if (!enabled) { 501 return; 502 } 503 registry.register("jvm.memory", new MemoryUsageGaugeSet()); 504 registry.register("jvm.garbage", new GarbageCollectorMetricSet()); 505 registry.register("jvm.threads", new ThreadStatesGaugeSet()); 506 registry.register("jvm.files", new FileDescriptorRatioGauge()); 507 registry.register("jvm.buffers", 508 new BufferPoolMetricSet(Framework.getService(ServerLocator.class).lookupServer())); 509 } 510 511 public void disable(MetricRegistry registry) { 512 if (!enabled) { 513 return; 514 } 515 registry.removeMatching((name, metric) -> name.startsWith("jvm.")); 516 517 } 518 } 519 520 @XNode("graphiteReporter") 521 public GraphiteDescriptor graphiteReporter = new GraphiteDescriptor(); 522 523 @XNode("csvReporter") 524 public CsvDescriptor csvReporter = new CsvDescriptor(); 525 526 @XNode("statsDReporter") 527 public StatsDDescriptor statsDReporter = new StatsDDescriptor(); 528 529 @XNode("log4jInstrumentation") 530 public Log4jInstrumentationDescriptor log4jInstrumentation = new Log4jInstrumentationDescriptor(); 531 532 @XNode("tomcatInstrumentation") 533 public TomcatInstrumentationDescriptor tomcatInstrumentation = new TomcatInstrumentationDescriptor(); 534 535 @XNode(value = "jvmInstrumentation") 536 public JvmInstrumentationDescriptor jvmInstrumentation = new JvmInstrumentationDescriptor(); 537 538 protected JmxReporter jmxReporter; 539 540 public void enable(MetricRegistry registry) { 541 jmxReporter = JmxReporter.forRegistry(registry).build(); 542 jmxReporter.start(); 543 graphiteReporter.enable(registry); 544 csvReporter.enable(registry); 545 log4jInstrumentation.enable(registry); 546 tomcatInstrumentation.enable(registry); 547 jvmInstrumentation.enable(registry); 548 statsDReporter.enable(registry); 549 } 550 551 public void disable(MetricRegistry registry) { 552 try { 553 graphiteReporter.disable(registry); 554 csvReporter.disable(registry); 555 log4jInstrumentation.disable(registry); 556 tomcatInstrumentation.disable(registry); 557 jvmInstrumentation.disable(registry); 558 statsDReporter.enable(registry); 559 jmxReporter.stop(); 560 } finally { 561 jmxReporter = null; 562 } 563 } 564 565}