001/*
002 * (C) Copyright 2013-2014 Nuxeo SA (http://nuxeo.com/) and others.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 *
016 * Contributors:
017 *     Delbosc Benoit
018 */
019package org.nuxeo.runtime.metrics;
020
021import static org.apache.logging.log4j.Level.INFO;
022import static org.apache.logging.log4j.LogManager.ROOT_LOGGER_NAME;
023
024import java.io.File;
025import java.io.Serializable;
026import java.net.InetAddress;
027import java.net.InetSocketAddress;
028import java.net.UnknownHostException;
029import java.text.DateFormat;
030import java.text.SimpleDateFormat;
031import java.util.ArrayList;
032import java.util.Arrays;
033import java.util.Calendar;
034import java.util.Date;
035import java.util.List;
036import java.util.concurrent.TimeUnit;
037
038import javax.management.MalformedObjectNameException;
039import javax.management.ObjectName;
040
041import org.apache.commons.logging.LogFactory;
042import org.apache.logging.log4j.LogManager;
043import org.apache.logging.log4j.core.LoggerContext;
044import org.apache.logging.log4j.core.config.Configuration;
045import org.nuxeo.common.Environment;
046import org.nuxeo.common.xmap.annotation.XNode;
047import org.nuxeo.common.xmap.annotation.XNodeList;
048import org.nuxeo.common.xmap.annotation.XObject;
049import org.nuxeo.runtime.api.Framework;
050import org.nuxeo.runtime.management.ServerLocator;
051
052import com.codahale.metrics.MetricRegistry;
053import com.codahale.metrics.graphite.Graphite;
054import com.codahale.metrics.graphite.GraphiteReporter;
055import com.codahale.metrics.jmx.JmxReporter;
056import com.codahale.metrics.jvm.BufferPoolMetricSet;
057import com.codahale.metrics.jvm.FileDescriptorRatioGauge;
058import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
059import com.codahale.metrics.jvm.JmxAttributeGauge;
060import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
061import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
062import com.codahale.metrics.log4j2.InstrumentedAppender;
063import com.readytalk.metrics.StatsDReporter;
064
065@XObject("metrics")
066public class MetricsDescriptor implements Serializable {
067
068    private static final long serialVersionUID = 7833869486922092460L;
069
070    public MetricsDescriptor() {
071        super();
072        graphiteReporter = new GraphiteDescriptor();
073        csvReporter = new CsvDescriptor();
074        tomcatInstrumentation = new TomcatInstrumentationDescriptor();
075        log4jInstrumentation = new Log4jInstrumentationDescriptor();
076    }
077
078    @XObject(value = "graphiteReporter")
079    public static class GraphiteDescriptor {
080
081        public static final String ENABLED_PROPERTY = "metrics.graphite.enabled";
082
083        public static final String HOST_PROPERTY = "metrics.graphite.host";
084
085        public static final String PORT_PROPERTY = "metrics.graphite.port";
086
087        public static final String PERIOD_PROPERTY = "metrics.graphite.period";
088
089        public static final String PREFIX_PROPERTY = "metrics.graphite.prefix";
090
091        /**
092         * A list of metric prefixes that if defined should be kept reported, separated by commas
093         *
094         * @since 9.3
095         */
096        public static final String ALLOWED_METRICS_PROPERTY = "metrics.graphite.allowedMetrics";
097
098        /**
099         * A list of metric prefixes that if defined should not be reported, separated by commas
100         *
101         * @since 9.3
102         */
103        public static final String DENIED_METRICS_PROPERTY = "metrics.graphite.deniedMetrics";
104
105        /**
106         * @since 9.3
107         */
108        public static final String DEFAULT_ALLOWED_METRICS = "nuxeo.cache.user-entry-cache,nuxeo.cache.group-entry-cache,nuxeo.directories.userDirectory,nuxeo.directories.groupDirectory";
109
110        /**
111         * @since 9.3
112         */
113        public static final String DEFAULT_DENIED_METRICS = "nuxeo.cache,nuxeo.directories";
114
115        /**
116         * @since 9.3
117         */
118        public static final String ALL_METRICS = "ALL";
119
120        @XNode("@enabled")
121        protected Boolean enabled = Boolean.valueOf(Framework.getProperty(ENABLED_PROPERTY, "false"));
122
123        @XNode("@host")
124        public String host = Framework.getProperty(HOST_PROPERTY, "0.0.0.0");
125
126        @XNode("@port")
127        public Integer port = Integer.valueOf(Framework.getProperty(PORT_PROPERTY, "2030"));
128
129        @XNode("@periodInSecond")
130        public Integer period = Integer.valueOf(Framework.getProperty(PERIOD_PROPERTY, "10"));
131
132        @XNode("@prefix")
133        public String prefix = getPrefix();
134
135        /**
136         * A list of metric prefixes that if defined should be kept reported
137         *
138         * @since 9.3
139         */
140        @XNodeList(value = "allowedMetrics/metric", type = ArrayList.class, componentType = String.class)
141        public List<String> allowedMetrics = Arrays.asList(
142                Framework.getProperty(ALLOWED_METRICS_PROPERTY, DEFAULT_ALLOWED_METRICS).split(","));
143
144        /**
145         * A list of metric prefixes that if defined should not be reported
146         *
147         * @since 9.3
148         */
149        @XNodeList(value = "deniedMetrics/metric", type = ArrayList.class, componentType = String.class)
150        public List<String> deniedMetrics = Arrays.asList(
151                Framework.getProperty(DENIED_METRICS_PROPERTY, DEFAULT_DENIED_METRICS).split(","));
152
153        public String getPrefix() {
154            if (prefix == null) {
155                prefix = Framework.getProperty(PREFIX_PROPERTY, "servers.${hostname}.nuxeo");
156            }
157            String hostname;
158            try {
159                hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
160            } catch (UnknownHostException e) {
161                hostname = "unknown";
162            }
163            return prefix.replace("${hostname}", hostname);
164        }
165
166        public boolean filter(String name) {
167            return allowedMetrics.stream().anyMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f))
168                    || deniedMetrics.stream().noneMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f));
169        }
170
171        @Override
172        public String toString() {
173            return String.format("graphiteReporter %s prefix: %s, host: %s, port: %d, period: %d",
174                    enabled ? "enabled" : "disabled", prefix, host, port, period);
175        }
176
177        protected GraphiteReporter reporter;
178
179        public void enable(MetricRegistry registry) {
180            if (!enabled) {
181                return;
182            }
183
184            InetSocketAddress address = new InetSocketAddress(host, port);
185            Graphite graphite = new Graphite(address);
186            reporter = GraphiteReporter.forRegistry(registry)
187                                       .convertRatesTo(TimeUnit.SECONDS)
188                                       .convertDurationsTo(TimeUnit.MICROSECONDS)
189                                       .prefixedWith(getPrefix())
190                                       .filter((name, metric) -> filter(name))
191                                       .build(graphite);
192            reporter.start(period, TimeUnit.SECONDS);
193        }
194
195        public void disable(MetricRegistry registry) {
196            if (reporter == null) {
197                return;
198            }
199            try {
200                reporter.stop();
201            } finally {
202                reporter = null;
203            }
204        }
205    }
206
207    @XObject(value = "csvReporter")
208    public static class CsvDescriptor {
209
210        public static final String ENABLED_PROPERTY = "metrics.csv.enabled";
211
212        public static final String PERIOD_PROPERTY = "metrics.csv.period";
213
214        public static final String OUTPUT_PROPERTY = "metrics.csv.output";
215
216        @XNode("@output")
217        public File outputDir = outputDir();
218
219        @XNode("@periodInSecond")
220        public Integer period = 10;
221
222        @XNode("@enabled")
223        public boolean enabled = Framework.isBooleanPropertyTrue(ENABLED_PROPERTY);
224
225        public int getPeriod() {
226            if (period == null) {
227                period = Integer.valueOf(Framework.getProperty(PERIOD_PROPERTY, "10"));
228            }
229            return period;
230        }
231
232        protected File outputDir() {
233            String path = Framework.getProperty(OUTPUT_PROPERTY, Framework.getProperty(Environment.NUXEO_LOG_DIR));
234            DateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");
235            Date today = Calendar.getInstance().getTime();
236            outputDir = new File(path, "metrics-" + df.format(today));
237            return outputDir;
238        }
239
240        @Override
241        public String toString() {
242            return String.format("csvReporter %s, outputDir: %s, period: %d", enabled ? "enabled" : "disabled",
243                    outputDir().toString(), getPeriod());
244        }
245
246        protected CsvReporter reporter;
247
248        public void enable(MetricRegistry registry) {
249            if (!enabled) {
250                return;
251            }
252            File parentDir = outputDir.getParentFile();
253            if (parentDir.exists() && parentDir.isDirectory()) {
254                outputDir.mkdir();
255                reporter = CsvReporter.forRegistry(registry).build(outputDir);
256                reporter.start(period, TimeUnit.SECONDS);
257            } else {
258                enabled = false;
259                LogFactory.getLog(MetricsServiceImpl.class).error("Invalid output directory, disabling: " + this);
260            }
261        }
262
263        public void disable(MetricRegistry registry) {
264            if (reporter == null) {
265                return;
266            }
267            try {
268                reporter.stop();
269            } finally {
270                reporter = null;
271            }
272        }
273
274    }
275
276    /**
277     * @since 10.3
278     */
279    @XObject(value = "statsDReporter")
280    public static class StatsDDescriptor {
281
282        public static final String ENABLED_PROPERTY = "metrics.statsd.enabled";
283
284        public static final String HOST_PROPERTY = "metrics.statsd.host";
285
286        public static final String PORT_PROPERTY = "metrics.statsd.port";
287
288        public static final String PERIOD_PROPERTY = "metrics.statsd.period";
289
290        public static final String PREFIX_PROPERTY = "metrics.statsd.prefix";
291
292        /**
293         * A list of metric prefixes that if defined should be kept reported, separated by commas
294         */
295        public static final String ALLOWED_METRICS_PROPERTY = "metrics.statsd.allowedMetrics";
296
297        /**
298         * A list of metric prefixes that if defined should not be reported, separated by commas
299         */
300        public static final String DENIED_METRICS_PROPERTY = "metrics.statsd.deniedMetrics";
301
302        public static final String DEFAULT_ALLOWED_METRICS = GraphiteDescriptor.DEFAULT_ALLOWED_METRICS;
303
304        public static final String DEFAULT_DENIED_METRICS = GraphiteDescriptor.DEFAULT_DENIED_METRICS;
305
306        public static final String ALL_METRICS = "ALL";
307
308        @XNode("@enabled")
309        protected Boolean enabled = Boolean.valueOf(Framework.getProperty(ENABLED_PROPERTY, "false"));
310
311        @XNode("@host")
312        public String host = Framework.getProperty(HOST_PROPERTY, "127.0.0.1");
313
314        @XNode("@port")
315        public Integer port = Integer.valueOf(Framework.getProperty(PORT_PROPERTY, "8125"));
316
317        @XNode("@periodInSecond")
318        public Integer period = Integer.valueOf(Framework.getProperty(PERIOD_PROPERTY, "10"));
319
320        @XNode("@prefix")
321        public String prefix = getPrefix();
322
323        /**
324         * A list of metric prefixes that if defined should be kept reported
325         */
326        @XNodeList(value = "allowedMetrics/metric", type = ArrayList.class, componentType = String.class)
327        public List<String> allowedMetrics = Arrays.asList(
328                Framework.getProperty(ALLOWED_METRICS_PROPERTY, DEFAULT_ALLOWED_METRICS).split(","));
329
330        /**
331         * A list of metric prefixes that if defined should not be reported
332         */
333        @XNodeList(value = "deniedMetrics/metric", type = ArrayList.class, componentType = String.class)
334        public List<String> deniedMetrics = Arrays.asList(
335                Framework.getProperty(DENIED_METRICS_PROPERTY, DEFAULT_DENIED_METRICS).split(","));
336
337        public String getPrefix() {
338            if (prefix == null) {
339                prefix = Framework.getProperty(PREFIX_PROPERTY, "servers.${hostname}.nuxeo");
340            }
341            String hostname;
342            try {
343                hostname = InetAddress.getLocalHost().getHostName().split("\\.")[0];
344            } catch (UnknownHostException e) {
345                hostname = "unknown";
346            }
347            return prefix.replace("${hostname}", hostname);
348        }
349
350        public boolean filter(String name) {
351            return allowedMetrics.stream().anyMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f))
352                    || deniedMetrics.stream().noneMatch(f -> ALL_METRICS.equals(f) || name.startsWith(f));
353        }
354
355        @Override
356        public String toString() {
357            return String.format("statdReporter %s prefix: %s, host: %s, port: %d, period: %d",
358                    enabled ? "enabled" : "disabled", prefix, host, port, period);
359        }
360
361        protected StatsDReporter reporter;
362
363        public void enable(MetricRegistry registry) {
364            if (!enabled) {
365                return;
366            }
367            reporter = StatsDReporter.forRegistry(registry).build(host, port);
368            reporter.start(period, TimeUnit.SECONDS);
369        }
370
371        public void disable(MetricRegistry registry) {
372            if (reporter == null) {
373                return;
374            }
375            try {
376                reporter.stop();
377            } finally {
378                reporter = null;
379            }
380        }
381    }
382
383    @XObject(value = "log4jInstrumentation")
384    public static class Log4jInstrumentationDescriptor {
385
386        public static final String ENABLED_PROPERTY = "metrics.log4j.enabled";
387
388        @XNode("@enabled")
389        protected boolean enabled = Boolean.parseBoolean(Framework.getProperty(ENABLED_PROPERTY, "false"));
390
391        private InstrumentedAppender appender;
392
393        @Override
394        public String toString() {
395            return String.format("log4jInstrumentation %s", enabled ? "enabled" : "disabled");
396        }
397
398        public void enable(MetricRegistry registry) {
399            if (!enabled) {
400                return;
401            }
402            LogFactory.getLog(MetricsServiceImpl.class).info(this);
403
404            InstrumentedAppender appender = new InstrumentedAppender(registry, null, null, false);
405            appender.start();
406
407            LoggerContext context = (LoggerContext) LogManager.getContext(false);
408            Configuration config = context.getConfiguration();
409            config.getLoggerConfig(ROOT_LOGGER_NAME).addAppender(appender, INFO, null);
410            context.updateLoggers(config);
411
412        }
413
414        public void disable(MetricRegistry registry) {
415            if (appender == null) {
416                return;
417            }
418            try {
419                LoggerContext context = (LoggerContext) LogManager.getContext(false);
420                Configuration config = context.getConfiguration();
421                config.getLoggerConfig(ROOT_LOGGER_NAME).removeAppender(appender.getName());
422                context.updateLoggers(config);
423            } finally {
424                appender = null;
425            }
426        }
427
428    }
429
430    @XObject(value = "tomcatInstrumentation")
431    public static class TomcatInstrumentationDescriptor {
432
433        public static final String ENABLED_PROPERTY = "metrics.tomcat.enabled";
434
435        @XNode("@enabled")
436        protected boolean enabled = Boolean.parseBoolean(Framework.getProperty(ENABLED_PROPERTY, "false"));
437
438        @Override
439        public String toString() {
440            return String.format("tomcatInstrumentation %s", enabled ? "enabled" : "disabled");
441        }
442
443        protected void registerTomcatGauge(String mbean, String attribute, MetricRegistry registry, String name) {
444            try {
445                registry.register(MetricRegistry.name("tomcat", name),
446                        new JmxAttributeGauge(new ObjectName(mbean), attribute));
447            } catch (MalformedObjectNameException | IllegalArgumentException e) {
448                throw new UnsupportedOperationException("Cannot compute object name of " + mbean, e);
449            }
450        }
451
452        public void enable(MetricRegistry registry) {
453            if (!enabled) {
454                return;
455            }
456            LogFactory.getLog(MetricsServiceImpl.class).info(this);
457            // TODO: do not hard code the common datasource
458            // nameenable(registry)
459            String pool = "org.nuxeo.ecm.core.management.jtajca:type=ConnectionPoolMonitor,name=jdbc/nuxeo";
460            String connector = String.format("Catalina:type=ThreadPool,name=\"http-nio-%s-%s\"",
461                    Framework.getProperty("nuxeo.bind.address", "0.0.0.0"),
462                    Framework.getProperty("nuxeo.bind.port", "8080"));
463            String requestProcessor = String.format("Catalina:type=GlobalRequestProcessor,name=\"http-nio-%s-%s\"",
464                    Framework.getProperty("nuxeo.bind.address", "0.0.0.0"),
465                    Framework.getProperty("nuxeo.bind.port", "8080"));
466            String manager = "Catalina:type=Manager,host=localhost,context=/nuxeo";
467            registerTomcatGauge(pool, "ConnectionCount", registry, "jdbc-numActive");
468            registerTomcatGauge(pool, "IdleConnectionCount", registry, "jdbc-numIdle");
469            registerTomcatGauge(connector, "currentThreadCount", registry, "currentThreadCount");
470            registerTomcatGauge(connector, "currentThreadsBusy", registry, "currentThreadBusy");
471            registerTomcatGauge(requestProcessor, "errorCount", registry, "errorCount");
472            registerTomcatGauge(requestProcessor, "requestCount", registry, "requestCount");
473            registerTomcatGauge(requestProcessor, "processingTime", registry, "processingTime");
474            registerTomcatGauge(requestProcessor, "bytesReceived", registry, "bytesReceived");
475            registerTomcatGauge(requestProcessor, "bytesSent", registry, "bytesSent");
476            registerTomcatGauge(manager, "activeSessions", registry, "activeSessions");
477        }
478
479        public void disable(MetricRegistry registry) {
480            registry.remove("tomcat.jdbc-numActive");
481            registry.remove("tomcat.jdbc-numIdle");
482            registry.remove("tomcat.currentThreadCount");
483            registry.remove("tomcat.currentThreadBusy");
484            registry.remove("tomcat.errorCount");
485            registry.remove("tomcat.requestCount");
486            registry.remove("tomcat.processingTime");
487            registry.remove("tomcat.activeSessions");
488        }
489    }
490
491    @XObject(value = "jvmInstrumentation")
492    public static class JvmInstrumentationDescriptor {
493
494        public static final String ENABLED_PROPERTY = "metrics.jvm.enabled";
495
496        @XNode("@enabled")
497        protected boolean enabled = Boolean.parseBoolean(Framework.getProperty(ENABLED_PROPERTY, "true"));
498
499        public void enable(MetricRegistry registry) {
500            if (!enabled) {
501                return;
502            }
503            registry.register("jvm.memory", new MemoryUsageGaugeSet());
504            registry.register("jvm.garbage", new GarbageCollectorMetricSet());
505            registry.register("jvm.threads", new ThreadStatesGaugeSet());
506            registry.register("jvm.files", new FileDescriptorRatioGauge());
507            registry.register("jvm.buffers",
508                    new BufferPoolMetricSet(Framework.getService(ServerLocator.class).lookupServer()));
509        }
510
511        public void disable(MetricRegistry registry) {
512            if (!enabled) {
513                return;
514            }
515            registry.removeMatching((name, metric) -> name.startsWith("jvm."));
516
517        }
518    }
519
520    @XNode("graphiteReporter")
521    public GraphiteDescriptor graphiteReporter = new GraphiteDescriptor();
522
523    @XNode("csvReporter")
524    public CsvDescriptor csvReporter = new CsvDescriptor();
525
526    @XNode("statsDReporter")
527    public StatsDDescriptor statsDReporter = new StatsDDescriptor();
528
529    @XNode("log4jInstrumentation")
530    public Log4jInstrumentationDescriptor log4jInstrumentation = new Log4jInstrumentationDescriptor();
531
532    @XNode("tomcatInstrumentation")
533    public TomcatInstrumentationDescriptor tomcatInstrumentation = new TomcatInstrumentationDescriptor();
534
535    @XNode(value = "jvmInstrumentation")
536    public JvmInstrumentationDescriptor jvmInstrumentation = new JvmInstrumentationDescriptor();
537
538    protected JmxReporter jmxReporter;
539
540    public void enable(MetricRegistry registry) {
541        jmxReporter = JmxReporter.forRegistry(registry).build();
542        jmxReporter.start();
543        graphiteReporter.enable(registry);
544        csvReporter.enable(registry);
545        log4jInstrumentation.enable(registry);
546        tomcatInstrumentation.enable(registry);
547        jvmInstrumentation.enable(registry);
548        statsDReporter.enable(registry);
549    }
550
551    public void disable(MetricRegistry registry) {
552        try {
553            graphiteReporter.disable(registry);
554            csvReporter.disable(registry);
555            log4jInstrumentation.disable(registry);
556            tomcatInstrumentation.disable(registry);
557            jvmInstrumentation.disable(registry);
558            statsDReporter.enable(registry);
559            jmxReporter.stop();
560        } finally {
561            jmxReporter = null;
562        }
563    }
564
565}