This is an automated email from the ASF dual-hosted git repository.
sebawagner pushed a commit to branch
feature/OPENMEETINGS-2567-investigate-performance-monitoring
in repository https://gitbox.apache.org/repos/asf/openmeetings.git
The following commit(s) were added to
refs/heads/feature/OPENMEETINGS-2567-investigate-performance-monitoring by this
push:
new e2e2cd5 OPENMEETINGS-2567 Enable basic Prometheus JVM statistics.
e2e2cd5 is described below
commit e2e2cd56a103498602f072a650b15f12ebb81688
Author: Sebastian Wagner <[email protected]>
AuthorDate: Wed Feb 3 16:16:02 2021 +1300
OPENMEETINGS-2567 Enable basic Prometheus JVM statistics.
---
openmeetings-web/pom.xml | 12 +
.../apache/openmeetings/web/app/Application.java | 2 +
.../web/util/logging/TomcatGenericExports.java | 287 +++++++++++++++++++++
openmeetings-web/src/main/webapp/WEB-INF/web.xml | 33 +++
pom.xml | 16 ++
5 files changed, 350 insertions(+)
diff --git a/openmeetings-web/pom.xml b/openmeetings-web/pom.xml
index 8fe2685..81a582f 100644
--- a/openmeetings-web/pom.xml
+++ b/openmeetings-web/pom.xml
@@ -599,6 +599,18 @@
<artifactId>spring-orm</artifactId>
</dependency>
<dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_servlet</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-catalina</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.wicketstuff</groupId>
<artifactId>wicketstuff-dashboard-core</artifactId>
<version>${wickets.version}</version>
diff --git
a/openmeetings-web/src/main/java/org/apache/openmeetings/web/app/Application.java
b/openmeetings-web/src/main/java/org/apache/openmeetings/web/app/Application.java
index 5debb49..7466f47 100644
---
a/openmeetings-web/src/main/java/org/apache/openmeetings/web/app/Application.java
+++
b/openmeetings-web/src/main/java/org/apache/openmeetings/web/app/Application.java
@@ -94,6 +94,7 @@ import
org.apache.openmeetings.web.user.record.PngRecordingResourceReference;
import org.apache.openmeetings.web.util.GroupLogoResourceReference;
import org.apache.openmeetings.web.util.ProfileImageResourceReference;
import org.apache.openmeetings.web.util.UserDashboardPersister;
+import org.apache.openmeetings.web.util.logging.TomcatGenericExports;
import org.apache.wicket.DefaultPageManagerProvider;
import org.apache.wicket.Localizer;
import org.apache.wicket.Page;
@@ -192,6 +193,7 @@ public class Application extends
AuthenticatedWebApplication implements IApplica
@Override
protected void init() {
+ new TomcatGenericExports(false).register();
setWicketApplicationName(super.getName());
getSecuritySettings().setAuthenticationStrategy(new
OmAuthenticationStrategy());
getApplicationSettings().setAccessDeniedPage(AccessDeniedPage.class);
diff --git
a/openmeetings-web/src/main/java/org/apache/openmeetings/web/util/logging/TomcatGenericExports.java
b/openmeetings-web/src/main/java/org/apache/openmeetings/web/util/logging/TomcatGenericExports.java
new file mode 100644
index 0000000..e910dc4
--- /dev/null
+++
b/openmeetings-web/src/main/java/org/apache/openmeetings/web/util/logging/TomcatGenericExports.java
@@ -0,0 +1,287 @@
+package org.apache.openmeetings.web.util.logging;
+
+import io.prometheus.client.Collector;
+import io.prometheus.client.CounterMetricFamily;
+import io.prometheus.client.GaugeMetricFamily;
+import org.apache.catalina.util.ServerInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.management.*;
+import java.lang.management.ManagementFactory;
+import java.util.*;
+
+/**
+ * Exports Tomcat metrics applicable to most most applications:
+ *
+ * - http session metrics - request processor metrics - thread pool metrics
+ *
+ * <p>
+ * Example usage:
+ *
+ * <pre>
+ * {@code
+ * new TomcatGenericExports(false).register();
+ * }
+ * </pre>
+ *
+ * Example metrics being exported:
+ *
+ * <pre>
+ * tomcat_info{version="7.0.61.0",build="Apr 29 2015 14:58:03 UTC",} 1.0
+ * tomcat_session_active_total{context="/foo",host="default",} 877.0
+ * tomcat_session_rejected_total{context="/foo",host="default",} 0.0
+ * tomcat_session_created_total{context="/foo",host="default",} 24428.0
+ * tomcat_session_expired_total{context="/foo",host="default",} 23832.0
+ * tomcat_session_alivetime_seconds_avg{context="/foo",host="default",} 633.0
+ * tomcat_session_alivetime_seconds_max{context="/foo",host="default",} 9883.0
+ * tomcat_requestprocessor_received_bytes{name="http-bio-0.0.0.0-8080",} 0.0
+ * tomcat_requestprocessor_sent_bytes{name="http-bio-0.0.0.0-8080",} 5056098.0
+ * tomcat_requestprocessor_time_seconds{name="http-bio-0.0.0.0-8080",} 127386.0
+ * tomcat_requestprocessor_error_count{name="http-bio-0.0.0.0-8080",} 0.0
+ * tomcat_requestprocessor_request_count{name="http-bio-0.0.0.0-8080",} 33709.0
+ * tomcat_threads_total{pool="http-bio-0.0.0.0-8080",} 10.0
+ * tomcat_threads_active_total{pool="http-bio-0.0.0.0-8080",} 2.0
+ * tomcat_threads_active_max{pool="http-bio-0.0.0.0-8080",} 200.0
+ * </pre>
+ */
+
+public class TomcatGenericExports extends Collector {
+
+ private static final Logger log =
LoggerFactory.getLogger(TomcatGenericExports.class);
+ private String jmxDomain = "Catalina";
+
+ public TomcatGenericExports(boolean embedded) {
+ if (embedded) {
+ jmxDomain = "Tomcat";
+ }
+ }
+
+ private void addRequestProcessorMetrics(List<MetricFamilySamples> mfs) {
+ try {
+ final MBeanServer server =
ManagementFactory.getPlatformMBeanServer();
+ ObjectName filterName = new ObjectName(jmxDomain +
":type=GlobalRequestProcessor,name=*");
+ Set<ObjectInstance> mBeans =
server.queryMBeans(filterName, null);
+
+ if (mBeans.size() > 0) {
+ List<String> labelNameList =
Collections.singletonList("name");
+
+ GaugeMetricFamily
requestProcessorBytesReceivedGauge = new GaugeMetricFamily(
+
"tomcat_requestprocessor_received_bytes", "Number of bytes received by this
request processor",
+ labelNameList);
+
+ GaugeMetricFamily
requestProcessorBytesSentGauge = new GaugeMetricFamily(
+
"tomcat_requestprocessor_sent_bytes", "Number of bytes sent by this request
processor",
+ labelNameList);
+
+ GaugeMetricFamily
requestProcessorProcessingTimeGauge = new GaugeMetricFamily(
+
"tomcat_requestprocessor_time_seconds", "The total time spend by this request
processor",
+ labelNameList);
+
+ CounterMetricFamily
requestProcessorErrorCounter = new CounterMetricFamily(
+
"tomcat_requestprocessor_error_count",
+ "The number of error request
served by this request processor", labelNameList);
+
+ CounterMetricFamily
requestProcessorRequestCounter = new CounterMetricFamily(
+
"tomcat_requestprocessor_request_count",
+ "The number of request served
by this request processor", labelNameList);
+
+ for (final ObjectInstance mBean : mBeans) {
+ List<String> labelValueList =
Collections
+
.singletonList(mBean.getObjectName().getKeyProperty("name").replaceAll("[\"\\\\]",
""));
+
+
requestProcessorBytesReceivedGauge.addMetric(labelValueList,
+ ((Long)
server.getAttribute(mBean.getObjectName(), "bytesReceived")).doubleValue());
+
+
requestProcessorBytesSentGauge.addMetric(labelValueList,
+ ((Long)
server.getAttribute(mBean.getObjectName(), "bytesSent")).doubleValue());
+
+
requestProcessorProcessingTimeGauge.addMetric(labelValueList,
+ ((Long)
server.getAttribute(mBean.getObjectName(), "processingTime")).doubleValue()
+ /
1000.0);
+
+
requestProcessorErrorCounter.addMetric(labelValueList,
+ ((Integer)
server.getAttribute(mBean.getObjectName(), "errorCount")).doubleValue());
+
+
requestProcessorRequestCounter.addMetric(labelValueList,
+ ((Integer)
server.getAttribute(mBean.getObjectName(), "requestCount")).doubleValue());
+ }
+
+ mfs.add(requestProcessorBytesReceivedGauge);
+ mfs.add(requestProcessorBytesSentGauge);
+ mfs.add(requestProcessorProcessingTimeGauge);
+ mfs.add(requestProcessorRequestCounter);
+ mfs.add(requestProcessorErrorCounter);
+ }
+ } catch (Exception e) {
+ log.error("Error retrieving metric.", e);
+ }
+ }
+
+ private void addSessionMetrics(List<MetricFamilySamples> mfs) {
+ try {
+ final MBeanServer server =
ManagementFactory.getPlatformMBeanServer();
+ ObjectName filterName = new ObjectName(jmxDomain +
":type=Manager,context=*,host=*");
+ Set<ObjectInstance> mBeans =
server.queryMBeans(filterName, null);
+
+ if (mBeans.size() > 0) {
+ List<String> labelNameList =
Arrays.asList("host", "context");
+
+ GaugeMetricFamily activeSessionCountGauge = new
GaugeMetricFamily("tomcat_session_active_total",
+ "Number of active sessions",
labelNameList);
+
+ GaugeMetricFamily rejectedSessionCountGauge =
new GaugeMetricFamily("tomcat_session_rejected_total",
+ "Number of sessions rejected
due to maxActive being reached", labelNameList);
+
+ GaugeMetricFamily createdSessionCountGauge =
new GaugeMetricFamily("tomcat_session_created_total",
+ "Number of sessions created",
labelNameList);
+
+ GaugeMetricFamily expiredSessionCountGauge =
new GaugeMetricFamily("tomcat_session_expired_total",
+ "Number of sessions that
expired", labelNameList);
+
+ GaugeMetricFamily sessionAvgAliveTimeGauge =
new GaugeMetricFamily(
+
"tomcat_session_alivetime_seconds_avg", "Average time an expired session had
been alive",
+ labelNameList);
+
+ GaugeMetricFamily sessionMaxAliveTimeGauge =
new GaugeMetricFamily(
+
"tomcat_session_alivetime_seconds_max", "Maximum time an expired session had
been alive",
+ labelNameList);
+
+ GaugeMetricFamily contextStateGauge = new
GaugeMetricFamily("tomcat_context_state_started",
+ "Indication if the lifecycle
state of this context is STARTED", labelNameList);
+
+ for (final ObjectInstance mBean : mBeans) {
+ List<String> labelValueList =
Arrays.asList(mBean.getObjectName().getKeyProperty("host"),
+
mBean.getObjectName().getKeyProperty("context"));
+
+
activeSessionCountGauge.addMetric(labelValueList,
+ ((Integer)
server.getAttribute(mBean.getObjectName(), "activeSessions")).doubleValue());
+
+
rejectedSessionCountGauge.addMetric(labelValueList,
+ ((Integer)
server.getAttribute(mBean.getObjectName(), "rejectedSessions")).doubleValue());
+
+
createdSessionCountGauge.addMetric(labelValueList,
+ ((Long)
server.getAttribute(mBean.getObjectName(), "sessionCounter")).doubleValue());
+
+
expiredSessionCountGauge.addMetric(labelValueList,
+ ((Long)
server.getAttribute(mBean.getObjectName(), "expiredSessions")).doubleValue());
+
+
sessionAvgAliveTimeGauge.addMetric(labelValueList,
+ ((Integer)
server.getAttribute(mBean.getObjectName(), "sessionAverageAliveTime"))
+
.doubleValue());
+
+
sessionMaxAliveTimeGauge.addMetric(labelValueList,
+ ((Integer)
server.getAttribute(mBean.getObjectName(), "sessionMaxAliveTime"))
+
.doubleValue());
+
+ if
(server.getAttribute(mBean.getObjectName(), "stateName").equals("STARTED")) {
+
contextStateGauge.addMetric(labelValueList, 1.0);
+ } else {
+
contextStateGauge.addMetric(labelValueList, 0.0);
+ }
+ }
+
+ mfs.add(activeSessionCountGauge);
+ mfs.add(rejectedSessionCountGauge);
+ mfs.add(createdSessionCountGauge);
+ mfs.add(expiredSessionCountGauge);
+ mfs.add(sessionAvgAliveTimeGauge);
+ mfs.add(sessionMaxAliveTimeGauge);
+ mfs.add(contextStateGauge);
+ }
+ } catch (Exception e) {
+ log.error("Error retrieving metric.", e);
+ }
+ }
+
+ private void addThreadPoolMetrics(List<MetricFamilySamples> mfs) {
+ try {
+ final MBeanServer server =
ManagementFactory.getPlatformMBeanServer();
+ ObjectName filterName = new ObjectName(jmxDomain +
":type=ThreadPool,name=*");
+ Set<ObjectInstance> mBeans =
server.queryMBeans(filterName, null);
+
+ if (mBeans.size() > 0) {
+ List<String> labelList =
Collections.singletonList("name");
+
+ GaugeMetricFamily threadPoolCurrentCountGauge =
new GaugeMetricFamily("tomcat_threads_total",
+ "Number threads in this pool.",
labelList);
+
+ GaugeMetricFamily threadPoolActiveCountGauge =
new GaugeMetricFamily("tomcat_threads_active_total",
+ "Number of active threads in
this pool.", labelList);
+
+ GaugeMetricFamily threadPoolMaxThreadsGauge =
new GaugeMetricFamily("tomcat_threads_max",
+ "Maximum number of threads
allowed in this pool.", labelList);
+
+ GaugeMetricFamily
threadPoolConnectionCountGauge = new GaugeMetricFamily(
+
"tomcat_connections_active_total", "Number of connections served by this
pool.", labelList);
+
+ GaugeMetricFamily threadPoolMaxConnectionGauge
= new GaugeMetricFamily("tomcat_connections_active_max",
+ "Maximum number of concurrent
connections served by this pool.", labelList);
+
+ String[] genericAttributes = new String[] {
"currentThreadCount", "currentThreadsBusy", "maxThreads",
+ "connectionCount",
"maxConnections" };
+
+ for (final ObjectInstance mBean : mBeans) {
+ List<String> labelValueList =
Collections
+
.singletonList(mBean.getObjectName().getKeyProperty("name").replaceAll("[\"\\\\]",
""));
+ AttributeList attributeList =
server.getAttributes(mBean.getObjectName(), genericAttributes);
+ for (Attribute attribute :
attributeList.asList()) {
+ switch (attribute.getName()) {
+ case "currentThreadCount":
+
threadPoolCurrentCountGauge.addMetric(labelValueList,
+
((Integer) attribute.getValue()).doubleValue());
+ break;
+ case "currentThreadsBusy":
+
threadPoolActiveCountGauge.addMetric(labelValueList,
+
((Integer) attribute.getValue()).doubleValue());
+ break;
+ case "maxThreads":
+
threadPoolMaxThreadsGauge.addMetric(labelValueList,
+
((Integer) attribute.getValue()).doubleValue());
+ break;
+ case "connectionCount":
+
threadPoolConnectionCountGauge.addMetric(labelValueList,
+ ((Long)
attribute.getValue()).doubleValue());
+ break;
+ case "maxConnections":
+
threadPoolMaxConnectionGauge.addMetric(labelValueList,
+
((Integer) attribute.getValue()).doubleValue());
+
+ }
+ }
+ }
+
+ addNonEmptyMetricFamily(mfs,
threadPoolCurrentCountGauge);
+ addNonEmptyMetricFamily(mfs,
threadPoolActiveCountGauge);
+ addNonEmptyMetricFamily(mfs,
threadPoolMaxThreadsGauge);
+ addNonEmptyMetricFamily(mfs,
threadPoolConnectionCountGauge);
+ addNonEmptyMetricFamily(mfs,
threadPoolMaxConnectionGauge);
+ }
+ } catch (Exception e) {
+ log.error("Error retrieving metric:" + e.getMessage());
+ }
+ }
+
+ private void addVersionInfo(List<MetricFamilySamples> mfs) {
+ GaugeMetricFamily tomcatInfo = new
GaugeMetricFamily("tomcat_info", "tomcat version info",
+ Arrays.asList("version", "build"));
+
tomcatInfo.addMetric(Arrays.asList(ServerInfo.getServerNumber(),
ServerInfo.getServerBuilt()), 1);
+ mfs.add(tomcatInfo);
+ }
+
+ private void addNonEmptyMetricFamily(List<MetricFamilySamples> mfs,
GaugeMetricFamily metricFamily) {
+ if (metricFamily.samples.size() > 0) {
+ mfs.add(metricFamily);
+ }
+ }
+
+ public List<MetricFamilySamples> collect() {
+ List<MetricFamilySamples> mfs = new
ArrayList<MetricFamilySamples>();
+ addSessionMetrics(mfs);
+ addThreadPoolMetrics(mfs);
+ addRequestProcessorMetrics(mfs);
+ addVersionInfo(mfs);
+ return mfs;
+
+ }
+}
diff --git a/openmeetings-web/src/main/webapp/WEB-INF/web.xml
b/openmeetings-web/src/main/webapp/WEB-INF/web.xml
index 62e5e2a..265231f 100644
--- a/openmeetings-web/src/main/webapp/WEB-INF/web.xml
+++ b/openmeetings-web/src/main/webapp/WEB-INF/web.xml
@@ -54,6 +54,30 @@
<param-value>css,docs,images,js,persistence,public,screenshare,data,services</param-value>
</init-param>
</filter>
+
+ <filter>
+ <filter-name>prometheusFilter</filter-name>
+ <filter-class>io.prometheus.client.filter.MetricsFilter</filter-class>
+ <init-param>
+ <param-name>metric-name</param-name>
+ <param-value>webapp_metrics_filter</param-value>
+ </init-param>
+ <init-param>
+ <param-name>help</param-name>
+ <param-value>This is the help for your metrics filter</param-value>
+ </init-param>
+ <init-param>
+ <param-name>buckets</param-name>
+
<param-value>0.005,0.01,0.025,0.05,0.075,0.1,0.25,0.5,0.75,1,2.5,5,7.5,10</param-value>
+ </init-param>
+ <!-- Optionally override path components; anything less than 1 (1 is
the default)
+ means full granularity -->
+ <init-param>
+ <param-name>path-components</param-name>
+ <param-value>0</param-value>
+ </init-param>
+ </filter>
+
<filter-mapping>
<filter-name>OpenmeetingsApplication</filter-name>
<url-pattern>/*</url-pattern>
@@ -77,6 +101,15 @@
<servlet-name>CXFServlet</servlet-name>
<url-pattern>/services/*</url-pattern>
</servlet-mapping>
+
+ <servlet>
+ <servlet-name>metrics</servlet-name>
+
<servlet-class>io.prometheus.client.exporter.MetricsServlet</servlet-class>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>metrics</servlet-name>
+ <url-pattern>/services/metrics/</url-pattern>
+ </servlet-mapping>
<mime-mapping>
<extension>inc</extension>
diff --git a/pom.xml b/pom.xml
index 5ebedee..3770f02 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,6 +84,7 @@
<tomcat.version>9.0.40</tomcat.version>
<ical4j.version>3.0.18</ical4j.version>
<cxf.version>3.4.1</cxf.version>
+ <io.prometheus.version>0.10.0</io.prometheus.version>
<simple-xml.version>2.7.1</simple-xml.version>
<jettison.version>1.4.1</jettison.version>
<site.basedir>${project.basedir}</site.basedir>
@@ -530,6 +531,21 @@
<version>${cxf.version}</version>
</dependency>
<dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient</artifactId>
+ <version>${io.prometheus.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.prometheus</groupId>
+ <artifactId>simpleclient_servlet</artifactId>
+ <version>${io.prometheus.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tomcat</groupId>
+ <artifactId>tomcat-catalina</artifactId>
+ <version>${tomcat.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
<version>${jettison.version}</version>
