This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new af6fd8a0e [CELEBORN-1127] Add JVM classloader metrics
af6fd8a0e is described below
commit af6fd8a0e6fdc049fb0ba580f17d3640ffad302f
Author: onebox-li <[email protected]>
AuthorDate: Thu Dec 7 09:47:23 2023 +0800
[CELEBORN-1127] Add JVM classloader metrics
### What changes were proposed in this pull request?
Add JVM classloader metrics for loaded and unloaded count.
### Why are the changes needed?
Ditto.
### Does this PR introduce _any_ user-facing change?
Add two classloader-related panels.
### How was this patch tested?
Cluster test.
Closes #2099 from onebox-li/add-classloader-metrics.
Authored-by: onebox-li <[email protected]>
Signed-off-by: zky.zhoukeyong <[email protected]>
---
METRICS.md | 2 +
assets/grafana/celeborn-jvm-dashboard.json | 203 +++++++++++++++++++++
.../celeborn/common/metrics/source/JVMSource.scala | 9 +-
3 files changed, 213 insertions(+), 1 deletion(-)
diff --git a/METRICS.md b/METRICS.md
index b4f10cdea..87ef2f11b 100644
--- a/METRICS.md
+++ b/METRICS.md
@@ -151,6 +151,8 @@ Here is an example of Grafana dashboard importing.
| jvm_thread_terminated_count | JVM |
The current number of threads having terminated state.
|
| jvm_thread_timed_waiting_count | JVM |
The current number of threads having timed_waiting state.
|
| jvm_thread_waiting_count | JVM |
The current number of threads having waiting state.
|
+| jvm_classloader_loaded | JVM |
The total number of classes loaded since the start of the JVM.
|
+| jvm_classloader_unloaded | JVM |
The total number of classes unloaded since the start of the JVM.
|
| JVMCPUTime | system |
The JVM costs cpu time.
|
| AvailableProcessors | system |
The amount of system available processors.
|
| LastMinuteSystemLoad | system |
The last minute load of system.
|
diff --git a/assets/grafana/celeborn-jvm-dashboard.json
b/assets/grafana/celeborn-jvm-dashboard.json
index 90631f4ee..4c20e5a0a 100644
--- a/assets/grafana/celeborn-jvm-dashboard.json
+++ b/assets/grafana/celeborn-jvm-dashboard.json
@@ -1835,6 +1835,209 @@
],
"title": "Mapped Buffers",
"type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 79
+ },
+ "id": 50,
+ "panels": [],
+ "title": "ClassLoader",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 80
+ },
+ "id": 47,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "aggregator": "sum",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "downsampleAggregator": "avg",
+ "downsampleFillPolicy": "none",
+ "editorMode": "code",
+ "expr":
"metrics_jvm_classloader_loaded_Value{instance=~\"${instance}\"}",
+ "legendFormat": "loaded_${baseLegend}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Class Loaded",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 80
+ },
+ "id": 48,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "aggregator": "sum",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "downsampleAggregator": "avg",
+ "downsampleFillPolicy": "none",
+ "editorMode": "code",
+ "expr":
"metrics_jvm_classloader_unloaded_Value{instance=~\"${instance}\"}",
+ "legendFormat": "unloaded_${baseLegend}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Class Unloaded",
+ "type": "timeseries"
}
],
"schemaVersion": 37,
diff --git
a/common/src/main/scala/org/apache/celeborn/common/metrics/source/JVMSource.scala
b/common/src/main/scala/org/apache/celeborn/common/metrics/source/JVMSource.scala
index a87924868..4c7c395fa 100644
---
a/common/src/main/scala/org/apache/celeborn/common/metrics/source/JVMSource.scala
+++
b/common/src/main/scala/org/apache/celeborn/common/metrics/source/JVMSource.scala
@@ -23,7 +23,7 @@ import scala.collection.JavaConverters._
import scala.collection.mutable
import com.codahale.metrics.{Gauge, MetricRegistry}
-import com.codahale.metrics.jvm.{BufferPoolMetricSet,
GarbageCollectorMetricSet, MemoryUsageGaugeSet, ThreadStatesGaugeSet}
+import com.codahale.metrics.jvm.{BufferPoolMetricSet, ClassLoadingGaugeSet,
GarbageCollectorMetricSet, MemoryUsageGaugeSet, ThreadStatesGaugeSet}
import org.apache.celeborn.common.CelebornConf
@@ -92,6 +92,12 @@ class JVMSource(conf: CelebornConf, role: String) extends
AbstractSource(conf, r
case (name, metric) => new IllegalArgumentException(s"Unknown metric type:
$name: $metric")
})
+ Seq(new ClassLoadingGaugeSet()).map(_.getMetrics.asScala.map {
+ case (name: String, metric: Gauge[_]) =>
+ addGauge(MetricRegistry.name(JVM_METRIC_CLASSLOADER_PREFIX, name),
metric)
+ case (name, metric) => new IllegalArgumentException(s"Unknown metric type:
$name: $metric")
+ })
+
// start cleaner
startCleaner()
}
@@ -100,6 +106,7 @@ object JVMSource {
private val JVM_METRIC_PREFIX = "jvm"
private val JVM_METRIC_MEMORY_PREFIX = JVM_METRIC_PREFIX + ".memory"
private val JVM_METRIC_THREAD_PREFIX = JVM_METRIC_PREFIX + ".thread"
+ private val JVM_METRIC_CLASSLOADER_PREFIX = JVM_METRIC_PREFIX +
".classloader"
private val WHITESPACE = "\\s+".r.pattern
}
