This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 74dd91f703c HDDS-15336. Dashboard for OM Performance overview (#10320)
74dd91f703c is described below
commit 74dd91f703c1dccdd21361f79748493e39587724
Author: Andrey Yarovoy <[email protected]>
AuthorDate: Fri May 29 17:25:51 2026 +0300
HDDS-15336. Dashboard for OM Performance overview (#10320)
---
.../grafana/dashboards/Ozone - OM Overview.json | 2796 ++++++++++++++++++++
1 file changed, 2796 insertions(+)
diff --git a/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone
- OM Overview.json
b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OM
Overview.json
new file mode 100644
index 00000000000..803711e28ec
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - OM
Overview.json
@@ -0,0 +1,2796 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Ozone Manager OM `/prom`: **Operations** pairs
**`om_metrics` rate rows** with **`OmClientProtocol`** latency
(**`OmClientProtocol.proto` `Type` enums**) via **`rate(time)/rate(counter)`**.
**Legends**: **right**, **mean** / **max** per series (**table** legend on time
series panels); non-JVM series use **`{{instance}}`** in legend text (scraped
target; avoids repeating **`hostname`** when it matches the host portion of
**`instance`**). JVM rows keep **`{{hostname}}`** [...]
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "links": [],
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 1,
+ "panels": [],
+ "title": "JVM",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "percentunit",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "description": "**Note:** OM **CPU gauges** (`CpuMetrics` → record
`JvmMetricsCpu`) generally **do not** carry `processname=\"OzoneManager\"`
(only `instance` scrape labels). Omit that filter.",
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "jvm_metrics_cpu_jvm_load{instance=~\"$instance\"}",
+ "legendFormat": "JVM · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "jvm_metrics_cpu_system_load{instance=~\"$instance\"}",
+ "legendFormat": "system · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "JVM CPU load",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "decmbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 9
+ },
+ "id": 3,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_mem_heap_used_m{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "used · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_mem_heap_committed_m{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "committed · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_mem_heap_max_m{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "max · {{hostname}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Heap — used / committed / max",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "decmbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 19
+ },
+ "id": 4,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_mem_non_heap_used_m{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "used · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_mem_non_heap_committed_m{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "committed · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_mem_non_heap_max_m{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "max · {{hostname}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Non-heap (native / metaspace) — used / committed / max",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "percentunit",
+ "min": 0
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 27
+ },
+ "id": 5,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"increase(jvm_metrics_gc_time_millis{instance=~\"$instance\",processname=\"OzoneManager\"}[1m])
/ 60000",
+ "legendFormat": "total · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"increase(jvm_metrics_gc_time_millis_g1_young_generation{instance=~\"$instance\",processname=\"OzoneManager\"}[1m])
/ 60000",
+ "legendFormat": "G1 young · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"increase(jvm_metrics_gc_time_millis_g1_old_generation{instance=~\"$instance\",processname=\"OzoneManager\"}[1m])
/ 60000",
+ "legendFormat": "G1 old · {{hostname}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "GC time (fraction of wall per minute)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 35
+ },
+ "id": 6,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"rate(jvm_metrics_gc_count{instance=~\"$instance\",processname=\"OzoneManager\"}[$__rate_interval])",
+ "legendFormat": "total · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"rate(jvm_metrics_gc_count_g1_young_generation{instance=~\"$instance\",processname=\"OzoneManager\"}[$__rate_interval])",
+ "legendFormat": "G1 young · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"rate(jvm_metrics_gc_count_g1_old_generation{instance=~\"$instance\",processname=\"OzoneManager\"}[$__rate_interval])",
+ "legendFormat": "G1 old · {{hostname}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "GC count rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 43
+ },
+ "id": 7,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "description": "**Note:** **`NettyMetrics`** does **not** set
`processname`; filter **only `instance`** (same CPU panel rationale). Direct
memory counters come from OM Ratis/Netty use.",
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "netty_metrics_used_direct_mem{instance=~\"$instance\"}",
+ "legendFormat": "used · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "netty_metrics_max_direct_mem{instance=~\"$instance\"}",
+ "legendFormat": "max · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Netty direct memory — used / max",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 55,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "normal"
+ },
+ "axisLabel": "Thread count"
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 51
+ },
+ "id": 8,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_threads_new{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "new · {{hostname}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_threads_runnable{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "runnable · {{hostname}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_threads_blocked{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "blocked · {{hostname}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_threads_waiting{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "waiting · {{hostname}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_threads_timed_waiting{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "timed_waiting · {{hostname}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"jvm_metrics_threads_terminated{instance=~\"$instance\",processname=\"OzoneManager\"}",
+ "legendFormat": "terminated · {{hostname}}",
+ "range": true,
+ "refId": "F"
+ }
+ ],
+ "title": "Thread count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "drawStyle": "line",
+ "fillOpacity": 8,
+ "lineWidth": 1,
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "axisLabel": "Threads (live / idle / max)",
+ "axisPlacement": "auto"
+ },
+ "unit": "none"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "D"
+ },
+ "properties": [
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ },
+ {
+ "id": "custom.axisLabel",
+ "value": "Queued tasks (waiting)"
+ },
+ {
+ "id": "custom.lineWidth",
+ "value": 2
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 61
+ },
+ "id": 9,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"(http_server2_metrics_http_server_thread_count{instance=~\"$instance\",server_name=~\"ozoneManager\"}
or
http_server2_metrics_http_server_thread_count{instance=~\"$instance\",servername=~\"ozoneManager\"})",
+ "legendFormat": "threads (live) · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"(http_server2_metrics_http_server_idle_thread_count{instance=~\"$instance\",server_name=~\"ozoneManager\"}
or
http_server2_metrics_http_server_idle_thread_count{instance=~\"$instance\",servername=~\"ozoneManager\"})",
+ "legendFormat": "idle · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"(http_server2_metrics_http_server_max_thread_count{instance=~\"$instance\",server_name=~\"ozoneManager\"}
or
http_server2_metrics_http_server_max_thread_count{instance=~\"$instance\",servername=~\"ozoneManager\"})",
+ "legendFormat": "max · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"(http_server2_metrics_http_server_thread_queue_waiting_task_count{instance=~\"$instance\",server_name=~\"ozoneManager\"}
or
http_server2_metrics_http_server_thread_queue_waiting_task_count{instance=~\"$instance\",servername=~\"ozoneManager\"})",
+ "legendFormat": "queue (waiting) · {{instance}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Jetty http server threads",
+ "description": "OM Jetty pools: **`HttpServer2Metrics`** tags
**`server_name=ozoneManager`** (camelCase). Some Hadoop stacks only expose
**`servername`**. Prometheus **cannot** OR label keys inside one `{...}`; use
**`series{...} or series{...}`** as written. If panels stay empty but JFR shows
Jetty threads, broaden to **`{instance=~\"$instance\"}`** only (one pool per
OM).",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 71
+ },
+ "id": 10,
+ "panels": [],
+ "title": "Operations",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 72
+ },
+ "id": 11,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(CreateVolume|SetVolumeProperty|CheckVolumeAccess|InfoVolume|DeleteVolume|ListVolume)$\"}[$__rate_interval])\n)",
+ "legendFormat": "volume tier · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(CreateBucket|InfoBucket|SetBucketProperty|DeleteBucket|ListBuckets|ServiceList|GetS3VolumeContext)$\"}[$__rate_interval])\n)",
+ "legendFormat": "bucket tier · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(CreateKey|LookupKey|RenameKey|DeleteKey|ListKeys|CommitKey|AllocateBlock|DeleteKeys|RenameKeys|GetKeyInfo|ListKeysLight|InitiateMultiPartUpload|CommitMultiPartUpload|CompleteMultiPartUpload|AbortMultiPartUpload|ListMultiPartUploadParts|ListMultipartUploads|ListOpenFiles|PutObjectTagging|GetObjectTagging|DeleteObjectTagging)$\"}[$__rate_interval])\n)",
+ "legendFormat": "key tier · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(GetFileStatus|CreateDirectory|CreateFile|LookupFile|ListStatus|ListStatusLight|RecoverLease)$\"}[$__rate_interval])\n)",
+ "legendFormat": "fs tier · {{instance}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Aggregate tiers — rate",
+ "type": "timeseries",
+ "description": "Tier **`rate(om_client_protocol_counter)`**/s (**4**
aggregates). Pair: **Aggregate tiers — latency** (tier-weighted mean ms)."
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "description": "Tier **`latency`** (**`rate(time)`/`rate(counter)`**,
ms); **4** series matching legend names on **Aggregate tiers — rate**.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 81
+ },
+ "id": 401,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=~\"^(CreateVolume|SetVolumeProperty|CheckVolumeAccess|InfoVolume|DeleteVolume|ListVolume)$\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance) (\n
clamp_min(rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(CreateVolume|SetVolumeProperty|CheckVolumeAccess|InfoVolume|DeleteVolume|ListVolume)$\"}[$__rate_interval]),
1e-12)\n )\n)",
+ "legendFormat": "volume tier · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=~\"^(CreateBucket|InfoBucket|SetBucketProperty|DeleteBucket|ListBuckets|ServiceList|GetS3VolumeContext)$\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance) (\n
clamp_min(rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(CreateBucket|InfoBucket|SetBucketProperty|DeleteBucket|ListBuckets|ServiceList|GetS3VolumeContext)$\"}[$__rate_interval]),
1 [...]
+ "legendFormat": "bucket tier · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=~\"^(CreateKey|LookupKey|RenameKey|DeleteKey|ListKeys|CommitKey|AllocateBlock|DeleteKeys|RenameKeys|GetKeyInfo|ListKeysLight|InitiateMultiPartUpload|CommitMultiPartUpload|CompleteMultiPartUpload|AbortMultiPartUpload|ListMultiPartUploadParts|ListMultipartUploads|ListOpenFiles|PutObjectTagging|GetObjectTagging|DeleteObjectTagging)$\"}[$__rate_interval])\n
)\n /\n sum by ( [...]
+ "legendFormat": "key tier · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=~\"^(GetFileStatus|CreateDirectory|CreateFile|LookupFile|ListStatus|ListStatusLight|RecoverLease)$\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance) (\n
clamp_min(rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"^(GetFileStatus|CreateDirectory|CreateFile|LookupFile|ListStatus|ListStatusLight|RecoverLease)$\"}[$__rate_interval]),
1e-12)\n )\n)",
+ "legendFormat": "fs tier · {{instance}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Aggregate tiers — latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 90
+ },
+ "id": 12,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_volume_creates{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "volume creates · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_volume_deletes{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "volume deletes · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_volume_updates{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "volume updates · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_volume_infos{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "volume info · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_volume_lists{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "volume list · {{instance}}",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Volume mutations & listings — rate",
+ "type": "timeseries",
+ "description": "**Volume mutations & listings — rate** pair:
**`om_metrics`** **`rate(...)`**/s (**5**). Legends match **… — latency**."
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "description": "**Volume mutations & listings — latency**:
**OmClientProtocol** (**5** **`latency`** queries). Legends match rate panel.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 99
+ },
+ "id": 402,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"CreateVolume\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"CreateVolume\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "volume creates · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"DeleteVolume\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"DeleteVolume\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "volume deletes · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"SetVolumeProperty\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"SetVolumeProperty\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "volume updates · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"InfoVolume\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"InfoVolume\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "volume info · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"ListVolume\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"ListVolume\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "volume list · {{instance}}",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Volume mutations & listings — latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 108
+ },
+ "id": 13,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n (\n
rate(om_metrics_num_bucket_creates{instance=~\"$instance\"}[$__rate_interval])\n
+ \n
rate(om_metrics_num_fso_bucket_creates{instance=~\"$instance\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket creates · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n (\n
rate(om_metrics_num_bucket_deletes{instance=~\"$instance\"}[$__rate_interval])\n
+ \n
rate(om_metrics_num_fso_bucket_deletes{instance=~\"$instance\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket deletes · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_bucket_updates{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "bucket updates · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_bucket_infos{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "bucket info · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_bucket_lists{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "bucket list · {{instance}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"ServiceList\"}[$__rate_interval])\n)",
+ "legendFormat": "service list · {{instance}}",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"GetS3VolumeContext\"}[$__rate_interval])\n)",
+ "legendFormat": "GetS3VolumeContext · {{instance}}",
+ "range": true,
+ "refId": "G"
+ }
+ ],
+ "title": "Buckets & layouts — rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "description": "**Paired legends** match **Buckets & layouts — rate**
(seven series). OBS+FSO create/delete **`om_metrics`** rows are summed on the
rate side. **`service list`** is **OmClient ServiceList RPC**
(**`om_client_protocol_*`** rate/latency), not
**`om_metrics_num_bucket_s3_lists`** (that counter has no callers in OM).
**`GetS3VolumeContext`** likewise uses **`om_client_protocol_*`**.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 24,
+ "x": 0,
+ "y": 117
+ },
+ "id": 403,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"CreateBucket\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"CreateBucket\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket creates · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"DeleteBucket\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"DeleteBucket\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket deletes · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"SetBucketProperty\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"SetBucketProperty\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket updates · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"InfoBucket\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"InfoBucket\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket info · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"ListBuckets\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"ListBuckets\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "bucket list · {{instance}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"ServiceList\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"ServiceList\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "service list · {{instance}}",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"GetS3VolumeContext\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"GetS3VolumeContext\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "GetS3VolumeContext · {{instance}}",
+ "range": true,
+ "refId": "G"
+ }
+ ],
+ "title": "Buckets & layouts — latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 128
+ },
+ "id": 14,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_allocate{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "allocate · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_commits{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "commit · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_h_syncs{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "hsync · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_deletes{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "delete · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_lists{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "key list · {{instance}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_lookup{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "lookup · {{instance}}",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_key_renames{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "rename · {{instance}}",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_block_allocations{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "block alloc · {{instance}}",
+ "range": true,
+ "refId": "H"
+ }
+ ],
+ "title": "Keys, commits & block alloc — rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "description": "**Paired legends** match **Keys … — rate** (eight
series). **`CommitKey`** OmClient latency is **shared**, while
**`om_metrics_num_key_commits`** vs **`om_metrics_num_key_h_syncs`**
**partition** **`CommitKey`** RPCs (**`hsync`** flag vs normal close);
**commit** / **hsync** latency multiply that aggregate latency by
**`(rate(counter) >bool 0)`** on **their paired counter**, so **hsync latency
stays zero when only non-hsync closes run** (**aligns with hsync rate zer [...]
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 11,
+ "w": 24,
+ "x": 0,
+ "y": 137
+ },
+ "id": 404,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"AllocateBlock\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"AllocateBlock\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "allocate · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n (\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"CommitKey\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"CommitKey\"}[$__rate_interval])\n
)\n )\n *\n (\n sum by (hostname, instance) (\n
rate(om_metrics_num_key_commits{instance=~\"$instance\"}[$__rate_interval])\n
) > bool 0\n )\n)",
+ "legendFormat": "commit · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n (\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"CommitKey\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"CommitKey\"}[$__rate_interval])\n
)\n )\n *\n (\n sum by (hostname, instance) (\n
rate(om_metrics_num_key_h_syncs{instance=~\"$instance\"}[$__rate_interval])\n
) > bool 0\n )\n)",
+ "legendFormat": "hsync · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"DeleteKey\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"DeleteKey\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "delete · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n (\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=~\"ListKeys|ListKeysLight\"}[$__rate_interval])\n
)\n )\n /\n (\n sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"ListKeys|ListKeysLight\"}[$__rate_interval])\n
)\n )\n)",
+ "legendFormat": "key list · {{instance}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"LookupKey\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"LookupKey\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "lookup · {{instance}}",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"RenameKey\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"RenameKey\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "rename · {{instance}}",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"AllocateBlock\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"AllocateBlock\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "block alloc · {{instance}}",
+ "range": true,
+ "refId": "H"
+ }
+ ],
+ "title": "Keys, commits & block alloc — latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 148
+ },
+ "id": 15,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_get_file_status{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "getFileStatus · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_create_directory{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "mkdir · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_create_file{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "create file · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_lookup_file{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "lookup file · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_metrics_num_list_status{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "listStatus · {{instance}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_client_protocol_counter{instance=~\"$instance\",type=\"ListStatusLight\"}[$__rate_interval]))",
+ "legendFormat": "listStatusLight · {{instance}}",
+ "range": true,
+ "refId": "F"
+ }
+ ],
+ "title": "FS/OFS primitives — rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "description": "**Paired legends** match **FS/OFS primitives — rate**
(six series). **`listStatus`** **rate** uses **`om_metrics_num_list_status`**,
incremented for **either** **`ListStatus`** **or** **`ListStatusLight`**
(**`OmMetadataReader.listStatusLight`** delegates into **`listStatus`**).
**`listStatus`** **latency** therefore blends **`om_client_protocol_*`** for
**`ListStatus|ListStatusLight`**. **`listStatusLight`** **rate** stays
**`om_client_protocol_counter{type=\"ListS [...]
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 157
+ },
+ "id": 405,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"GetFileStatus\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"GetFileStatus\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "getFileStatus · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"CreateDirectory\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"CreateDirectory\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "mkdir · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"CreateFile\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"CreateFile\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "create file · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"LookupFile\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"LookupFile\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "lookup file · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n (\n sum by (hostname, instance) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=~\"ListStatus|ListStatusLight\"}[$__rate_interval])\n
)\n )\n /\n (\n sum by (hostname, instance) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=~\"ListStatus|ListStatusLight\"}[$__rate_interval])\n
)\n )\n)",
+ "legendFormat": "listStatus · {{instance}}",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_time{instance=~\"$instance\",type=\"ListStatusLight\"}[$__rate_interval])\n
)\n /\n sum by (hostname, instance, type) (\n
rate(om_client_protocol_counter{instance=~\"$instance\",type=\"ListStatusLight\"}[$__rate_interval])\n
)\n)",
+ "legendFormat": "listStatusLight · {{instance}}",
+ "range": true,
+ "refId": "F"
+ }
+ ],
+ "title": "FS/OFS primitives — latency",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 167
+ },
+ "id": 18,
+ "panels": [],
+ "title": "Deleting Service Metrics",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byFrameRefID",
+ "options": "E"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "Bps"
+ },
+ {
+ "id": "custom.axisPlacement",
+ "value": "right"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 168
+ },
+ "id": 19,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(deleting_service_metrics_num_keys_processed{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "keys processed · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(deleting_service_metrics_num_keys_purged{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "keys purged · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(deleting_service_metrics_num_dirs_purged{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "dirs purged · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(deleting_service_metrics_keys_reclaimed_in_interval{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "keys reclaimed (interval counter) · {{instance}}",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(deleting_service_metrics_reclaimed_size_in_interval{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "reclaimed logical volume · {{instance}}",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Deletion pipeline",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 177
+ },
+ "id": 20,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"om_performance_metrics_key_deleting_service_latency_ms{instance=~\"$instance\"}",
+ "legendFormat": "KeyDeletingService · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"om_performance_metrics_directory_deleting_service_latency_ms{instance=~\"$instance\"}",
+ "legendFormat": "DirectoryDeletingService · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"om_performance_metrics_open_key_cleanup_service_latency_ms{instance=~\"$instance\"}",
+ "legendFormat": "OpenKeyCleanup · {{instance}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Per-iteration OM deletion service timings (milliseconds)",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 185
+ },
+ "id": 21,
+ "panels": [],
+ "title": "OM Ratis",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 186
+ },
+ "id": 22,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_performance_metrics_pre_execute_latency_ns_num_ops{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "preExecute · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_performance_metrics_submit_to_ratis_latency_ns_num_ops{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "submitToRatis · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_performance_metrics_validate_response_latency_ns_num_ops{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "validateResponse · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(rate(om_performance_metrics_create_om_response_latency_ns_num_ops{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "createOmResponse · {{instance}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Ratis Operations rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 24,
+ "x": 0,
+ "y": 195
+ },
+ "id": 23,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(om_performance_metrics_pre_execute_latency_ns_avg_time{instance=~\"$instance\"}
/ 1e6)",
+ "legendFormat": "preExecute · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(om_performance_metrics_submit_to_ratis_latency_ns_avg_time{instance=~\"$instance\"}
/ 1e6)",
+ "legendFormat": "submitToRatis · {{instance}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(om_performance_metrics_validate_response_latency_ns_avg_time{instance=~\"$instance\"}
/ 1e6)",
+ "legendFormat": "validateResponse · {{instance}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(om_performance_metrics_create_om_response_latency_ns_avg_time{instance=~\"$instance\"}
/ 1e6)",
+ "legendFormat": "createOmResponse · {{instance}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Ratis Operations latency",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 204
+ },
+ "id": 24,
+ "panels": [],
+ "title": "HA",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "type": "value",
+ "options": {
+ "0": {
+ "text": "Follower"
+ }
+ }
+ },
+ {
+ "type": "value",
+ "options": {
+ "1": {
+ "text": "Leader"
+ }
+ }
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "blue",
+ "value": null
+ },
+ {
+ "color": "green",
+ "value": 1
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 6,
+ "w": 24,
+ "x": 0,
+ "y": 205
+ },
+ "id": 25,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "orientation": "auto",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "value_and_name",
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr":
"omha_metrics_ozone_manager_ha_leader_state{instance=~\"$instance\"}",
+ "legendFormat": "{{nodeid}} · {{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OM HA leader state (1 = leader, 0 = follower)",
+ "description": "`omha_metrics_ozone_manager_ha_leader_state`; tag
exposes OM `node_id` from OMHAMetrics.",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 211
+ },
+ "id": 26,
+ "panels": [],
+ "title": "Storage Utilization",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 24,
+ "x": 0,
+ "y": 212
+ },
+ "id": 27,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(bucket_utilization_metrics_bucket_used_bytes{instance=~\"$instance\"})",
+ "legendFormat": "used logical bytes · {{instance}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance)
(bucket_utilization_metrics_bucket_snapshot_used_bytes{instance=~\"$instance\"})",
+ "legendFormat": "snapshot-held bytes · {{instance}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Total logical used bytes across all buckets (instantaneous
sum)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "unit": "bytes",
+ "decimals": null,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 220
+ },
+ "id": 28,
+ "options": {
+ "orientation": "horizontal",
+ "displayMode": "gradient",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showUnfilled": true,
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": false,
+ "calcs": [
+ "mean",
+ "max"
+ ]
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "topk(10, sum by (hostname, volumename, instance)
(bucket_utilization_metrics_bucket_used_bytes{instance=~\"$instance\"}))",
+ "legendFormat": "{{volumename}} · {{instance}}",
+ "range": false,
+ "instant": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Top 10 volumes by summed bucket-used bytes",
+ "description": "Buckets per volume are summed; tag `volumename`
originates from OM bucket utilization Metrics2 export.",
+ "type": "bargauge"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 230
+ },
+ "id": 29,
+ "panels": [],
+ "title": "RPC handlers",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 231
+ },
+ "id": 30,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "sum by (hostname, instance, type)
(rate(om_client_protocol_counter{instance=~\"$instance\"}[$__rate_interval]))",
+ "legendFormat": "{{type}} · {{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "RPC rates",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "drawStyle": "line",
+ "fillOpacity": 10,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "showPoints": "never",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ }
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 241
+ },
+ "id": 31,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean",
+ "max"
+ ],
+ "displayMode": "table",
+ "placement": "right",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.4.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus"
+ },
+ "editorMode": "code",
+ "expr": "(\n sum by (hostname, instance, type)
(rate(om_client_protocol_time{instance=~\"$instance\"}[$__rate_interval]))\n
/\n sum by (hostname, instance, type)
(rate(om_client_protocol_counter{instance=~\"$instance\"}[$__rate_interval]))\n)",
+ "legendFormat": "{{type}} · {{instance}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "description": "Mean handler milliseconds per protobuf RPC type ≈
**`rate(sum duration) / rate(count)`**. **Duration** increments use monotonic
millis from `ProtocolMessageMetrics` (same clock as Hadoop
`Time.monotonicNow()`). Series disappear when denominators drop to zero;
**+Inf** gaps are omitted by Grafana.",
+ "title": "RPC latency",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 39,
+ "tags": [
+ "ozone",
+ "om",
+ "overview",
+ "jvm",
+ "prometheus",
+ "metrics2"
+ ],
+ "templating": {
+ "list": [
+ {
+ "allValue": ".*",
+ "current": {
+ "selected": true,
+ "text": [
+ "All"
+ ],
+ "value": [
+ "$__all"
+ ]
+ },
+ "datasource": {
+ "type": "prometheus"
+ },
+ "definition":
"label_values(jvm_metrics_mem_heap_used_m{processname=\"OzoneManager\"},instance)",
+ "hide": 0,
+ "includeAll": true,
+ "label": "OM instance",
+ "multi": true,
+ "name": "instance",
+ "options": [],
+ "query": {
+ "query":
"label_values(jvm_metrics_mem_heap_used_m{processname=\"OzoneManager\"},instance)",
+ "refId": "StandardVariableQuery"
+ },
+ "refresh": 2,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "type": "query"
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "browser",
+ "title": "Ozone - OM Overview",
+ "uid": "ozone-om-overview",
+ "version": 22,
+ "weekStart": ""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
