This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 442d59ab5 [CELEBORN-933] Add metrics about active shuffle data size
442d59ab5 is described below
commit 442d59ab555dcabce67791c2f4f3b4ed43636b18
Author: mingji <[email protected]>
AuthorDate: Wed Aug 30 18:04:57 2023 +0800
[CELEBORN-933] Add metrics about active shuffle data size
### What changes were proposed in this pull request?
Add metrics about active shuffle data size in every worker and update
Grafana dashboard. The metric value will decrease when shuffle is expired.
### Why are the changes needed?
Ditto.
### Does this PR introduce _any_ user-facing change?
NO.
### How was this patch tested?
Cluster.
<img width="733" alt="截屏2023-08-30 17 00 11"
src="https://github.com/apache/incubator-celeborn/assets/4150993/48e28c1c-2b49-45d7-b3ba-358674ff3f3d";>
Closes #1867 from FMX/CELEBORN-933.
Authored-by: mingji <[email protected]>
Signed-off-by: zky.zhoukeyong <[email protected]>
---
assets/grafana/celeborn-dashboard.json | 268 +++++++++++++++------
.../org/apache/celeborn/common/meta/FileInfo.java | 4 +
.../celeborn/service/deploy/worker/Worker.scala | 3 +
.../service/deploy/worker/WorkerSource.scala | 3 +
.../deploy/worker/storage/StorageManager.scala | 5 +
5 files changed, 208 insertions(+), 75 deletions(-)
diff --git a/assets/grafana/celeborn-dashboard.json
b/assets/grafana/celeborn-dashboard.json
index 6ace813c5..12549d877 100644
--- a/assets/grafana/celeborn-dashboard.json
+++ b/assets/grafana/celeborn-dashboard.json
@@ -324,7 +324,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 19
+ "y": 27
},
"id": 121,
"options": {
@@ -416,7 +416,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 19
+ "y": 27
},
"id": 120,
"options": {
@@ -507,7 +507,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 27
+ "y": 35
},
"id": 100,
"options": {
@@ -596,7 +596,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 27
+ "y": 35
},
"id": 102,
"options": {
@@ -685,7 +685,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 35
+ "y": 43
},
"id": 36,
"options": {
@@ -773,7 +773,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 35
+ "y": 43
},
"id": 117,
"options": {
@@ -1352,6 +1352,124 @@
],
"title": "ActiveConnectionCount",
"type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+
"metrics_ActiveShuffleSize_Value{instance=\"core-1-1:9096\", job=\"RSS\",
role=\"Worker\"}"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 30
+ },
+ "id": 181,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "editorMode": "builder",
+ "expr": "metrics_ActiveShuffleSize_Value",
+ "instant": false,
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "metrics_ActiveShuffleSize_Value",
+ "type": "timeseries"
}
],
"title": "Worker",
@@ -1428,7 +1546,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 47
+ "y": 55
},
"id": 68,
"options": {
@@ -1517,7 +1635,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 47
+ "y": 55
},
"id": 70,
"options": {
@@ -1606,7 +1724,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 56
+ "y": 64
},
"id": 72,
"options": {
@@ -1695,7 +1813,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 56
+ "y": 64
},
"id": 74,
"options": {
@@ -1783,7 +1901,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 65
+ "y": 73
},
"id": 76,
"options": {
@@ -1873,7 +1991,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 65
+ "y": 73
},
"id": 129,
"options": {
@@ -1963,7 +2081,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 73
+ "y": 81
},
"id": 128,
"options": {
@@ -2053,7 +2171,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 73
+ "y": 81
},
"id": 131,
"options": {
@@ -2143,7 +2261,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 81
+ "y": 89
},
"id": 132,
"options": {
@@ -2233,7 +2351,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 81
+ "y": 89
},
"id": 130,
"options": {
@@ -2338,7 +2456,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 48
+ "y": 56
},
"id": 66,
"options": {
@@ -2427,7 +2545,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 48
+ "y": 56
},
"id": 96,
"options": {
@@ -2516,7 +2634,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 56
+ "y": 64
},
"id": 17,
"options": {
@@ -2605,7 +2723,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 56
+ "y": 64
},
"id": 18,
"options": {
@@ -2708,7 +2826,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 49
+ "y": 57
},
"id": 78,
"options": {
@@ -2797,7 +2915,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 49
+ "y": 57
},
"id": 80,
"options": {
@@ -2886,7 +3004,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 57
+ "y": 65
},
"id": 4,
"options": {
@@ -2975,7 +3093,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 57
+ "y": 65
},
"id": 6,
"options": {
@@ -3064,7 +3182,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 66
+ "y": 74
},
"id": 56,
"options": {
@@ -3153,7 +3271,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 66
+ "y": 74
},
"id": 58,
"options": {
@@ -3256,7 +3374,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 23
+ "y": 31
},
"id": 19,
"options": {
@@ -3345,7 +3463,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 23
+ "y": 31
},
"id": 20,
"options": {
@@ -3434,7 +3552,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 31
+ "y": 39
},
"id": 165,
"options": {
@@ -3525,7 +3643,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 31
+ "y": 39
},
"id": 166,
"options": {
@@ -3616,7 +3734,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 39
+ "y": 47
},
"id": 167,
"options": {
@@ -3707,7 +3825,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 39
+ "y": 47
},
"id": 168,
"options": {
@@ -3798,7 +3916,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 47
+ "y": 55
},
"id": 169,
"options": {
@@ -3889,7 +4007,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 47
+ "y": 55
},
"id": 170,
"options": {
@@ -3979,7 +4097,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 55
+ "y": 63
},
"id": 108,
"options": {
@@ -4071,7 +4189,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 55
+ "y": 63
},
"id": 104,
"options": {
@@ -4162,7 +4280,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 63
+ "y": 71
},
"id": 106,
"options": {
@@ -4268,7 +4386,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 8
+ "y": 16
},
"id": 44,
"options": {
@@ -4357,7 +4475,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 8
+ "y": 16
},
"id": 46,
"options": {
@@ -4445,7 +4563,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 16
+ "y": 24
},
"id": 48,
"options": {
@@ -4533,7 +4651,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 16
+ "y": 24
},
"id": 180,
"options": {
@@ -4622,7 +4740,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 24
+ "y": 32
},
"id": 88,
"options": {
@@ -4710,7 +4828,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 24
+ "y": 32
},
"id": 135,
"options": {
@@ -4816,7 +4934,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 9
+ "y": 17
},
"id": 159,
"options": {
@@ -4908,7 +5026,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 9
+ "y": 17
},
"id": 160,
"options": {
@@ -5000,7 +5118,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 17
+ "y": 25
},
"id": 161,
"options": {
@@ -5105,7 +5223,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 74
+ "y": 82
},
"id": 139,
"options": {
@@ -5196,7 +5314,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 74
+ "y": 82
},
"id": 141,
"options": {
@@ -5287,7 +5405,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 83
+ "y": 91
},
"id": 142,
"options": {
@@ -5378,7 +5496,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 83
+ "y": 91
},
"id": 143,
"options": {
@@ -5469,7 +5587,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 92
+ "y": 100
},
"id": 144,
"options": {
@@ -5560,7 +5678,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 92
+ "y": 100
},
"id": 145,
"options": {
@@ -5651,7 +5769,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 101
+ "y": 109
},
"id": 146,
"options": {
@@ -5742,7 +5860,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 101
+ "y": 109
},
"id": 147,
"options": {
@@ -5833,7 +5951,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 110
+ "y": 118
},
"id": 148,
"options": {
@@ -5924,7 +6042,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 110
+ "y": 118
},
"id": 149,
"options": {
@@ -6015,7 +6133,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 119
+ "y": 127
},
"id": 150,
"options": {
@@ -6106,7 +6224,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 119
+ "y": 127
},
"id": 151,
"options": {
@@ -6196,7 +6314,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 128
+ "y": 136
},
"id": 153,
"options": {
@@ -6286,7 +6404,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 128
+ "y": 136
},
"id": 154,
"options": {
@@ -6376,7 +6494,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 136
+ "y": 144
},
"id": 155,
"options": {
@@ -6480,7 +6598,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 89
+ "y": 97
},
"id": 112,
"options": {
@@ -6571,7 +6689,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 89
+ "y": 97
},
"id": 116,
"options": {
@@ -6677,7 +6795,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 28
+ "y": 36
},
"id": 125,
"options": {
@@ -6770,7 +6888,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 28
+ "y": 36
},
"id": 126,
"options": {
@@ -6863,7 +6981,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 36
+ "y": 44
},
"id": 163,
"options": {
@@ -6956,7 +7074,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 36
+ "y": 44
},
"id": 162,
"options": {
@@ -7049,7 +7167,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 44
+ "y": 52
},
"id": 127,
"options": {
@@ -7155,7 +7273,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 29
+ "y": 37
},
"id": 174,
"options": {
@@ -7247,7 +7365,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 29
+ "y": 37
},
"id": 176,
"options": {
@@ -7338,7 +7456,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 38
+ "y": 46
},
"id": 175,
"options": {
@@ -7430,7 +7548,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 38
+ "y": 46
},
"id": 177,
"options": {
@@ -7465,7 +7583,7 @@
"type": "row"
}
],
- "refresh": "5m",
+ "refresh": "5s",
"revision": 1,
"schemaVersion": 38,
"style": "dark",
@@ -7474,13 +7592,13 @@
"list": []
},
"time": {
- "from": "now-1h",
+ "from": "now-5m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Celeborn",
"uid": "U_qgru_7z",
- "version": 3,
+ "version": 1,
"weekStart": ""
}
\ No newline at end of file
diff --git a/common/src/main/java/org/apache/celeborn/common/meta/FileInfo.java
b/common/src/main/java/org/apache/celeborn/common/meta/FileInfo.java
index 3b9a540d2..5193ffac8 100644
--- a/common/src/main/java/org/apache/celeborn/common/meta/FileInfo.java
+++ b/common/src/main/java/org/apache/celeborn/common/meta/FileInfo.java
@@ -232,4 +232,8 @@ public class FileInfo {
public void setMountPoint(String mountPoint) {
this.mountPoint = mountPoint;
}
+
+ public long getBytesFlushed() {
+ return bytesFlushed;
+ }
}
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index c263f2222..5ae372c2f 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -297,6 +297,9 @@ private[celeborn] class Worker(
workerSource.addGauge(WorkerSource.READ_BUFFER_ALLOCATED_COUNT) { () =>
memoryManager.getAllocatedReadBuffers
}
+ workerSource.addGauge(WorkerSource.ACTIVE_SHUFFLE_SIZE) { () =>
+ storageManager.getActiveShuffleSize()
+ }
private def highWorkload: Boolean = {
(memoryManager.currentServingState, conf.workerActiveConnectionMax) match {
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
index e1f247a08..5690548a3 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
@@ -127,4 +127,7 @@ object WorkerSource {
val POTENTIAL_CONSUME_SPEED = "PotentialConsumeSpeed"
val USER_PRODUCE_SPEED = "UserProduceSpeed"
val WORKER_CONSUME_SPEED = "WorkerConsumeSpeed"
+
+ // active shuffle size
+ val ACTIVE_SHUFFLE_SIZE = "ActiveShuffleSize"
}
diff --git
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index 355c77e15..b4c4a4985 100644
---
a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++
b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -796,6 +796,11 @@ final private[worker] class StorageManager(conf:
CelebornConf, workerSource: Abs
fileInfo: FileInfo): Unit = {
committedFileInfos.computeIfAbsent(shuffleKey, newMapFunc).put(fileName,
fileInfo)
}
+
+ def getActiveShuffleSize(): Long = {
+
fileInfos.values().asScala.map(_.values().asScala.map(_.getBytesFlushed).sum).sum
+ }
+
}
object StorageManager {
