This is an automated email from the ASF dual-hosted git repository. nicholasjiang pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/celeborn.git
commit e8622e1e8f3e6b17fa87fb5f45e22d89e432bc6c Author: TheodoreLx <[email protected]> AuthorDate: Mon Jul 21 16:40:57 2025 +0800 [CELEBORN-2061] Introduce metrics to count the amount of data flushed into different storage types Added metrics for the amount of data written to different storage types, including Local, HDFS, OSS, and S3 Currently, there is a lack of data volume written to each storage, and it is impossible to monitor the size and speed of writing. no Cluster Test Closes #3362 from TheodoreLx/add-flush-count-metric. Authored-by: TheodoreLx <[email protected] > Signed-off-by: SteNicholas <[email protected]> --- assets/grafana/celeborn-dashboard.json | 2591 +++++++++++++------- .../service/deploy/worker/WorkerSource.scala | 17 + .../service/deploy/worker/storage/FlushTask.scala | 29 +- .../service/deploy/worker/storage/TierWriter.scala | 6 +- 4 files changed, 1755 insertions(+), 888 deletions(-) diff --git a/assets/grafana/celeborn-dashboard.json b/assets/grafana/celeborn-dashboard.json index ff921f9a0..0ce5d6576 100644 --- a/assets/grafana/celeborn-dashboard.json +++ b/assets/grafana/celeborn-dashboard.json @@ -130,7 +130,7 @@ "h": 9, "w": 12, "x": 0, - "y": 49 + "y": 139 }, "id": 94, "options": { @@ -221,7 +221,7 @@ "h": 9, "w": 12, "x": 12, - "y": 49 + "y": 139 }, "id": 2, "options": { @@ -313,7 +313,7 @@ "h": 9, "w": 12, "x": 0, - "y": 58 + "y": 148 }, "id": 185, "options": { @@ -405,7 +405,7 @@ "h": 9, "w": 12, "x": 12, - "y": 58 + "y": 148 }, "id": 186, "options": { @@ -496,7 +496,7 @@ "h": 9, "w": 12, "x": 0, - "y": 67 + "y": 157 }, "id": 95, "options": { @@ -602,7 +602,7 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 140 }, "id": 121, "options": { @@ -696,7 +696,7 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 140 }, "id": 120, "options": { @@ -789,7 +789,7 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 148 }, "id": 189, "options": { @@ -881,7 +881,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 148 }, "id": 124, "options": { @@ -975,7 +975,7 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 156 }, "id": 122, "options": { @@ -1068,7 +1068,7 @@ "h": 8, "w": 12, "x": 12, - "y": 66 + "y": 156 }, "id": 102, "options": { @@ -1159,7 +1159,7 @@ "h": 8, "w": 12, "x": 0, - "y": 74 + "y": 164 }, "id": 100, "options": { @@ -1250,7 +1250,7 @@ "h": 8, "w": 12, "x": 12, - "y": 74 + "y": 164 }, "id": 117, "options": { @@ -1343,7 +1343,7 @@ "h": 8, "w": 12, "x": 0, - "y": 82 + "y": 172 }, "id": 194, "options": { @@ -1380,7 +1380,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The count of workers in lost list.", + "description": "The total count of shuffle including celeborn shuffle and engine built-in shuffle.", "fieldConfig": { "defaults": { "color": { @@ -1435,10 +1435,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 82 + "x": 0, + "y": 172 }, - "id": 36, + "id": 218, "options": { "legend": { "calcs": [], @@ -1458,13 +1458,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_LostWorkerCount_Value{instance=~\"${instance}\"}", + "expr": "metrics_ShuffleTotalCount_Value{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_LostWorkerCount_Value", + "title": "metrics_ShuffleTotalCount_Value", "type": "timeseries" }, { @@ -1472,7 +1472,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The total count of shuffle including celeborn shuffle and engine built-in shuffle.", + "description": "The count of workers in lost list.", "fieldConfig": { "defaults": { "color": { @@ -1527,10 +1527,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 82 + "x": 12, + "y": 172 }, - "id": 218, + "id": 36, "options": { "legend": { "calcs": [], @@ -1550,13 +1550,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_ShuffleTotalCount_Value{instance=~\"${instance}\"}", + "expr": "metrics_LostWorkerCount_Value{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_ShuffleTotalCount_Value", + "title": "metrics_LostWorkerCount_Value", "type": "timeseries" }, { @@ -1616,7 +1616,7 @@ "h": 8, "w": 12, "x": 12, - "y": 82 + "y": 172 }, "id": 219, "options": { @@ -1708,7 +1708,7 @@ "h": 8, "w": 12, "x": 0, - "y": 90 + "y": 180 }, "id": 239, "options": { @@ -1796,7 +1796,7 @@ "h": 8, "w": 12, "x": 12, - "y": 90 + "y": 180 }, "id": 240, "options": { @@ -1888,7 +1888,7 @@ "h": 8, "w": 12, "x": 0, - "y": 98 + "y": 188 }, "id": 118, "options": { @@ -1966,8 +1966,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1982,7 +1981,7 @@ "h": 8, "w": 12, "x": 12, - "y": 98 + "y": 188 }, "id": 136, "options": { @@ -2088,7 +2087,7 @@ "h": 10, "w": 24, "x": 0, - "y": 51 + "y": 141 }, "id": 84, "options": { @@ -2179,7 +2178,7 @@ "h": 9, "w": 12, "x": 0, - "y": 61 + "y": 151 }, "id": 60, "options": { @@ -2270,7 +2269,7 @@ "h": 9, "w": 12, "x": 12, - "y": 61 + "y": 151 }, "id": 62, "options": { @@ -2362,7 +2361,7 @@ "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 160 }, "id": 90, "options": { @@ -2395,6 +2394,7 @@ }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -2404,11 +2404,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2417,6 +2419,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2452,10 +2455,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 70 + "x": 0, + "y": 160 }, - "id": 92, + "id": 238, "options": { "legend": { "calcs": [], @@ -2464,7 +2467,6 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } @@ -2475,13 +2477,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_PausePushDataAndReplicate_Value{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", + "editorMode": "builder", + "expr": "metrics_UnreleasedPartitionLocationCount_Value{role=\"Worker\", instance=~\"${instance}\"}", + "instant": false, "range": true, "refId": "A" } ], - "title": "metrics_PausePushDataAndReplicate_Value", + "title": "metrics_UnreleasedPartitionLocationCount_Value", "type": "timeseries" }, { @@ -2536,18 +2539,17 @@ "value": 80 } ] - }, - "unit": "ms" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 78 + "x": 12, + "y": 160 }, - "id": 182, + "id": 92, "options": { "legend": { "calcs": [], @@ -2567,17 +2569,18 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_PausePushDataTime_Value{instance=~\"${instance}\"}", + "expr": "metrics_PausePushDataAndReplicate_Value{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_PausePushDataTime_Value", + "title": "metrics_PausePushDataAndReplicate_Value", "type": "timeseries" }, { "datasource": { + "default": false, "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -2587,11 +2590,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2600,6 +2605,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2628,8 +2634,7 @@ "value": 80 } ] - }, - "unit": "ms" + } }, "overrides": [] }, @@ -2637,9 +2642,9 @@ "h": 8, "w": 12, "x": 12, - "y": 78 + "y": 160 }, - "id": 184, + "id": 241, "options": { "legend": { "calcs": [], @@ -2648,7 +2653,6 @@ "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } @@ -2659,13 +2663,12 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_PausePushDataAndReplicateTime_Value{instance=~\"${instance}\"}", + "expr": "metrics_RegisterWithMasterFailCount_Count{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", - "range": true, "refId": "A" } ], - "title": "metrics_PausePushDataAndReplicateTime_Value", + "title": "metrics_RegisterWithMasterFailCount_Count", "type": "timeseries" }, { @@ -2673,7 +2676,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The active shuffle size of a worker including master replica and slave replica.", "fieldConfig": { "defaults": { "color": { @@ -2722,7 +2724,7 @@ } ] }, - "unit": "decbytes" + "unit": "ms" }, "overrides": [] }, @@ -2730,9 +2732,9 @@ "h": 8, "w": 12, "x": 0, - "y": 86 + "y": 168 }, - "id": 181, + "id": 182, "options": { "legend": { "calcs": [], @@ -2752,15 +2754,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "metrics_ActiveShuffleSize_Value{role=\"Worker\", instance=~\"${instance}\"}", - "instant": false, + "expr": "metrics_PausePushDataTime_Value{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_ActiveShuffleSize_Value", + "title": "metrics_PausePushDataTime_Value", "type": "timeseries" }, { @@ -2768,7 +2768,6 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "The active shuffle file count of a worker including master replica and slave replica.", "fieldConfig": { "defaults": { "color": { @@ -2816,7 +2815,8 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, @@ -2824,9 +2824,9 @@ "h": 8, "w": 12, "x": 12, - "y": 86 + "y": 168 }, - "id": 183, + "id": 184, "options": { "legend": { "calcs": [], @@ -2846,15 +2846,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "metrics_ActiveShuffleFileCount_Value{role=\"Worker\", instance=~\"${instance}\"}", - "instant": false, + "expr": "metrics_PausePushDataAndReplicateTime_Value{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_ActiveShuffleFileCount_Value", + "title": "metrics_PausePushDataAndReplicateTime_Value", "type": "timeseries" }, { @@ -2862,6 +2860,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "The active shuffle size of a worker including master replica and slave replica.", "fieldConfig": { "defaults": { "color": { @@ -2909,7 +2908,8 @@ "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, @@ -2917,9 +2917,9 @@ "h": 8, "w": 12, "x": 0, - "y": 94 + "y": 176 }, - "id": 179, + "id": 181, "options": { "legend": { "calcs": [], @@ -2939,13 +2939,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_ActiveConnectionCount_Count{instance=~\"${instance}\"}", + "editorMode": "builder", + "expr": "metrics_ActiveShuffleSize_Value{role=\"Worker\", instance=~\"${instance}\"}", + "instant": false, "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_ActiveConnectionCount_Count", + "title": "metrics_ActiveShuffleSize_Value", "type": "timeseries" }, { @@ -2953,6 +2955,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "The active shuffle file count of a worker including master replica and slave replica.", "fieldConfig": { "defaults": { "color": { @@ -3008,9 +3011,9 @@ "h": 8, "w": 12, "x": 12, - "y": 94 + "y": 176 }, - "id": 48, + "id": 183, "options": { "legend": { "calcs": [], @@ -3030,13 +3033,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_ActiveSlotsCount_Value{instance=~\"${instance}\"}", + "editorMode": "builder", + "expr": "metrics_ActiveShuffleFileCount_Value{role=\"Worker\", instance=~\"${instance}\"}", + "instant": false, "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_ActiveSlotsCount_Value", + "title": "metrics_ActiveShuffleFileCount_Value", "type": "timeseries" }, { @@ -3099,9 +3104,9 @@ "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 184 }, - "id": 193, + "id": 179, "options": { "legend": { "calcs": [], @@ -3121,19 +3126,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "metrics_FlushWorkingQueueSize_Value{instance=~\"${instance}\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, + "expr": "metrics_ActiveConnectionCount_Count{instance=~\"${instance}\"}", "legendFormat": "${baseLegend}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "metrics_FlushWorkingQueueSize_Value", + "title": "metrics_ActiveConnectionCount_Count", "type": "timeseries" }, { @@ -3196,9 +3195,9 @@ "h": 8, "w": 12, "x": 12, - "y": 102 + "y": 184 }, - "id": 195, + "id": 48, "options": { "legend": { "calcs": [], @@ -3207,6 +3206,7 @@ "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -3217,14 +3217,13 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "metrics_UnreleasedShuffleCount_Value{instance=~\"${instance}\"}", - "instant": false, + "expr": "metrics_ActiveSlotsCount_Value{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_UnreleasedShuffleCount_Value", + "title": "metrics_ActiveSlotsCount_Value", "type": "timeseries" }, { @@ -3287,9 +3286,9 @@ "h": 8, "w": 12, "x": 0, - "y": 110 + "y": 192 }, - "id": 217, + "id": 193, "options": { "legend": { "calcs": [], @@ -3298,6 +3297,7 @@ "showLegend": true }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -3308,20 +3308,23 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "metrics_IsDecommissioningWorker_Value{instance=~\"${instance}\"}", + "expr": "metrics_FlushWorkingQueueSize_Value{instance=~\"${instance}\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, "legendFormat": "${baseLegend}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "metrics_IsDecommissioningWorker_Value", + "title": "metrics_FlushWorkingQueueSize_Value", "type": "timeseries" }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, @@ -3331,13 +3334,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3346,7 +3347,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3383,9 +3383,9 @@ "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 192 }, - "id": 241, + "id": 195, "options": { "legend": { "calcs": [], @@ -3404,12 +3404,14 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "metrics_RegisterWithMasterFailCount_Count{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", + "editorMode": "builder", + "expr": "metrics_UnreleasedShuffleCount_Value{instance=~\"${instance}\"}", + "instant": false, + "range": true, "refId": "A" } ], - "title": "metrics_RegisterWithMasterFailCount_Count", + "title": "metrics_UnreleasedShuffleCount_Value", "type": "timeseries" }, { @@ -3417,20 +3419,17 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "mean size of partition files in bytes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -3439,7 +3438,6 @@ "tooltip": false, "viz": false }, - "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -3461,26 +3459,24 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "decbytes" + } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 110 + "x": 0, + "y": 200 }, - "id": 235, + "id": 217, "options": { "legend": { "calcs": [], @@ -3499,16 +3495,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "code", - "expr": "metrics_PartitionFileSizeBytes_Mean{role=\"Worker\", instance=~\"${instance}\"}", - "hide": false, + "editorMode": "builder", + "expr": "metrics_IsDecommissioningWorker_Value{instance=~\"${instance}\"}", "instant": false, "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_PartitionFileSizeBytes_Mean", + "title": "metrics_IsDecommissioningWorker_Value", "type": "timeseries" }, { @@ -3516,7 +3511,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "p99 size of partition files in size", + "description": "mean size of partition files in bytes", "fieldConfig": { "defaults": { "color": { @@ -3560,8 +3555,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3576,10 +3570,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 118 + "x": 12, + "y": 200 }, - "id": 236, + "id": 235, "options": { "legend": { "calcs": [], @@ -3599,7 +3593,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "metrics_PartitionFileSizeBytes_99thPercentile{role=\"Worker\", instance=~\"${instance}\"}", + "expr": "metrics_PartitionFileSizeBytes_Mean{role=\"Worker\", instance=~\"${instance}\"}", "hide": false, "instant": false, "legendFormat": "${baseLegend}", @@ -3607,7 +3601,7 @@ "refId": "A" } ], - "title": "metrics_PartitionFileSizeBytes_99thPercentile", + "title": "metrics_PartitionFileSizeBytes_Mean", "type": "timeseries" }, { @@ -3615,7 +3609,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "max size of partition files in size", + "description": "p99 size of partition files in size", "fieldConfig": { "defaults": { "color": { @@ -3659,8 +3653,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3675,10 +3668,10 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 118 + "x": 0, + "y": 208 }, - "id": 237, + "id": 236, "options": { "legend": { "calcs": [], @@ -3698,7 +3691,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "metrics_PartitionFileSizeBytes_Max{role=\"Worker\", instance=~\"${instance}\"}", + "expr": "metrics_PartitionFileSizeBytes_99thPercentile{role=\"Worker\", instance=~\"${instance}\"}", "hide": false, "instant": false, "legendFormat": "${baseLegend}", @@ -3706,15 +3699,15 @@ "refId": "A" } ], - "title": "metrics_PartitionFileSizeBytes_Max", + "title": "metrics_PartitionFileSizeBytes_99thPercentile", "type": "timeseries" }, { "datasource": { - "default": false, "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "description": "max size of partition files in size", "fieldConfig": { "defaults": { "color": { @@ -3758,25 +3751,25 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, - "x": 0, - "y": 70 + "x": 12, + "y": 208 }, - "id": 238, + "id": 237, "options": { "legend": { "calcs": [], @@ -3795,14 +3788,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editorMode": "builder", - "expr": "metrics_UnreleasedPartitionLocationCount_Value{role=\"Worker\", instance=~\"${instance}\"}", + "editorMode": "code", + "expr": "metrics_PartitionFileSizeBytes_Max{role=\"Worker\", instance=~\"${instance}\"}", + "hide": false, "instant": false, + "legendFormat": "${baseLegend}", "range": true, "refId": "A" } ], - "title": "metrics_UnreleasedPartitionLocationCount_Value", + "title": "metrics_PartitionFileSizeBytes_Max", "type": "timeseries" } ], @@ -3880,7 +3875,7 @@ "h": 9, "w": 12, "x": 0, - "y": 52 + "y": 142 }, "id": 68, "options": { @@ -3970,7 +3965,7 @@ "h": 9, "w": 12, "x": 12, - "y": 52 + "y": 142 }, "id": 70, "options": { @@ -4060,7 +4055,7 @@ "h": 9, "w": 12, "x": 0, - "y": 61 + "y": 151 }, "id": 72, "options": { @@ -4150,7 +4145,7 @@ "h": 9, "w": 12, "x": 12, - "y": 61 + "y": 151 }, "id": 74, "options": { @@ -4239,7 +4234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 160 }, "id": 83, "options": { @@ -4330,7 +4325,7 @@ "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 160 }, "id": 76, "options": { @@ -4421,7 +4416,7 @@ "h": 8, "w": 12, "x": 0, - "y": 78 + "y": 168 }, "id": 128, "options": { @@ -4512,7 +4507,7 @@ "h": 8, "w": 12, "x": 12, - "y": 78 + "y": 168 }, "id": 129, "options": { @@ -4603,7 +4598,7 @@ "h": 8, "w": 12, "x": 0, - "y": 86 + "y": 176 }, "id": 130, "options": { @@ -4694,7 +4689,7 @@ "h": 8, "w": 12, "x": 12, - "y": 86 + "y": 176 }, "id": 132, "options": { @@ -4785,7 +4780,7 @@ "h": 8, "w": 12, "x": 0, - "y": 94 + "y": 184 }, "id": 131, "options": { @@ -4876,7 +4871,7 @@ "h": 8, "w": 12, "x": 12, - "y": 94 + "y": 184 }, "id": 133, "options": { @@ -4967,7 +4962,7 @@ "h": 8, "w": 12, "x": 0, - "y": 102 + "y": 192 }, "id": 79, "options": { @@ -5059,8 +5054,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5152,8 +5146,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5245,8 +5238,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5262,7 +5254,7 @@ "h": 8, "w": 12, "x": 0, - "y": 13 + "y": 103 }, "id": 17, "options": { @@ -5338,8 +5330,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5355,7 +5346,7 @@ "h": 8, "w": 12, "x": 12, - "y": 13 + "y": 103 }, "id": 18, "options": { @@ -5431,8 +5422,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5447,7 +5437,7 @@ "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 111 }, "id": 81, "options": { @@ -5525,8 +5515,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5541,7 +5530,7 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 111 }, "id": 77, "options": { @@ -5619,8 +5608,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5635,7 +5623,7 @@ "h": 8, "w": 12, "x": 0, - "y": 29 + "y": 119 }, "id": 82, "options": { @@ -5713,8 +5701,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5729,7 +5716,7 @@ "h": 8, "w": 12, "x": 12, - "y": 29 + "y": 119 }, "id": 75, "options": { @@ -5807,8 +5794,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5823,7 +5809,7 @@ "h": 8, "w": 12, "x": 0, - "y": 37 + "y": 127 }, "id": 73, "options": { @@ -5901,8 +5887,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5917,7 +5902,7 @@ "h": 8, "w": 12, "x": 12, - "y": 37 + "y": 127 }, "id": 220, "options": { @@ -6000,8 +5985,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6017,7 +6001,7 @@ "h": 8, "w": 12, "x": 0, - "y": 45 + "y": 135 }, "id": 222, "options": { @@ -6100,8 +6084,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6117,7 +6100,7 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 135 }, "id": 221, "options": { @@ -6200,8 +6183,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6216,7 +6198,7 @@ "h": 8, "w": 12, "x": 0, - "y": 53 + "y": 143 }, "id": 223, "options": { @@ -6299,8 +6281,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6316,7 +6297,7 @@ "h": 8, "w": 12, "x": 12, - "y": 53 + "y": 143 }, "id": 224, "options": { @@ -6399,8 +6380,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6416,7 +6396,7 @@ "h": 8, "w": 12, "x": 0, - "y": 61 + "y": 151 }, "id": 225, "options": { @@ -6457,7 +6437,7 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, @@ -6465,639 +6445,1460 @@ "y": 5 }, "id": 10, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "panels": [], + "title": "FlushDataRelatives", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "metrics_TakeBufferTime_Mean{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" + } + ], + "title": "metrics_TakeBufferTime_Mean", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "metrics_TakeBufferTime_Max{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" + } + ], + "title": "metrics_TakeBufferTime_Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "metrics_FlushDataTime_Mean{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" + } + ], + "title": "metrics_FlushDataTime_Mean", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "metrics_FlushDataTime_Max{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" + } + ], + "title": "metrics_FlushDataTime_Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 243, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "metrics_LocalFlushCount_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_LocalFlushCount_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "unit": "ms" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 244, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "metrics_LocalFlushSize_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_LocalFlushSize_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 54 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 245, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 78, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "expr": "metrics_HdfsFlushCount_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_HdfsFlushCount_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 246, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "metrics_HdfsFlushSize_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_HdfsFlushSize_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_TakeBufferTime_Mean{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_TakeBufferTime_Mean", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 247, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 54 + "editorMode": "code", + "expr": "metrics_OssFlushSize_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_OssFlushSize_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 80, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_TakeBufferTime_Max{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_TakeBufferTime_Max", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + { + "color": "red", + "value": 80 + } + ] }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 248, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 62 + "editorMode": "code", + "expr": "metrics_OssFlushSize_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_OssFlushSize_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_FlushDataTime_Mean{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_FlushDataTime_Mean", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 249, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 62 + "editorMode": "code", + "expr": "metrics_S3FlushSize_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_S3FlushSize_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_FlushDataTime_Max{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_FlushDataTime_Max", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 250, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 71 + "editorMode": "code", + "expr": "metrics_S3FlushSize_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "range": true, + "refId": "A" + } + ], + "title": "metrics_S3FlushSize_Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 56, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_CommitFilesTime_Mean{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_CommitFilesTime_Mean", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 71 + "expr": "metrics_CommitFilesTime_Mean{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" + } + ], + "title": "metrics_CommitFilesTime_Mean", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 58, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_CommitFilesTime_Max{instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_CommitFilesTime_Max", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 71 + "expr": "metrics_CommitFilesTime_Max{instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" + } + ], + "title": "metrics_CommitFilesTime_Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 242, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "expr": "metrics_CommitFilesFailCount_Count{role=\"Worker\", instance=~\"${instance}\"}", - "legendFormat": "${baseLegend}", - "refId": "A" - } - ], - "title": "metrics_CommitFilesFailCount_Count", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 242, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.1.0-91295", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "metrics_CommitFilesFailCount_Count{role=\"Worker\", instance=~\"${instance}\"}", + "legendFormat": "${baseLegend}", + "refId": "A" } ], - "title": "FlushDataRelatives", - "type": "row" + "title": "metrics_CommitFilesFailCount_Count", + "type": "timeseries" }, { "collapsed": true, @@ -7105,7 +7906,7 @@ "h": 1, "w": 24, "x": 0, - "y": 6 + "y": 71 }, "id": 8, "panels": [ @@ -7170,7 +7971,7 @@ "h": 8, "w": 12, "x": 0, - "y": 55 + "y": 145 }, "id": 19, "options": { @@ -7263,7 +8064,7 @@ "h": 8, "w": 12, "x": 12, - "y": 55 + "y": 145 }, "id": 190, "options": { @@ -7357,7 +8158,7 @@ "h": 8, "w": 12, "x": 0, - "y": 63 + "y": 153 }, "id": 191, "options": { @@ -7450,7 +8251,7 @@ "h": 8, "w": 12, "x": 12, - "y": 63 + "y": 153 }, "id": 188, "options": { @@ -7544,7 +8345,7 @@ "h": 8, "w": 12, "x": 0, - "y": 71 + "y": 161 }, "id": 20, "options": { @@ -7634,7 +8435,7 @@ "h": 8, "w": 12, "x": 12, - "y": 71 + "y": 161 }, "id": 187, "options": { @@ -7727,7 +8528,7 @@ "h": 8, "w": 12, "x": 0, - "y": 79 + "y": 169 }, "id": 165, "options": { @@ -7819,7 +8620,7 @@ "h": 8, "w": 12, "x": 12, - "y": 79 + "y": 169 }, "id": 166, "options": { @@ -7906,7 +8707,7 @@ "h": 8, "w": 12, "x": 0, - "y": 87 + "y": 177 }, "id": 158, "options": { @@ -7993,7 +8794,7 @@ "h": 8, "w": 12, "x": 12, - "y": 87 + "y": 177 }, "id": 164, "options": { @@ -8080,7 +8881,7 @@ "h": 8, "w": 12, "x": 0, - "y": 95 + "y": 185 }, "id": 201, "options": { @@ -8167,7 +8968,7 @@ "h": 8, "w": 12, "x": 12, - "y": 95 + "y": 185 }, "id": 202, "options": { @@ -8254,7 +9055,7 @@ "h": 8, "w": 12, "x": 0, - "y": 103 + "y": 193 }, "id": 171, "options": { @@ -8341,7 +9142,7 @@ "h": 8, "w": 12, "x": 12, - "y": 103 + "y": 193 }, "id": 173, "options": { @@ -8428,7 +9229,7 @@ "h": 8, "w": 12, "x": 0, - "y": 111 + "y": 201 }, "id": 178, "options": { @@ -8520,7 +9321,7 @@ "h": 8, "w": 12, "x": 12, - "y": 111 + "y": 201 }, "id": 167, "options": { @@ -8612,7 +9413,7 @@ "h": 8, "w": 12, "x": 0, - "y": 119 + "y": 209 }, "id": 168, "options": { @@ -8699,7 +9500,7 @@ "h": 8, "w": 12, "x": 12, - "y": 119 + "y": 209 }, "id": 203, "options": { @@ -8786,7 +9587,7 @@ "h": 8, "w": 12, "x": 0, - "y": 127 + "y": 217 }, "id": 204, "options": { @@ -8873,7 +9674,7 @@ "h": 8, "w": 12, "x": 12, - "y": 127 + "y": 217 }, "id": 205, "options": { @@ -8960,7 +9761,7 @@ "h": 8, "w": 12, "x": 0, - "y": 135 + "y": 225 }, "id": 206, "options": { @@ -9047,7 +9848,7 @@ "h": 8, "w": 12, "x": 12, - "y": 135 + "y": 225 }, "id": 207, "options": { @@ -9134,7 +9935,7 @@ "h": 8, "w": 12, "x": 0, - "y": 143 + "y": 233 }, "id": 208, "options": { @@ -9221,7 +10022,7 @@ "h": 8, "w": 12, "x": 12, - "y": 143 + "y": 233 }, "id": 209, "options": { @@ -9313,7 +10114,7 @@ "h": 8, "w": 12, "x": 0, - "y": 151 + "y": 241 }, "id": 169, "options": { @@ -9405,7 +10206,7 @@ "h": 8, "w": 12, "x": 12, - "y": 151 + "y": 241 }, "id": 170, "options": { @@ -9492,7 +10293,7 @@ "h": 8, "w": 12, "x": 0, - "y": 159 + "y": 249 }, "id": 210, "options": { @@ -9579,7 +10380,7 @@ "h": 8, "w": 12, "x": 12, - "y": 159 + "y": 249 }, "id": 211, "options": { @@ -9666,7 +10467,7 @@ "h": 8, "w": 12, "x": 0, - "y": 167 + "y": 257 }, "id": 212, "options": { @@ -9753,7 +10554,7 @@ "h": 8, "w": 12, "x": 12, - "y": 167 + "y": 257 }, "id": 213, "options": { @@ -9840,7 +10641,7 @@ "h": 8, "w": 12, "x": 0, - "y": 175 + "y": 265 }, "id": 214, "options": { @@ -9927,7 +10728,7 @@ "h": 8, "w": 12, "x": 12, - "y": 175 + "y": 265 }, "id": 215, "options": { @@ -10014,7 +10815,7 @@ "h": 8, "w": 12, "x": 0, - "y": 183 + "y": 273 }, "id": 216, "options": { @@ -10105,7 +10906,7 @@ "h": 8, "w": 12, "x": 12, - "y": 183 + "y": 273 }, "id": 108, "options": { @@ -10197,7 +10998,7 @@ "h": 8, "w": 12, "x": 0, - "y": 191 + "y": 281 }, "id": 104, "options": { @@ -10288,7 +11089,7 @@ "h": 8, "w": 12, "x": 12, - "y": 191 + "y": 281 }, "id": 106, "options": { @@ -10329,7 +11130,7 @@ "h": 1, "w": 24, "x": 0, - "y": 7 + "y": 72 }, "id": 50, "panels": [], @@ -10347,11 +11148,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10360,6 +11163,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10382,7 +11186,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "red", @@ -10398,7 +11202,7 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 73 }, "id": 44, "options": { @@ -10409,10 +11213,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10444,6 +11250,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10452,6 +11259,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10473,7 +11281,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10489,7 +11298,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 73 }, "id": 46, "options": { @@ -10500,10 +11309,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10535,6 +11346,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10565,7 +11377,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10580,7 +11393,7 @@ "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 81 }, "id": 192, "options": { @@ -10591,10 +11404,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10627,6 +11442,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10635,6 +11451,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10656,7 +11473,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10671,7 +11489,7 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 81 }, "id": 180, "options": { @@ -10682,10 +11500,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10712,11 +11532,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10725,6 +11547,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10746,7 +11569,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10762,7 +11586,7 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 89 }, "id": 88, "options": { @@ -10773,10 +11597,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10803,11 +11629,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10816,6 +11644,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -10837,7 +11666,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": 0 }, { "color": "red", @@ -10852,7 +11682,7 @@ "h": 8, "w": 12, "x": 12, - "y": 24 + "y": 89 }, "id": 135, "options": { @@ -10863,10 +11693,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10900,6 +11732,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -10931,7 +11764,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "red", @@ -10946,7 +11779,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 97 }, "id": 234, "options": { @@ -10957,10 +11790,12 @@ "showLegend": true }, "tooltip": { + "hideZeros": false, "mode": "single", "sort": "none" } }, + "pluginVersion": "12.1.0-91295", "targets": [ { "datasource": { @@ -10983,7 +11818,7 @@ "h": 1, "w": 24, "x": 0, - "y": 40 + "y": 105 }, "id": 157, "panels": [ @@ -11049,7 +11884,7 @@ "h": 8, "w": 12, "x": 0, - "y": 958 + "y": 1048 }, "id": 159, "options": { @@ -11142,7 +11977,7 @@ "h": 8, "w": 12, "x": 12, - "y": 958 + "y": 1048 }, "id": 160, "options": { @@ -11235,7 +12070,7 @@ "h": 8, "w": 12, "x": 0, - "y": 966 + "y": 1056 }, "id": 161, "options": { @@ -11276,7 +12111,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 106 }, "id": 137, "panels": [ @@ -11341,7 +12176,7 @@ "h": 9, "w": 12, "x": 0, - "y": 641 + "y": 731 }, "id": 139, "options": { @@ -11433,7 +12268,7 @@ "h": 9, "w": 12, "x": 12, - "y": 641 + "y": 731 }, "id": 141, "options": { @@ -11525,7 +12360,7 @@ "h": 9, "w": 12, "x": 0, - "y": 714 + "y": 804 }, "id": 142, "options": { @@ -11617,7 +12452,7 @@ "h": 9, "w": 12, "x": 12, - "y": 714 + "y": 804 }, "id": 143, "options": { @@ -11709,7 +12544,7 @@ "h": 9, "w": 12, "x": 0, - "y": 723 + "y": 813 }, "id": 144, "options": { @@ -11801,7 +12636,7 @@ "h": 9, "w": 12, "x": 12, - "y": 723 + "y": 813 }, "id": 145, "options": { @@ -11893,7 +12728,7 @@ "h": 9, "w": 12, "x": 0, - "y": 732 + "y": 822 }, "id": 146, "options": { @@ -11985,7 +12820,7 @@ "h": 9, "w": 12, "x": 12, - "y": 732 + "y": 822 }, "id": 147, "options": { @@ -12077,7 +12912,7 @@ "h": 9, "w": 12, "x": 0, - "y": 741 + "y": 831 }, "id": 148, "options": { @@ -12169,7 +13004,7 @@ "h": 9, "w": 12, "x": 12, - "y": 741 + "y": 831 }, "id": 149, "options": { @@ -12261,7 +13096,7 @@ "h": 9, "w": 12, "x": 0, - "y": 750 + "y": 840 }, "id": 150, "options": { @@ -12353,7 +13188,7 @@ "h": 9, "w": 12, "x": 12, - "y": 750 + "y": 840 }, "id": 151, "options": { @@ -12444,7 +13279,7 @@ "h": 8, "w": 12, "x": 0, - "y": 759 + "y": 849 }, "id": 153, "options": { @@ -12535,7 +13370,7 @@ "h": 8, "w": 12, "x": 12, - "y": 759 + "y": 849 }, "id": 154, "options": { @@ -12626,7 +13461,7 @@ "h": 8, "w": 12, "x": 0, - "y": 767 + "y": 857 }, "id": 155, "options": { @@ -12717,7 +13552,7 @@ "h": 8, "w": 12, "x": 12, - "y": 767 + "y": 857 }, "id": 200, "options": { @@ -12809,7 +13644,7 @@ "h": 9, "w": 12, "x": 0, - "y": 775 + "y": 865 }, "id": 198, "options": { @@ -12901,7 +13736,7 @@ "h": 9, "w": 12, "x": 12, - "y": 775 + "y": 865 }, "id": 199, "options": { @@ -12993,7 +13828,7 @@ "h": 9, "w": 12, "x": 0, - "y": 784 + "y": 874 }, "id": 196, "options": { @@ -13085,7 +13920,7 @@ "h": 9, "w": 12, "x": 12, - "y": 784 + "y": 874 }, "id": 197, "options": { @@ -13126,7 +13961,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 107 }, "id": 110, "panels": [ @@ -13190,7 +14025,7 @@ "h": 8, "w": 12, "x": 0, - "y": 1046 + "y": 1136 }, "id": 112, "options": { @@ -13281,7 +14116,7 @@ "h": 8, "w": 12, "x": 12, - "y": 1046 + "y": 1136 }, "id": 116, "options": { @@ -13322,7 +14157,7 @@ "h": 1, "w": 24, "x": 0, - "y": 43 + "y": 108 }, "id": 123, "panels": [ @@ -13387,7 +14222,7 @@ "h": 8, "w": 12, "x": 0, - "y": 1055 + "y": 1145 }, "id": 125, "options": { @@ -13480,7 +14315,7 @@ "h": 8, "w": 12, "x": 12, - "y": 1055 + "y": 1145 }, "id": 126, "options": { @@ -13573,7 +14408,7 @@ "h": 8, "w": 12, "x": 0, - "y": 1063 + "y": 1153 }, "id": 163, "options": { @@ -13666,7 +14501,7 @@ "h": 8, "w": 12, "x": 12, - "y": 1063 + "y": 1153 }, "id": 162, "options": { @@ -13759,7 +14594,7 @@ "h": 8, "w": 12, "x": 0, - "y": 1071 + "y": 1161 }, "id": 127, "options": { @@ -13800,7 +14635,7 @@ "h": 1, "w": 24, "x": 0, - "y": 44 + "y": 109 }, "id": 172, "panels": [ @@ -13865,7 +14700,7 @@ "h": 9, "w": 12, "x": 0, - "y": 1080 + "y": 1170 }, "id": 174, "options": { @@ -13958,7 +14793,7 @@ "h": 9, "w": 12, "x": 12, - "y": 1080 + "y": 1170 }, "id": 176, "options": { @@ -14050,7 +14885,7 @@ "h": 9, "w": 12, "x": 0, - "y": 1089 + "y": 1179 }, "id": 175, "options": { @@ -14143,7 +14978,7 @@ "h": 9, "w": 12, "x": 12, - "y": 1089 + "y": 1179 }, "id": 177, "options": { @@ -14184,7 +15019,7 @@ "h": 1, "w": 24, "x": 0, - "y": 45 + "y": 110 }, "id": 226, "panels": [ @@ -14433,7 +15268,7 @@ "h": 8, "w": 12, "x": 0, - "y": 9 + "y": 99 }, "id": 229, "options": { @@ -14524,7 +15359,7 @@ "h": 8, "w": 12, "x": 12, - "y": 9 + "y": 99 }, "id": 230, "options": { @@ -14615,7 +15450,7 @@ "h": 8, "w": 12, "x": 0, - "y": 17 + "y": 107 }, "id": 231, "options": { @@ -14707,7 +15542,7 @@ "h": 8, "w": 12, "x": 12, - "y": 17 + "y": 107 }, "id": 232, "options": { @@ -14798,7 +15633,7 @@ "h": 8, "w": 12, "x": 0, - "y": 25 + "y": 115 }, "id": 233, "options": { @@ -14833,24 +15668,24 @@ "type": "row" } ], + "preload": false, "refresh": "5s", - "revision": 1, - "schemaVersion": 39, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { - "current": {}, + "current": { + "text": "", + "value": "" + }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(metrics_JVMCPUTime_Value, instance)", - "hide": 0, "includeAll": true, "label": "instance", - "mapping": "", - "mappingOnLegend": true, "multi": true, "name": "instance", "options": [], @@ -14860,21 +15695,17 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": { - "selected": false, "text": "__auto", "value": "__auto" }, "description": "The base legend format to apply for all metrics", - "hide": 0, "includeAll": false, "label": "baseLegend", - "multi": false, "name": "baseLegend", "options": [ { @@ -14884,8 +15715,6 @@ } ], "query": "__auto", - "queryValue": "", - "skipUrlSync": false, "type": "custom" } ] diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala index 56a8ea94c..f4990d650 100644 --- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala +++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala @@ -63,6 +63,15 @@ class WorkerSource(conf: CelebornConf) extends AbstractSource(conf, Role.WORKER) addCounter(COMMIT_FILES_FAIL_COUNT) + addCounter(LOCAL_FLUSH_COUNT) + addCounter(LOCAL_FLUSH_SIZE) + addCounter(HDFS_FLUSH_COUNT) + addCounter(HDFS_FLUSH_SIZE) + addCounter(OSS_FLUSH_COUNT) + addCounter(OSS_FLUSH_SIZE) + addCounter(S3_FLUSH_COUNT) + addCounter(S3_FLUSH_SIZE) + // add timers addTimer(COMMIT_FILES_TIME) addTimer(RESERVE_SLOTS_TIME) @@ -195,6 +204,14 @@ object WorkerSource { val COMMIT_FILES_TIME = "CommitFilesTime" val COMMIT_FILES_FAIL_COUNT = "CommitFilesFailCount" val FLUSH_WORKING_QUEUE_SIZE = "FlushWorkingQueueSize" + val LOCAL_FLUSH_COUNT = "LocalFlushCount" + val LOCAL_FLUSH_SIZE = "LocalFlushSize" + val HDFS_FLUSH_COUNT = "HdfsFlushCount" + val HDFS_FLUSH_SIZE = "HdfsFlushSize" + val OSS_FLUSH_COUNT = "OssFlushCount" + val OSS_FLUSH_SIZE = "OssFlushSize" + val S3_FLUSH_COUNT = "S3FlushCount" + val S3_FLUSH_SIZE = "S3FlushSize" // slots val SLOTS_ALLOCATED = "SlotsAllocated" diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/FlushTask.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/FlushTask.scala index 16a11c088..e9c7f7490 100644 --- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/FlushTask.scala +++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/FlushTask.scala @@ -23,13 +23,16 @@ import java.nio.channels.FileChannel import io.netty.buffer.{ByteBufUtil, CompositeByteBuf} import org.apache.hadoop.fs.Path +import org.apache.celeborn.common.metrics.source.AbstractSource import org.apache.celeborn.common.protocol.StorageInfo.Type import org.apache.celeborn.server.common.service.mpu.MultipartUploadHandler +import org.apache.celeborn.service.deploy.worker.WorkerSource abstract private[worker] class FlushTask( val buffer: CompositeByteBuf, val notifier: FlushNotifier, - val keepBuffer: Boolean) { + val keepBuffer: Boolean, + val source: AbstractSource) { def flush(): Unit } @@ -38,8 +41,10 @@ private[worker] class LocalFlushTask( fileChannel: FileChannel, notifier: FlushNotifier, keepBuffer: Boolean, - gatherApiEnabled: Boolean) extends FlushTask(buffer, notifier, keepBuffer) { + source: AbstractSource, + gatherApiEnabled: Boolean) extends FlushTask(buffer, notifier, keepBuffer, source) { override def flush(): Unit = { + val readableBytes = buffer.readableBytes() val buffers = buffer.nioBuffers() if (gatherApiEnabled) { val readableBytes = buffer.readableBytes() @@ -54,6 +59,8 @@ private[worker] class LocalFlushTask( } } } + source.incCounter(WorkerSource.LOCAL_FLUSH_COUNT) + source.incCounter(WorkerSource.LOCAL_FLUSH_SIZE, readableBytes) // TODO: force flush file channel in scenarios where the upstream task writes and the downstream task reads simultaneously, such as flink hybrid shuffle. } } @@ -62,12 +69,16 @@ private[worker] class HdfsFlushTask( buffer: CompositeByteBuf, val path: Path, notifier: FlushNotifier, - keepBuffer: Boolean) extends FlushTask(buffer, notifier, keepBuffer) { + keepBuffer: Boolean, + source: AbstractSource) extends FlushTask(buffer, notifier, keepBuffer, source) { override def flush(): Unit = { + val readableBytes = buffer.readableBytes() val hadoopFs = StorageManager.hadoopFs.get(Type.HDFS) val hdfsStream = hadoopFs.append(path, 256 * 1024) hdfsStream.write(ByteBufUtil.getBytes(buffer)) hdfsStream.close() + source.incCounter(WorkerSource.HDFS_FLUSH_COUNT) + source.incCounter(WorkerSource.HDFS_FLUSH_SIZE, readableBytes) } } @@ -75,15 +86,19 @@ private[worker] class S3FlushTask( buffer: CompositeByteBuf, notifier: FlushNotifier, keepBuffer: Boolean, + source: AbstractSource, s3MultipartUploader: MultipartUploadHandler, partNumber: Int, finalFlush: Boolean = false) - extends FlushTask(buffer, notifier, keepBuffer) { + extends FlushTask(buffer, notifier, keepBuffer, source) { override def flush(): Unit = { + val readableBytes = buffer.readableBytes() val bytes = ByteBufUtil.getBytes(buffer) val inputStream = new ByteArrayInputStream(bytes) s3MultipartUploader.putPart(inputStream, partNumber, finalFlush) + source.incCounter(WorkerSource.S3_FLUSH_COUNT) + source.incCounter(WorkerSource.S3_FLUSH_SIZE, readableBytes) } } @@ -91,14 +106,18 @@ private[worker] class OssFlushTask( buffer: CompositeByteBuf, notifier: FlushNotifier, keepBuffer: Boolean, + source: AbstractSource, ossMultipartUploader: MultipartUploadHandler, partNumber: Int, finalFlush: Boolean = false) - extends FlushTask(buffer, notifier, keepBuffer) { + extends FlushTask(buffer, notifier, keepBuffer, source) { override def flush(): Unit = { + val readableBytes = buffer.readableBytes() val bytes = ByteBufUtil.getBytes(buffer) val inputStream = new ByteArrayInputStream(bytes) ossMultipartUploader.putPart(inputStream, partNumber, finalFlush) + source.incCounter(WorkerSource.OSS_FLUSH_COUNT) + source.incCounter(WorkerSource.OSS_FLUSH_SIZE, readableBytes) } } diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/TierWriter.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/TierWriter.scala index 00dd90bd5..7c3f1b966 100644 --- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/TierWriter.scala +++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/TierWriter.scala @@ -407,7 +407,7 @@ class LocalTierWriter( override def genFlushTask(finalFlush: Boolean, keepBuffer: Boolean): FlushTask = { notifier.numPendingFlushes.incrementAndGet() - new LocalFlushTask(flushBuffer, channel, notifier, true, gatherApiEnabled) + new LocalFlushTask(flushBuffer, channel, notifier, true, source, gatherApiEnabled) } override def writeInternal(buf: ByteBuf): Unit = { @@ -587,12 +587,13 @@ class DfsTierWriter( override def genFlushTask(finalFlush: Boolean, keepBuffer: Boolean): FlushTask = { notifier.numPendingFlushes.incrementAndGet() if (hdfsFileInfo.isHdfs) { - new HdfsFlushTask(flushBuffer, hdfsFileInfo.getDfsPath(), notifier, true) + new HdfsFlushTask(flushBuffer, hdfsFileInfo.getDfsPath(), notifier, true, source) } else if (hdfsFileInfo.isOSS) { val flushTask = new OssFlushTask( flushBuffer, notifier, true, + source, ossMultipartUploadHandler, partNumber, finalFlush) @@ -603,6 +604,7 @@ class DfsTierWriter( flushBuffer, notifier, true, + source, s3MultipartUploadHandler, partNumber, finalFlush)
