This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new eef47c5dda Add window spill metrics (#8777)
eef47c5dda is described below
commit eef47c5dda7b5406c0b58a5b5c749b2935591325
Author: Joey <[email protected]>
AuthorDate: Thu Feb 20 03:36:09 2025 +0800
Add window spill metrics (#8777)
Since Velox SortWindow already supports spill, we need to add the spill
metrics for window.
---
.../scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala | 6 +++++-
.../main/scala/org/apache/gluten/metrics/WindowMetricsUpdater.scala | 4 ++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
index bf92dac2dd..bbc9ec8268 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
@@ -336,7 +336,11 @@ class VeloxMetricsApi extends MetricsApi with Logging {
"peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak
memory bytes"),
"numMemoryAllocations" -> SQLMetrics.createMetric(
sparkContext,
- "number of memory allocations")
+ "number of memory allocations"),
+ "spilledBytes" -> SQLMetrics.createSizeMetric(sparkContext, "bytes
written for spilling"),
+ "spilledRows" -> SQLMetrics.createMetric(sparkContext, "total rows
written for spilling"),
+ "spilledPartitions" -> SQLMetrics.createMetric(sparkContext, "total
spilled partitions"),
+ "spilledFiles" -> SQLMetrics.createMetric(sparkContext, "total spilled
files")
)
override def genWindowTransformerMetricsUpdater(metrics: Map[String,
SQLMetric]): MetricsUpdater =
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/metrics/WindowMetricsUpdater.scala
b/backends-velox/src/main/scala/org/apache/gluten/metrics/WindowMetricsUpdater.scala
index 82a41e0103..4a098102b1 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/metrics/WindowMetricsUpdater.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/metrics/WindowMetricsUpdater.scala
@@ -30,6 +30,10 @@ class WindowMetricsUpdater(val metrics: Map[String,
SQLMetric]) extends MetricsU
metrics("wallNanos") += operatorMetrics.wallNanos
metrics("peakMemoryBytes") += operatorMetrics.peakMemoryBytes
metrics("numMemoryAllocations") += operatorMetrics.numMemoryAllocations
+ metrics("spilledBytes") += operatorMetrics.spilledBytes
+ metrics("spilledRows") += operatorMetrics.spilledRows
+ metrics("spilledPartitions") += operatorMetrics.spilledPartitions
+ metrics("spilledFiles") += operatorMetrics.spilledFiles
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
