This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 860c9c3cb [VL] Add wallnanos for WriteFiles (#6976)
860c9c3cb is described below
commit 860c9c3cb6883c779467dc004c7bc88736a9396a
Author: Yang Zhang <[email protected]>
AuthorDate: Fri Aug 23 13:47:26 2024 +0800
[VL] Add wallnanos for WriteFiles (#6976)
---
.../scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala | 5 ++++-
.../scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
index 49d45a396..88aa8c90d 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxMetricsApi.scala
@@ -338,7 +338,10 @@ class VeloxMetricsApi extends MetricsApi with Logging {
def genWriteFilesTransformerMetrics(sparkContext: SparkContext): Map[String,
SQLMetric] =
Map(
- "physicalWrittenBytes" -> SQLMetrics.createMetric(sparkContext, "number
of written bytes"),
+ "physicalWrittenBytes" -> SQLMetrics.createSizeMetric(
+ sparkContext,
+ "number of written bytes"),
+ "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext, "time of
write"),
"numWrittenFiles" -> SQLMetrics.createMetric(sparkContext, "number of
written files")
)
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala
b/gluten-data/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala
index 5a04b4043..30ed4aaa7 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala
@@ -24,6 +24,7 @@ class WriteFilesMetricsUpdater(val metrics: Map[String,
SQLMetric]) extends Metr
if (opMetrics != null) {
val operatorMetrics = opMetrics.asInstanceOf[OperatorMetrics]
metrics("physicalWrittenBytes") += operatorMetrics.physicalWrittenBytes
+ metrics("wallNanos") += operatorMetrics.wallNanos
metrics("numWrittenFiles") += operatorMetrics.numWrittenFiles
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
