This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 939f929af [VL] Add more metrics for generate (#5608)
939f929af is described below
commit 939f929afe9db02e9a342fd8fddf0e43c359ebfc
Author: Xiduo You <[email protected]>
AuthorDate: Mon May 6 16:05:59 2024 +0800
[VL] Add more metrics for generate (#5608)
---
.../apache/gluten/execution/GenerateExecTransformer.scala | 12 +++++++++++-
.../org/apache/gluten/execution/VeloxMetricsSuite.scala | 15 +++++++++++++++
.../apache/gluten/metrics/GenerateMetricsUpdater.scala | 6 ++++++
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
index 08d2937b5..830fe396b 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/execution/GenerateExecTransformer.scala
@@ -51,7 +51,17 @@ case class GenerateExecTransformer(
@transient
override lazy val metrics =
- Map("numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of
output rows"))
+ Map(
+ "numOutputRows" -> SQLMetrics.createMetric(sparkContext, "number of
output rows"),
+ "numOutputVectors" -> SQLMetrics.createMetric(sparkContext, "number of
output vectors"),
+ "numOutputBytes" -> SQLMetrics.createSizeMetric(sparkContext, "number of
output bytes"),
+ "wallNanos" -> SQLMetrics.createNanoTimingMetric(sparkContext,
"totaltime of generate"),
+ "cpuCount" -> SQLMetrics.createMetric(sparkContext, "cpu wall time
count"),
+ "peakMemoryBytes" -> SQLMetrics.createSizeMetric(sparkContext, "peak
memory bytes"),
+ "numMemoryAllocations" -> SQLMetrics.createMetric(
+ sparkContext,
+ "number of memory allocations")
+ )
override def metricsUpdater(): MetricsUpdater = new
GenerateMetricsUpdater(metrics)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
index ac9911bba..ce8450fea 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxMetricsSuite.scala
@@ -128,6 +128,21 @@ class VeloxMetricsSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
}
+ test("Generate metrics") {
+ runQueryAndCompare("SELECT explode(array(c1, c2, 1)) FROM metrics_t1") {
+ df =>
+ val generate = find(df.queryExecution.executedPlan) {
+ case _: GenerateExecTransformer => true
+ case _ => false
+ }
+ assert(generate.isDefined)
+ val metrics = generate.get.metrics
+ assert(metrics("numOutputRows").value == 300)
+ assert(metrics("numOutputVectors").value > 0)
+ assert(metrics("numOutputBytes").value > 0)
+ }
+ }
+
test("Write metrics") {
if (SparkShimLoader.getSparkVersion.startsWith("3.4")) {
withSQLConf(("spark.gluten.sql.native.writer.enabled", "true")) {
diff --git
a/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
b/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
index 670fd1c4d..0a3dccd64 100644
---
a/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
+++
b/gluten-data/src/main/scala/org/apache/gluten/metrics/GenerateMetricsUpdater.scala
@@ -23,6 +23,12 @@ class GenerateMetricsUpdater(val metrics: Map[String,
SQLMetric]) extends Metric
if (operatorMetrics != null) {
val nativeMetrics = operatorMetrics.asInstanceOf[OperatorMetrics]
metrics("numOutputRows") += nativeMetrics.outputRows
+ metrics("numOutputVectors") += nativeMetrics.outputVectors
+ metrics("numOutputBytes") += nativeMetrics.outputBytes
+ metrics("cpuCount") += nativeMetrics.cpuCount
+ metrics("wallNanos") += nativeMetrics.wallNanos
+ metrics("peakMemoryBytes") += nativeMetrics.peakMemoryBytes
+ metrics("numMemoryAllocations") += nativeMetrics.numMemoryAllocations
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
