This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c873a640c3 [GLUTEN-9003][CORE] Carry native counter counts into SQL
metrics (#13358)
c873a640c3 is described below
commit c873a640c31413eae297d7d27f57df1cfa724741
Author: Mohammad Linjawi <[email protected]>
AuthorDate: Thu Nov 12 14:18:45 2026 +0300
[GLUTEN-9003][CORE] Carry native counter counts into SQL metrics (#13358)
---
.../org/apache/gluten/metrics/OperatorMetrics.java | 3 +
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 1 +
.../gluten/metrics/FilterMetricsUpdater.scala | 2 +
.../org/apache/gluten/metrics/MetricsUtil.scala | 24 ++++++-
.../gluten/metrics/ProjectMetricsUpdater.scala | 5 ++
.../apache/gluten/metrics/MetricsUtilSuite.scala | 69 ++++++++++++++++++++
.../metrics/ProjectMetricsUpdaterSuite.scala | 75 ++++++++++++++++++++++
.../gluten/backendsapi/BackendSettingsApi.scala | 7 ++
.../gluten/expression/IncrementMetricCall.scala | 71 ++++++++++++++++++++
.../apache/gluten/expression/ExpressionNames.scala | 1 +
10 files changed, 257 insertions(+), 1 deletion(-)
diff --git
a/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
b/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
index 8e906ca494..9d06a88cb3 100644
---
a/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
+++
b/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
@@ -17,6 +17,9 @@
package org.apache.gluten.metrics;
public class OperatorMetrics implements IOperatorMetrics {
+ /** Rows processed per Delta counter slot function, keyed by its name; see
IncrementMetricCall. */
+ public java.util.Map<String, Long> expressionStats = new
java.util.HashMap<>();
+
public long inputRows;
public long inputVectors;
public long inputBytes;
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index ce107b6f74..ce136790e2 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -1418,6 +1418,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with
Logging {
Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG),
Sig[MapFilter](ExpressionNames.MAP_FILTER),
Sig[AssertNotNull](ExpressionNames.ASSERT_NOT_NULL),
+ Sig[IncrementMetricCall](ExpressionNames.INCREMENT_METRIC),
Sig[MonotonicallyIncreasingID](ExpressionNames.MONOTONICALLY_INCREASING_ID),
// For test purpose.
Sig[VeloxDummyExpression](VeloxDummyExpression.VELOX_DUMMY_EXPRESSION)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
b/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
index 03a5f604e5..9a07862c64 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
@@ -37,6 +37,8 @@ class FilterMetricsUpdater(
extraMetrics.foreach {
case (name, metric) =>
name match {
+ // Delta's filter counters never run natively (no
IncrementMetricCall): only a stack
+ // over a literal true is offloaded, and it keeps every row, so
its count is the output.
case "increment_metric" => metric += operatorMetrics.outputRows
case _ => // do nothing
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
b/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
index 689f7e7219..9ed102d5a2 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
@@ -51,8 +51,20 @@ object MetricsUtil extends Logging {
.map(_.asLong())
.getOrElse(0L)
+ private[metrics] def expressionStatsFromJson(node: JsonNode): JMap[String,
java.lang.Long] = {
+ val stats = new java.util.HashMap[String, java.lang.Long]()
+ Option(node.get("expressionStats")).foreach {
+ exprStats =>
+ exprStats.fields().asScala.foreach {
+ entry => stats.put(entry.getKey,
java.lang.Long.valueOf(entry.getValue.asLong()))
+ }
+ }
+ stats
+ }
+
private def operatorMetricFromJson(node: JsonNode): OperatorMetrics = {
val metrics = new OperatorMetrics()
+ metrics.expressionStats = expressionStatsFromJson(node)
metrics.inputRows = value(node, "inputRows")
metrics.inputVectors = value(node, "inputVectors")
metrics.inputBytes = value(node, "inputBytes")
@@ -207,7 +219,7 @@ object MetricsUtil extends Logging {
* @return
* the merged metrics
*/
- private def mergeMetrics(operatorMetrics: JList[OperatorMetrics]):
OperatorMetrics = {
+ private[metrics] def mergeMetrics(operatorMetrics: JList[OperatorMetrics]):
OperatorMetrics = {
if (operatorMetrics.size() == 0) {
return null
}
@@ -359,6 +371,16 @@ object MetricsUtil extends Logging {
aggregated.bloomFilterTestedRows = bloomFilterTestedRows
aggregated.bloomFilterAcceptedRows = bloomFilterAcceptedRows
aggregated.bloomFilterBypassed = bloomFilterBypassed
+ val expressionStats = new java.util.HashMap[String, java.lang.Long]()
+ operatorMetrics.asScala.foreach {
+ metrics =>
+ metrics.expressionStats.asScala.foreach {
+ case (name, rows) =>
+ val sum =
Option(expressionStats.get(name)).map(_.longValue()).getOrElse(0L) + rows
+ expressionStats.put(name, java.lang.Long.valueOf(sum))
+ }
+ }
+ aggregated.expressionStats = expressionStats
aggregated
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
b/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
index e297bf356f..f29ffff6e7 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
@@ -16,6 +16,8 @@
*/
package org.apache.gluten.metrics
+import org.apache.gluten.expression.IncrementMetricCall
+
import org.apache.spark.sql.execution.metric.SQLMetric
class ProjectMetricsUpdater(
@@ -38,6 +40,9 @@ class ProjectMetricsUpdater(
case (name, metric) =>
name match {
case "increment_metric" => metric += operatorMetrics.outputRows
+ case slot if
slot.startsWith(IncrementMetricCall.functionNamePrefix) =>
+ // The counter ran natively: credit the rows Velox evaluated it
on.
+ metric += operatorMetrics.expressionStats.getOrDefault(slot, 0L)
case _ => // do nothing
}
}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/metrics/MetricsUtilSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/metrics/MetricsUtilSuite.scala
new file mode 100644
index 0000000000..154334fc2a
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/gluten/metrics/MetricsUtilSuite.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.scalatest.funsuite.AnyFunSuite
+
+import java.util.{Arrays => JArrays}
+
+import scala.collection.JavaConverters._
+
+class MetricsUtilSuite extends AnyFunSuite {
+
+ private val mapper = new ObjectMapper()
+
+ private def expressionStats(json: String): Map[String, Long] =
+ MetricsUtil
+ .expressionStatsFromJson(mapper.readTree(json))
+ .asScala
+ .map { case (name, rows) => name -> rows.longValue() }
+ .toMap
+
+ private def operatorMetrics(expressionStats: (String, Long)*):
OperatorMetrics = {
+ val metrics = new OperatorMetrics()
+ expressionStats.foreach {
+ case (name, rows) => metrics.expressionStats.put(name,
java.lang.Long.valueOf(rows))
+ }
+ metrics
+ }
+
+ test("expression stats are read from an operator's metrics") {
+ val json =
+ """{"outputRows": 4, "expressionStats": {"increment_metric_0": 3,
"increment_metric_1": 0}}"""
+ assert(
+ expressionStats(json) === Map("increment_metric_0" -> 3L,
"increment_metric_1" -> 0L))
+ }
+
+ test("an operator without expression stats has none") {
+ assert(expressionStats("""{"outputRows": 4}""").isEmpty)
+ assert(expressionStats("""{"outputRows": 4, "expressionStats":
{}}""").isEmpty)
+ }
+
+ test("merging operator metrics sums the expression stats of each function") {
+ val merged = MetricsUtil.mergeMetrics(
+ JArrays.asList(
+ operatorMetrics("increment_metric_0" -> 3L, "increment_metric_1" ->
1L),
+ operatorMetrics("increment_metric_0" -> 2L),
+ operatorMetrics()))
+ assert(
+ merged.expressionStats.asScala.map { case (name, rows) => name ->
rows.longValue() } ===
+ Map("increment_metric_0" -> 5L, "increment_metric_1" -> 1L))
+ // A concrete map, like the one every OperatorMetrics starts with.
+ assert(merged.expressionStats.isInstanceOf[java.util.HashMap[_, _]])
+ }
+}
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/metrics/ProjectMetricsUpdaterSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/metrics/ProjectMetricsUpdaterSuite.scala
new file mode 100644
index 0000000000..fef873ddaa
--- /dev/null
+++
b/backends-velox/src/test/scala/org/apache/gluten/metrics/ProjectMetricsUpdaterSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics
+
+import org.apache.spark.sql.execution.metric.SQLMetric
+
+import org.scalatest.funsuite.AnyFunSuite
+
+class ProjectMetricsUpdaterSuite extends AnyFunSuite {
+
+ private def projectMetrics(): Map[String, SQLMetric] = Seq(
+ "numOutputRows",
+ "outputVectors",
+ "outputBytes",
+ "cpuCount",
+ "wallNanos",
+ "peakMemoryBytes",
+ "numMemoryAllocations",
+ "loadLazyVectorTime").map(_ -> new SQLMetric("sum")).toMap
+
+ private def operatorMetrics(
+ outputRows: Long,
+ expressionStats: (String, Long)*): OperatorMetrics = {
+ val metrics = new OperatorMetrics()
+ metrics.outputRows = outputRows
+ expressionStats.foreach {
+ case (name, rows) => metrics.expressionStats.put(name,
java.lang.Long.valueOf(rows))
+ }
+ metrics
+ }
+
+ test("a native counter is credited with the rows its function processed") {
+ val updated = new SQLMetric("sum")
+ val copied = new SQLMetric("sum")
+ val updater = new ProjectMetricsUpdater(
+ projectMetrics(),
+ Seq("increment_metric_0" -> updated, "increment_metric_1" -> copied))
+ updater.updateNativeMetrics(
+ operatorMetrics(
+ 10,
+ "increment_metric_0" -> 3L,
+ "increment_metric_1" -> 7L,
+ "increment_metric_2" -> 10L))
+ assert(updated.value === 3)
+ assert(copied.value === 7)
+ }
+
+ test("a native counter that never ran is credited with zero rows") {
+ val metric = new SQLMetric("sum")
+ val updater = new ProjectMetricsUpdater(projectMetrics(),
Seq("increment_metric_0" -> metric))
+ updater.updateNativeMetrics(operatorMetrics(10))
+ assert(metric.value === 0)
+ }
+
+ test("a counter removed from the native plan is credited with the output
rows") {
+ val metric = new SQLMetric("sum")
+ val updater = new ProjectMetricsUpdater(projectMetrics(),
Seq("increment_metric" -> metric))
+ updater.updateNativeMetrics(operatorMetrics(10, "increment_metric_0" ->
3L))
+ assert(metric.value === 10)
+ }
+}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
index a4c7d8c3b5..75444dd952 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
@@ -62,6 +62,13 @@ trait BackendSettingsApi {
def supportNativeRowIndexColumn(): Boolean = true
+ /**
+ * Whether the backend evaluates Delta's `IncrementMetric` natively through
+ * [[org.apache.gluten.expression.IncrementMetricCall]] and reports the
processed-row count of
+ * each counter function in its operator metrics.
+ */
+ def supportNativeIncrementMetric(): Boolean = false
+
def supportSortExec(): Boolean = false
def supportSortMergeJoinExec(): Boolean = true
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/IncrementMetricCall.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/IncrementMetricCall.scala
new file mode 100644
index 0000000000..35f29e6e4b
--- /dev/null
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/IncrementMetricCall.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.expression
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression,
Unevaluable}
+import org.apache.spark.sql.types.DataType
+
+/**
+ * A native pass-through call that makes the backend count how many rows
reached this point of an
+ * expression tree. Delta's project transformer emits it in place of
`IncrementMetric` when the
+ * backend supports native counting; filters never carry one. `functionName`
is one of the backend's
+ * fixed counter slots and is unique to one SQL metric within its projection:
the backend keys its
+ * per-expression statistics by function name, and the metrics updater credits
the metric from that
+ * entry. Never evaluated on Spark; it only exists in the expression list
handed to the backend.
+ *
+ * Gluten maps it by class (`ExpressionNames.INCREMENT_METRIC`, which is also
the name to put in the
+ * expression blacklist), and `getTransformer` sends `functionName` to the
backend. Its `prettyName`
+ * is the slot name, never `increment_metric`, which is how Gluten recognizes
Delta's own
+ * `IncrementMetric` without depending on Delta.
+ */
+case class IncrementMetricCall(child: Expression, functionName: String)
+ extends UnaryExpression
+ with Unevaluable
+ with Transformable {
+
+ override def dataType: DataType = child.dataType
+
+ override def nullable: Boolean = child.nullable
+
+ override def prettyName: String = functionName
+
+ override protected def withNewChildInternal(newChild: Expression):
IncrementMetricCall =
+ copy(child = newChild)
+
+ override def getTransformer(
+ childrenTransformers: Seq[ExpressionTransformer]): ExpressionTransformer
=
+ GenericExpressionTransformer(functionName, childrenTransformers, this)
+}
+
+object IncrementMetricCall {
+
+ /** Prefix of every counter slot name. */
+ val functionNamePrefix: String = "increment_metric_"
+
+ /**
+ * Number of counter slots the backend registers at startup
(`increment_metric_0` to
+ * `increment_metric_31`). Mirrors `kIncrementMetricFunctionSlots` in the
Velox backend. A
+ * projection with more distinct metrics than this falls back.
+ */
+ val maxCounters: Int = 32
+
+ /** Name of the counter slot `slot`, which must be below [[maxCounters]]. */
+ def functionName(slot: Int): String = {
+ require(slot >= 0 && slot < maxCounters, s"counter slot $slot out of
range")
+ functionNamePrefix + slot
+ }
+}
diff --git
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 50e7c0097f..d1d1896f91 100644
---
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -361,6 +361,7 @@ object ExpressionNames {
final val AT_LEAST_N_NON_NULLS = "at_least_n_non_nulls"
final val ASSERT_TRUE = "assert_true"
final val ASSERT_NOT_NULL = "assert_not_null"
+ final val INCREMENT_METRIC = "increment_metric"
final val NULLIF = "nullif"
final val NVL = "nvl"
final val NVL2 = "nvl2"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]