This is an automated email from the ASF dual-hosted git repository.

zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new c873a640c3 [GLUTEN-9003][CORE] Carry native counter counts into SQL 
metrics (#13358)
c873a640c3 is described below

commit c873a640c31413eae297d7d27f57df1cfa724741
Author: Mohammad Linjawi <[email protected]>
AuthorDate: Thu Nov 12 14:18:45 2026 +0300

    [GLUTEN-9003][CORE] Carry native counter counts into SQL metrics (#13358)
---
 .../org/apache/gluten/metrics/OperatorMetrics.java |  3 +
 .../backendsapi/velox/VeloxSparkPlanExecApi.scala  |  1 +
 .../gluten/metrics/FilterMetricsUpdater.scala      |  2 +
 .../org/apache/gluten/metrics/MetricsUtil.scala    | 24 ++++++-
 .../gluten/metrics/ProjectMetricsUpdater.scala     |  5 ++
 .../apache/gluten/metrics/MetricsUtilSuite.scala   | 69 ++++++++++++++++++++
 .../metrics/ProjectMetricsUpdaterSuite.scala       | 75 ++++++++++++++++++++++
 .../gluten/backendsapi/BackendSettingsApi.scala    |  7 ++
 .../gluten/expression/IncrementMetricCall.scala    | 71 ++++++++++++++++++++
 .../apache/gluten/expression/ExpressionNames.scala |  1 +
 10 files changed, 257 insertions(+), 1 deletion(-)

diff --git 
a/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java 
b/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
index 8e906ca494..9d06a88cb3 100644
--- 
a/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
+++ 
b/backends-velox/src/main/java/org/apache/gluten/metrics/OperatorMetrics.java
@@ -17,6 +17,9 @@
 package org.apache.gluten.metrics;
 
 public class OperatorMetrics implements IOperatorMetrics {
+  /** Rows processed per Delta counter slot function, keyed by its name; see 
IncrementMetricCall. */
+  public java.util.Map<String, Long> expressionStats = new 
java.util.HashMap<>();
+
   public long inputRows;
   public long inputVectors;
   public long inputBytes;
diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
index ce107b6f74..ce136790e2 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -1418,6 +1418,7 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi with 
Logging {
       Sig[VeloxBloomFilterAggregate](ExpressionNames.BLOOM_FILTER_AGG),
       Sig[MapFilter](ExpressionNames.MAP_FILTER),
       Sig[AssertNotNull](ExpressionNames.ASSERT_NOT_NULL),
+      Sig[IncrementMetricCall](ExpressionNames.INCREMENT_METRIC),
       
Sig[MonotonicallyIncreasingID](ExpressionNames.MONOTONICALLY_INCREASING_ID),
       // For test purpose.
       Sig[VeloxDummyExpression](VeloxDummyExpression.VELOX_DUMMY_EXPRESSION)
diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
index 03a5f604e5..9a07862c64 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/metrics/FilterMetricsUpdater.scala
@@ -37,6 +37,8 @@ class FilterMetricsUpdater(
       extraMetrics.foreach {
         case (name, metric) =>
           name match {
+            // Delta's filter counters never run natively (no 
IncrementMetricCall): only a stack
+            // over a literal true is offloaded, and it keeps every row, so 
its count is the output.
             case "increment_metric" => metric += operatorMetrics.outputRows
             case _ => // do nothing
           }
diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala 
b/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
index 689f7e7219..9ed102d5a2 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/metrics/MetricsUtil.scala
@@ -51,8 +51,20 @@ object MetricsUtil extends Logging {
       .map(_.asLong())
       .getOrElse(0L)
 
+  private[metrics] def expressionStatsFromJson(node: JsonNode): JMap[String, 
java.lang.Long] = {
+    val stats = new java.util.HashMap[String, java.lang.Long]()
+    Option(node.get("expressionStats")).foreach {
+      exprStats =>
+        exprStats.fields().asScala.foreach {
+          entry => stats.put(entry.getKey, 
java.lang.Long.valueOf(entry.getValue.asLong()))
+        }
+    }
+    stats
+  }
+
   private def operatorMetricFromJson(node: JsonNode): OperatorMetrics = {
     val metrics = new OperatorMetrics()
+    metrics.expressionStats = expressionStatsFromJson(node)
     metrics.inputRows = value(node, "inputRows")
     metrics.inputVectors = value(node, "inputVectors")
     metrics.inputBytes = value(node, "inputBytes")
@@ -207,7 +219,7 @@ object MetricsUtil extends Logging {
    * @return
    *   the merged metrics
    */
-  private def mergeMetrics(operatorMetrics: JList[OperatorMetrics]): 
OperatorMetrics = {
+  private[metrics] def mergeMetrics(operatorMetrics: JList[OperatorMetrics]): 
OperatorMetrics = {
     if (operatorMetrics.size() == 0) {
       return null
     }
@@ -359,6 +371,16 @@ object MetricsUtil extends Logging {
     aggregated.bloomFilterTestedRows = bloomFilterTestedRows
     aggregated.bloomFilterAcceptedRows = bloomFilterAcceptedRows
     aggregated.bloomFilterBypassed = bloomFilterBypassed
+    val expressionStats = new java.util.HashMap[String, java.lang.Long]()
+    operatorMetrics.asScala.foreach {
+      metrics =>
+        metrics.expressionStats.asScala.foreach {
+          case (name, rows) =>
+            val sum = 
Option(expressionStats.get(name)).map(_.longValue()).getOrElse(0L) + rows
+            expressionStats.put(name, java.lang.Long.valueOf(sum))
+        }
+    }
+    aggregated.expressionStats = expressionStats
     aggregated
   }
 
diff --git 
a/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
 
b/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
index e297bf356f..f29ffff6e7 100644
--- 
a/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
+++ 
b/backends-velox/src/main/scala/org/apache/gluten/metrics/ProjectMetricsUpdater.scala
@@ -16,6 +16,8 @@
  */
 package org.apache.gluten.metrics
 
+import org.apache.gluten.expression.IncrementMetricCall
+
 import org.apache.spark.sql.execution.metric.SQLMetric
 
 class ProjectMetricsUpdater(
@@ -38,6 +40,9 @@ class ProjectMetricsUpdater(
         case (name, metric) =>
           name match {
             case "increment_metric" => metric += operatorMetrics.outputRows
+            case slot if 
slot.startsWith(IncrementMetricCall.functionNamePrefix) =>
+              // The counter ran natively: credit the rows Velox evaluated it 
on.
+              metric += operatorMetrics.expressionStats.getOrDefault(slot, 0L)
             case _ => // do nothing
           }
       }
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/metrics/MetricsUtilSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/metrics/MetricsUtilSuite.scala
new file mode 100644
index 0000000000..154334fc2a
--- /dev/null
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/metrics/MetricsUtilSuite.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import org.scalatest.funsuite.AnyFunSuite
+
+import java.util.{Arrays => JArrays}
+
+import scala.collection.JavaConverters._
+
+class MetricsUtilSuite extends AnyFunSuite {
+
+  private val mapper = new ObjectMapper()
+
+  private def expressionStats(json: String): Map[String, Long] =
+    MetricsUtil
+      .expressionStatsFromJson(mapper.readTree(json))
+      .asScala
+      .map { case (name, rows) => name -> rows.longValue() }
+      .toMap
+
+  private def operatorMetrics(expressionStats: (String, Long)*): 
OperatorMetrics = {
+    val metrics = new OperatorMetrics()
+    expressionStats.foreach {
+      case (name, rows) => metrics.expressionStats.put(name, 
java.lang.Long.valueOf(rows))
+    }
+    metrics
+  }
+
+  test("expression stats are read from an operator's metrics") {
+    val json =
+      """{"outputRows": 4, "expressionStats": {"increment_metric_0": 3, 
"increment_metric_1": 0}}"""
+    assert(
+      expressionStats(json) === Map("increment_metric_0" -> 3L, 
"increment_metric_1" -> 0L))
+  }
+
+  test("an operator without expression stats has none") {
+    assert(expressionStats("""{"outputRows": 4}""").isEmpty)
+    assert(expressionStats("""{"outputRows": 4, "expressionStats": 
{}}""").isEmpty)
+  }
+
+  test("merging operator metrics sums the expression stats of each function") {
+    val merged = MetricsUtil.mergeMetrics(
+      JArrays.asList(
+        operatorMetrics("increment_metric_0" -> 3L, "increment_metric_1" -> 
1L),
+        operatorMetrics("increment_metric_0" -> 2L),
+        operatorMetrics()))
+    assert(
+      merged.expressionStats.asScala.map { case (name, rows) => name -> 
rows.longValue() } ===
+        Map("increment_metric_0" -> 5L, "increment_metric_1" -> 1L))
+    // A concrete map, like the one every OperatorMetrics starts with.
+    assert(merged.expressionStats.isInstanceOf[java.util.HashMap[_, _]])
+  }
+}
diff --git 
a/backends-velox/src/test/scala/org/apache/gluten/metrics/ProjectMetricsUpdaterSuite.scala
 
b/backends-velox/src/test/scala/org/apache/gluten/metrics/ProjectMetricsUpdaterSuite.scala
new file mode 100644
index 0000000000..fef873ddaa
--- /dev/null
+++ 
b/backends-velox/src/test/scala/org/apache/gluten/metrics/ProjectMetricsUpdaterSuite.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.metrics
+
+import org.apache.spark.sql.execution.metric.SQLMetric
+
+import org.scalatest.funsuite.AnyFunSuite
+
+class ProjectMetricsUpdaterSuite extends AnyFunSuite {
+
+  private def projectMetrics(): Map[String, SQLMetric] = Seq(
+    "numOutputRows",
+    "outputVectors",
+    "outputBytes",
+    "cpuCount",
+    "wallNanos",
+    "peakMemoryBytes",
+    "numMemoryAllocations",
+    "loadLazyVectorTime").map(_ -> new SQLMetric("sum")).toMap
+
+  private def operatorMetrics(
+      outputRows: Long,
+      expressionStats: (String, Long)*): OperatorMetrics = {
+    val metrics = new OperatorMetrics()
+    metrics.outputRows = outputRows
+    expressionStats.foreach {
+      case (name, rows) => metrics.expressionStats.put(name, 
java.lang.Long.valueOf(rows))
+    }
+    metrics
+  }
+
+  test("a native counter is credited with the rows its function processed") {
+    val updated = new SQLMetric("sum")
+    val copied = new SQLMetric("sum")
+    val updater = new ProjectMetricsUpdater(
+      projectMetrics(),
+      Seq("increment_metric_0" -> updated, "increment_metric_1" -> copied))
+    updater.updateNativeMetrics(
+      operatorMetrics(
+        10,
+        "increment_metric_0" -> 3L,
+        "increment_metric_1" -> 7L,
+        "increment_metric_2" -> 10L))
+    assert(updated.value === 3)
+    assert(copied.value === 7)
+  }
+
+  test("a native counter that never ran is credited with zero rows") {
+    val metric = new SQLMetric("sum")
+    val updater = new ProjectMetricsUpdater(projectMetrics(), 
Seq("increment_metric_0" -> metric))
+    updater.updateNativeMetrics(operatorMetrics(10))
+    assert(metric.value === 0)
+  }
+
+  test("a counter removed from the native plan is credited with the output 
rows") {
+    val metric = new SQLMetric("sum")
+    val updater = new ProjectMetricsUpdater(projectMetrics(), 
Seq("increment_metric" -> metric))
+    updater.updateNativeMetrics(operatorMetrics(10, "increment_metric_0" -> 
3L))
+    assert(metric.value === 10)
+  }
+}
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
index a4c7d8c3b5..75444dd952 100644
--- 
a/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala
@@ -62,6 +62,13 @@ trait BackendSettingsApi {
 
   def supportNativeRowIndexColumn(): Boolean = true
 
+  /**
+   * Whether the backend evaluates Delta's `IncrementMetric` natively through
+   * [[org.apache.gluten.expression.IncrementMetricCall]] and reports the 
processed-row count of
+   * each counter function in its operator metrics.
+   */
+  def supportNativeIncrementMetric(): Boolean = false
+
   def supportSortExec(): Boolean = false
 
   def supportSortMergeJoinExec(): Boolean = true
diff --git 
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/IncrementMetricCall.scala
 
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/IncrementMetricCall.scala
new file mode 100644
index 0000000000..35f29e6e4b
--- /dev/null
+++ 
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/IncrementMetricCall.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.expression
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, UnaryExpression, 
Unevaluable}
+import org.apache.spark.sql.types.DataType
+
+/**
+ * A native pass-through call that makes the backend count how many rows 
reached this point of an
+ * expression tree. Delta's project transformer emits it in place of 
`IncrementMetric` when the
+ * backend supports native counting; filters never carry one. `functionName` 
is one of the backend's
+ * fixed counter slots and is unique to one SQL metric within its projection: 
the backend keys its
+ * per-expression statistics by function name, and the metrics updater credits 
the metric from that
+ * entry. Never evaluated on Spark; it only exists in the expression list 
handed to the backend.
+ *
+ * Gluten maps it by class (`ExpressionNames.INCREMENT_METRIC`, which is also 
the name to put in the
+ * expression blacklist), and `getTransformer` sends `functionName` to the 
backend. Its `prettyName`
+ * is the slot name, never `increment_metric`, which is how Gluten recognizes 
Delta's own
+ * `IncrementMetric` without depending on Delta.
+ */
+case class IncrementMetricCall(child: Expression, functionName: String)
+  extends UnaryExpression
+  with Unevaluable
+  with Transformable {
+
+  override def dataType: DataType = child.dataType
+
+  override def nullable: Boolean = child.nullable
+
+  override def prettyName: String = functionName
+
+  override protected def withNewChildInternal(newChild: Expression): 
IncrementMetricCall =
+    copy(child = newChild)
+
+  override def getTransformer(
+      childrenTransformers: Seq[ExpressionTransformer]): ExpressionTransformer 
=
+    GenericExpressionTransformer(functionName, childrenTransformers, this)
+}
+
+object IncrementMetricCall {
+
+  /** Prefix of every counter slot name. */
+  val functionNamePrefix: String = "increment_metric_"
+
+  /**
+   * Number of counter slots the backend registers at startup 
(`increment_metric_0` to
+   * `increment_metric_31`). Mirrors `kIncrementMetricFunctionSlots` in the 
Velox backend. A
+   * projection with more distinct metrics than this falls back.
+   */
+  val maxCounters: Int = 32
+
+  /** Name of the counter slot `slot`, which must be below [[maxCounters]]. */
+  def functionName(slot: Int): String = {
+    require(slot >= 0 && slot < maxCounters, s"counter slot $slot out of 
range")
+    functionNamePrefix + slot
+  }
+}
diff --git 
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
 
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
index 50e7c0097f..d1d1896f91 100644
--- 
a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
+++ 
b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala
@@ -361,6 +361,7 @@ object ExpressionNames {
   final val AT_LEAST_N_NON_NULLS = "at_least_n_non_nulls"
   final val ASSERT_TRUE = "assert_true"
   final val ASSERT_NOT_NULL = "assert_not_null"
+  final val INCREMENT_METRIC = "increment_metric"
   final val NULLIF = "nullif"
   final val NVL = "nvl"
   final val NVL2 = "nvl2"


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to