This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 166dbc560441 [SPARK-57158][SQL] ExplainUtils: extract operator ID 
assignment phase from processPlan into a private helper
166dbc560441 is described below

commit 166dbc560441c0c39a8368a82c7b1d451ca71ea9
Author: Mark Jarvin <[email protected]>
AuthorDate: Thu Jul 30 17:08:55 2026 +0800

    [SPARK-57158][SQL] ExplainUtils: extract operator ID assignment phase from 
processPlan into a private helper
    
    ### What changes were proposed in this pull request?
    
    Extract the operator-ID-assignment phase of `ExplainUtils.processPlan` into 
a private `assignOperatorIds(plan, idMap)` helper. `processPlan` now 
initializes the idMap, delegates all ID assignment to `assignOperatorIds`, then 
performs the text-output pass over the returned subqueries and optimized-out 
exchanges.
    
    ### Why are the changes needed?
    
    `processPlan` conflated two independent sequential phases in one ~40-line 
body:
    
    1. **ID assignment** — traversing the plan tree, subqueries, and 
adaptively-optimized-out exchanges (SPARK-42753) to populate an 
`IdentityHashMap` with monotonically-increasing operator IDs.
    2. **Text output** — calling `processPlanSkippingSubqueries` on each 
discovered subtree to format the verbose explain string.
    
    These phases are sequential and independent: the text-output pass only 
begins after ID assignment is fully complete. Extracting phase 1 into 
`assignOperatorIds` shortens `processPlan` to its output logic and makes the 
boundary between the two phases explicit. No behavior change.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added `ExplainUtilsSuite` covering:
    - Operator IDs assigned to all visible plan nodes are unique
    - `processPlan` sets `CODEGEN_ID_TAG` on nodes inside 
`WholeStageCodegenExec`
    - Thread-local `localIdMap` is restored to its prior value after 
`processPlan` returns
    - Subquery section is emitted in the explain output when subqueries are 
present
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude (Anthropic)
    
    Closes #56216 from 
markj-db/mark-jarvin_data/explain-utils-generate-plan-ids.
    
    Authored-by: Mark Jarvin <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
---
 .../apache/spark/sql/execution/ExplainUtils.scala  | 62 ++++++++++--------
 .../spark/sql/execution/ExplainUtilsSuite.scala    | 76 ++++++++++++++++++++++
 2 files changed, 112 insertions(+), 26 deletions(-)

diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index 2e878c21dc7a..a81ddb1b1da1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -87,32 +87,7 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
       // overwrites and to allow intentional overwriting of IDs generated in 
previous AQE iteration
       val idMap = new IdentityHashMap[QueryPlan[_], Int]()
       localIdMap.set(idMap)
-      // Initialize an array of ReusedExchanges to help find Adaptively 
Optimized Out
-      // Exchanges as part of SPARK-42753
-      val reusedExchanges = ArrayBuffer.empty[ReusedExchangeExec]
-
-      var currentOperatorID = 0
-      currentOperatorID = generateOperatorIDs(plan, currentOperatorID, idMap, 
reusedExchanges,
-        true)
-
-      val subqueries = ArrayBuffer.empty[(SparkPlan, Expression, 
BaseSubqueryExec)]
-      getSubqueries(plan, subqueries)
-
-      currentOperatorID = subqueries.foldLeft(currentOperatorID) {
-        (curId, plan) => generateOperatorIDs(plan._3.child, curId, idMap, 
reusedExchanges,
-          true)
-      }
-
-      // SPARK-42753: Process subtree for a ReusedExchange with unknown child
-      val optimizedOutExchanges = ArrayBuffer.empty[Exchange]
-      reusedExchanges.foreach{ reused =>
-        val child = reused.child
-        if (!idMap.containsKey(child)) {
-          optimizedOutExchanges.append(child)
-          currentOperatorID = generateOperatorIDs(child, currentOperatorID, 
idMap,
-            reusedExchanges, false)
-        }
-      }
+      val (subqueries, optimizedOutExchanges) = assignOperatorIds(plan, idMap)
 
       val collectedOperators = BitSet.empty
       processPlanSkippingSubqueries(plan, append, collectedOperators)
@@ -150,6 +125,41 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
     }
   }
 
+  /**
+   * Assigns operator IDs to all operators across the full plan tree -- the 
main plan,
+   * any subqueries, and any adaptively-optimized-out exchanges (SPARK-42753) 
-- by populating
+   * the supplied idMap. Returns the discovered subqueries and optimized-out 
exchanges so
+   * the caller ([[processPlan]]) can perform post-assignment work (text 
output) without
+   * rediscovering them.
+   */
+  private def assignOperatorIds(
+      plan: QueryPlan[_],
+      idMap: java.util.Map[QueryPlan[_], Int])
+      : (ArrayBuffer[(SparkPlan, Expression, BaseSubqueryExec)], 
ArrayBuffer[Exchange]) = {
+    // Initialize an array of ReusedExchanges to help find Adaptively 
Optimized Out
+    // Exchanges as part of SPARK-42753
+    val reusedExchanges = ArrayBuffer.empty[ReusedExchangeExec]
+    var currentOperatorID = generateOperatorIDs(plan, 0, idMap, 
reusedExchanges, true)
+
+    val subqueries = ArrayBuffer.empty[(SparkPlan, Expression, 
BaseSubqueryExec)]
+    getSubqueries(plan, subqueries)
+    currentOperatorID = subqueries.foldLeft(currentOperatorID) {
+      (curId, sub) => generateOperatorIDs(sub._3.child, curId, idMap, 
reusedExchanges, true)
+    }
+
+    // SPARK-42753: Process subtree for a ReusedExchange with unknown child
+    val optimizedOutExchanges = ArrayBuffer.empty[Exchange]
+    reusedExchanges.foreach { reused =>
+      val child = reused.child
+      if (!idMap.containsKey(child)) {
+        optimizedOutExchanges.append(child)
+        currentOperatorID = generateOperatorIDs(child, currentOperatorID, 
idMap,
+          reusedExchanges, false)
+      }
+    }
+    (subqueries, optimizedOutExchanges)
+  }
+
   /**
    * Traverses the supplied input plan in a bottom-up fashion and records the 
operator id via
    * setting a tag in the operator.
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExplainUtilsSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExplainUtilsSuite.scala
new file mode 100644
index 000000000000..4830f5d9e78d
--- /dev/null
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExplainUtilsSuite.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.catalyst.plans.logical.Range
+import org.apache.spark.sql.catalyst.util.StringConcat
+import org.apache.spark.sql.test.SharedSparkSession
+
+/**
+ * Tests for [[ExplainUtils.processPlan]]: operator ID assignment, 
WholeStageCodegen tag
+ * propagation, thread-local lifecycle, and subquery handling.
+ */
+class ExplainUtilsSuite extends QueryTest with SharedSparkSession {
+
+  private def explainOutput(plan: SparkPlan): String = {
+    val concat = new StringConcat()
+    ExplainUtils.processPlan(plan, concat.append)
+    concat.toString
+  }
+
+  test("processPlan assigns unique operator IDs to all visible plan nodes") {
+    val df = spark.range(100).filter("id > 10").select("id")
+    val output = explainOutput(df.queryExecution.executedPlan)
+    // Each operator ID appears both in the tree header ("Filter (2)") and as 
the header of its
+    // verbose section ("(2) Filter"). Anchor to the verbose-section headers, 
which are the only
+    // lines that begin with "(N)", so each operator contributes exactly one 
ID.
+    val ids = 
"(?m)^\\((\\d+)\\)".r.findAllMatchIn(output).map(_.group(1).toInt).toSeq
+    assert(ids.nonEmpty, "processPlan should assign at least one operator ID")
+    assert(ids == ids.distinct, s"processPlan operator IDs should be unique: 
$ids")
+  }
+
+  test("processPlan tags a WholeStageCodegenExec child with its codegen stage 
id") {
+    // Build the plan directly so the assertion does not depend on the planner 
producing a
+    // WholeStageCodegenExec for any particular query. processPlan should 
stamp the child of a
+    // WholeStageCodegenExec with CODEGEN_ID_TAG carrying that node's 
codegenStageId.
+    val child = RangeExec(Range(0, 10, 1, 1))
+    val plan = WholeStageCodegenExec(child)(codegenStageId = 7)
+    ExplainUtils.processPlan[SparkPlan](plan, _ => ())
+    assert(child.getTagValue(QueryPlan.CODEGEN_ID_TAG).contains(7),
+      "processPlan should tag a WholeStageCodegenExec child with its 
codegenStageId")
+  }
+
+  test("processPlan restores localIdMap to its prior value after completion") {
+    val prev = ExplainUtils.localIdMap.get()
+    ExplainUtils.processPlan(spark.range(10).filter("id > 
3").queryExecution.executedPlan,
+      _ => ())
+    assert(ExplainUtils.localIdMap.get() eq prev,
+      "processPlan should restore the thread-local localIdMap on completion")
+  }
+
+  test("processPlan includes subquery output in the explain string") {
+    withSQLConf("spark.sql.adaptive.enabled" -> "false") {
+      val df = spark.range(100).filter("id > 0")
+        .filter("id < (SELECT max(id) FROM range(5))")
+      val output = explainOutput(df.queryExecution.executedPlan)
+      assert(output.contains("Subqueries"), "processPlan should emit a 
Subqueries section")
+    }
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to