This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 166dbc560441 [SPARK-57158][SQL] ExplainUtils: extract operator ID
assignment phase from processPlan into a private helper
166dbc560441 is described below
commit 166dbc560441c0c39a8368a82c7b1d451ca71ea9
Author: Mark Jarvin <[email protected]>
AuthorDate: Thu Jul 30 17:08:55 2026 +0800
[SPARK-57158][SQL] ExplainUtils: extract operator ID assignment phase from
processPlan into a private helper
### What changes were proposed in this pull request?
Extract the operator-ID-assignment phase of `ExplainUtils.processPlan` into
a private `assignOperatorIds(plan, idMap)` helper. `processPlan` now
initializes the idMap, delegates all ID assignment to `assignOperatorIds`, then
performs the text-output pass over the returned subqueries and optimized-out
exchanges.
### Why are the changes needed?
`processPlan` conflated two independent sequential phases in one ~40-line
body:
1. **ID assignment** — traversing the plan tree, subqueries, and
adaptively-optimized-out exchanges (SPARK-42753) to populate an
`IdentityHashMap` with monotonically-increasing operator IDs.
2. **Text output** — calling `processPlanSkippingSubqueries` on each
discovered subtree to format the verbose explain string.
These phases are sequential and independent: the text-output pass only
begins after ID assignment is fully complete. Extracting phase 1 into
`assignOperatorIds` shortens `processPlan` to its output logic and makes the
boundary between the two phases explicit. No behavior change.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added `ExplainUtilsSuite` covering:
- Operator IDs assigned to all visible plan nodes are unique
- `processPlan` sets `CODEGEN_ID_TAG` on nodes inside
`WholeStageCodegenExec`
- Thread-local `localIdMap` is restored to its prior value after
`processPlan` returns
- Subquery section is emitted in the explain output when subqueries are
present
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude (Anthropic)
Closes #56216 from
markj-db/mark-jarvin_data/explain-utils-generate-plan-ids.
Authored-by: Mark Jarvin <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
---
.../apache/spark/sql/execution/ExplainUtils.scala | 62 ++++++++++--------
.../spark/sql/execution/ExplainUtilsSuite.scala | 76 ++++++++++++++++++++++
2 files changed, 112 insertions(+), 26 deletions(-)
diff --git
a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
index 2e878c21dc7a..a81ddb1b1da1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/ExplainUtils.scala
@@ -87,32 +87,7 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
// overwrites and to allow intentional overwriting of IDs generated in
previous AQE iteration
val idMap = new IdentityHashMap[QueryPlan[_], Int]()
localIdMap.set(idMap)
- // Initialize an array of ReusedExchanges to help find Adaptively
Optimized Out
- // Exchanges as part of SPARK-42753
- val reusedExchanges = ArrayBuffer.empty[ReusedExchangeExec]
-
- var currentOperatorID = 0
- currentOperatorID = generateOperatorIDs(plan, currentOperatorID, idMap,
reusedExchanges,
- true)
-
- val subqueries = ArrayBuffer.empty[(SparkPlan, Expression,
BaseSubqueryExec)]
- getSubqueries(plan, subqueries)
-
- currentOperatorID = subqueries.foldLeft(currentOperatorID) {
- (curId, plan) => generateOperatorIDs(plan._3.child, curId, idMap,
reusedExchanges,
- true)
- }
-
- // SPARK-42753: Process subtree for a ReusedExchange with unknown child
- val optimizedOutExchanges = ArrayBuffer.empty[Exchange]
- reusedExchanges.foreach{ reused =>
- val child = reused.child
- if (!idMap.containsKey(child)) {
- optimizedOutExchanges.append(child)
- currentOperatorID = generateOperatorIDs(child, currentOperatorID,
idMap,
- reusedExchanges, false)
- }
- }
+ val (subqueries, optimizedOutExchanges) = assignOperatorIds(plan, idMap)
val collectedOperators = BitSet.empty
processPlanSkippingSubqueries(plan, append, collectedOperators)
@@ -150,6 +125,41 @@ object ExplainUtils extends AdaptiveSparkPlanHelper {
}
}
+ /**
+ * Assigns operator IDs to all operators across the full plan tree -- the
main plan,
+ * any subqueries, and any adaptively-optimized-out exchanges (SPARK-42753)
-- by populating
+ * the supplied idMap. Returns the discovered subqueries and optimized-out
exchanges so
+ * the caller ([[processPlan]]) can perform post-assignment work (text
output) without
+ * rediscovering them.
+ */
+ private def assignOperatorIds(
+ plan: QueryPlan[_],
+ idMap: java.util.Map[QueryPlan[_], Int])
+ : (ArrayBuffer[(SparkPlan, Expression, BaseSubqueryExec)],
ArrayBuffer[Exchange]) = {
+ // Initialize an array of ReusedExchanges to help find Adaptively
Optimized Out
+ // Exchanges as part of SPARK-42753
+ val reusedExchanges = ArrayBuffer.empty[ReusedExchangeExec]
+ var currentOperatorID = generateOperatorIDs(plan, 0, idMap,
reusedExchanges, true)
+
+ val subqueries = ArrayBuffer.empty[(SparkPlan, Expression,
BaseSubqueryExec)]
+ getSubqueries(plan, subqueries)
+ currentOperatorID = subqueries.foldLeft(currentOperatorID) {
+ (curId, sub) => generateOperatorIDs(sub._3.child, curId, idMap,
reusedExchanges, true)
+ }
+
+ // SPARK-42753: Process subtree for a ReusedExchange with unknown child
+ val optimizedOutExchanges = ArrayBuffer.empty[Exchange]
+ reusedExchanges.foreach { reused =>
+ val child = reused.child
+ if (!idMap.containsKey(child)) {
+ optimizedOutExchanges.append(child)
+ currentOperatorID = generateOperatorIDs(child, currentOperatorID,
idMap,
+ reusedExchanges, false)
+ }
+ }
+ (subqueries, optimizedOutExchanges)
+ }
+
/**
* Traverses the supplied input plan in a bottom-up fashion and records the
operator id via
* setting a tag in the operator.
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ExplainUtilsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExplainUtilsSuite.scala
new file mode 100644
index 000000000000..4830f5d9e78d
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ExplainUtilsSuite.scala
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.catalyst.plans.QueryPlan
+import org.apache.spark.sql.catalyst.plans.logical.Range
+import org.apache.spark.sql.catalyst.util.StringConcat
+import org.apache.spark.sql.test.SharedSparkSession
+
+/**
+ * Tests for [[ExplainUtils.processPlan]]: operator ID assignment,
WholeStageCodegen tag
+ * propagation, thread-local lifecycle, and subquery handling.
+ */
+class ExplainUtilsSuite extends QueryTest with SharedSparkSession {
+
+ private def explainOutput(plan: SparkPlan): String = {
+ val concat = new StringConcat()
+ ExplainUtils.processPlan(plan, concat.append)
+ concat.toString
+ }
+
+ test("processPlan assigns unique operator IDs to all visible plan nodes") {
+ val df = spark.range(100).filter("id > 10").select("id")
+ val output = explainOutput(df.queryExecution.executedPlan)
+ // Each operator ID appears both in the tree header ("Filter (2)") and as
the header of its
+ // verbose section ("(2) Filter"). Anchor to the verbose-section headers,
which are the only
+ // lines that begin with "(N)", so each operator contributes exactly one
ID.
+ val ids =
"(?m)^\\((\\d+)\\)".r.findAllMatchIn(output).map(_.group(1).toInt).toSeq
+ assert(ids.nonEmpty, "processPlan should assign at least one operator ID")
+ assert(ids == ids.distinct, s"processPlan operator IDs should be unique:
$ids")
+ }
+
+ test("processPlan tags a WholeStageCodegenExec child with its codegen stage
id") {
+ // Build the plan directly so the assertion does not depend on the planner
producing a
+ // WholeStageCodegenExec for any particular query. processPlan should
stamp the child of a
+ // WholeStageCodegenExec with CODEGEN_ID_TAG carrying that node's
codegenStageId.
+ val child = RangeExec(Range(0, 10, 1, 1))
+ val plan = WholeStageCodegenExec(child)(codegenStageId = 7)
+ ExplainUtils.processPlan[SparkPlan](plan, _ => ())
+ assert(child.getTagValue(QueryPlan.CODEGEN_ID_TAG).contains(7),
+ "processPlan should tag a WholeStageCodegenExec child with its
codegenStageId")
+ }
+
+ test("processPlan restores localIdMap to its prior value after completion") {
+ val prev = ExplainUtils.localIdMap.get()
+ ExplainUtils.processPlan(spark.range(10).filter("id >
3").queryExecution.executedPlan,
+ _ => ())
+ assert(ExplainUtils.localIdMap.get() eq prev,
+ "processPlan should restore the thread-local localIdMap on completion")
+ }
+
+ test("processPlan includes subquery output in the explain string") {
+ withSQLConf("spark.sql.adaptive.enabled" -> "false") {
+ val df = spark.range(100).filter("id > 0")
+ .filter("id < (SELECT max(id) FROM range(5))")
+ val output = explainOutput(df.queryExecution.executedPlan)
+ assert(output.contains("Subqueries"), "processPlan should emit a
Subqueries section")
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]