This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new dfe005835c4c [SPARK-47017][SQL] Show internal SQL plans for RDDScanExec
dfe005835c4c is described below

commit dfe005835c4c87bb899182a1302460c20102f075
Author: Aleksandr Efimov <[email protected]>
AuthorDate: Mon Aug 3 09:17:39 2026 -0700

    [SPARK-47017][SQL] Show internal SQL plans for RDDScanExec
    
    ### What changes were proposed in this pull request?
    
    This PR preserves the original SQL physical plan inside `SQLExecutionRDD` 
and uses it when building `SparkPlanInfo` for `RDDScanExec`.
    
    When a DataFrame is converted to an RDD and then back to a DataFrame, Spark 
currently shows only `Scan ExistingRDD` in the SQL UI. This change lets 
`SparkPlanInfo` discover the underlying SQL plan from the `SQLExecutionRDD` 
lineage and attach it under the `RDDScanExec` node.
    
    The RDD lineage traversal is defensive:
    
    - it deduplicates RDDs by id;
    - it stops at each discovered `SQLExecutionRDD`;
    - it tolerates dependency lookup failures by skipping that branch.
    
    ### Why are the changes needed?
    
    Without this, the SQL UI loses useful operator visibility and metrics after 
workflows such as:
    
    ```scala
    val source = spark.range(10).filter($"id" > 3)
    val recreated = spark.createDataFrame(source.rdd, source.schema)
    recreated.collect()
    ```
    
    The UI only exposes `Scan ExistingRDD`, while the internal SQL operators, 
such as `Filter`, are hidden. This makes query diagnosis harder for 
DataFrame/RDD/DataFrame workflows.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. For SQL executions that scan an RDD backed by a previous SQL 
execution, the SQL UI can now show the internal SQL plan below `Scan 
ExistingRDD`, including metrics such as `number of output rows`.
    
    ### How was this patch tested?
    
    Added regression coverage in `SparkPlanInfoSuite` for:
    
    - exposing the internal SQL plan under `RDDScanExec`;
    - preserving SQL metrics in the SQL UI plan graph;
    - deduplicating shared RDD lineage;
    - nested `createDataFrame(df.rdd, df.schema)` cases.
    
    Ran:
    
    ```bash
    build/sbt "sql/testOnly *SparkPlanInfoSuite -- -z SPARK-47017"
    ```
    
    Result:
    
    ```text
    Tests: succeeded 1, failed 0
    All tests passed.
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-5)
    
    Closes #56225 from alexandrefimov/SPARK-47017-rddscan-history-metrics.
    
    Authored-by: Aleksandr Efimov <[email protected]>
    Signed-off-by: Holden Karau <[email protected]>
    (cherry picked from commit 4f147ecca67159982ee709871bacda1b41f9e047)
    Signed-off-by: Holden Karau <[email protected]>
---
 .../spark/sql/execution/QueryExecution.scala       |  5 +-
 .../spark/sql/execution/SQLExecutionRDD.scala      | 11 ++-
 .../apache/spark/sql/execution/SparkPlanInfo.scala | 33 +++++++++
 .../sql/execution/ui/SparkPlanInfoSuite.scala      | 80 ++++++++++++++++++++++
 4 files changed, 127 insertions(+), 2 deletions(-)

diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
index 0a1185aa4ea5..1fa70351f2d0 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/QueryExecution.scala
@@ -376,7 +376,10 @@ class QueryExecution(
   def assertExecutedPlanPrepared(): Unit = executedPlan
 
   val lazyToRdd = LazyTry {
-    new SQLExecutionRDD(executedPlan.execute(), sparkSession.sessionState.conf)
+    new SQLExecutionRDD(
+      executedPlan.execute(),
+      sparkSession.sessionState.conf,
+      SparkPlanInfo.fromSparkPlan(executedPlan))
   }
 
   /**
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala
index 45b9cadc4aed..7c38ccee4983 100644
--- 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala
+++ 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SQLExecutionRDD.scala
@@ -29,9 +29,18 @@ import org.apache.spark.sql.internal.SQLConf
  *
  * @param sqlRDD the `RDD` generated by the SQL plan
  * @param conf the `SQLConf` to apply to the execution of the SQL plan
+ * @param sparkPlanInfo the physical plan information for `sqlRDD`
  */
 class SQLExecutionRDD(
-    var sqlRDD: RDD[InternalRow], @transient conf: SQLConf) extends 
RDD[InternalRow](sqlRDD) {
+    var sqlRDD: RDD[InternalRow],
+    @transient conf: SQLConf,
+    @transient val sparkPlanInfo: SparkPlanInfo = SparkPlanInfo.EMPTY)
+  extends RDD[InternalRow](sqlRDD) {
+
+  def this(sqlRDD: RDD[InternalRow], conf: SQLConf) = {
+    this(sqlRDD, conf, SparkPlanInfo.EMPTY)
+  }
+
   private val sqlConfigs = conf.getAllConfs
   private lazy val sqlConfExecutorSide = {
     val newConf = new SQLConf()
diff --git 
a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala 
b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
index 4410fe50912f..32bff876d454 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlanInfo.scala
@@ -17,7 +17,10 @@
 
 package org.apache.spark.sql.execution
 
+import scala.util.control.NonFatal
+
 import org.apache.spark.annotation.DeveloperApi
+import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.plans.logical.{EmptyRelation, LogicalPlan}
 import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, 
QueryStageExec}
 import org.apache.spark.sql.execution.adaptive.LogicalQueryStage
@@ -74,6 +77,7 @@ private[execution] object SparkPlanInfo {
       case a: AdaptiveSparkPlanExec => a.executedPlan :: Nil
       case stage: QueryStageExec => stage.plan :: Nil
       case inMemTab: InMemoryTableScanExec => inMemTab.relation.cachedPlan :: 
Nil
+      case rddScan: RDDScanExec => sparkPlanInfosFromRDD(rddScan.rdd)
       case EmptyRelationExec(logical) => (logical :: Nil)
       case _ => plan.children ++ plan.subqueries
     }
@@ -91,6 +95,8 @@ private[execution] object SparkPlanInfo {
         Some(fromSparkPlan(child))
       case child: LogicalPlan =>
         Some(fromLogicalPlan(child))
+      case child: SparkPlanInfo =>
+        Some(child)
       case _ => None
     }
     new SparkPlanInfo(
@@ -102,4 +108,31 @@ private[execution] object SparkPlanInfo {
   }
 
   final lazy val EMPTY: SparkPlanInfo = new SparkPlanInfo("", "", Nil, 
Map.empty, Nil)
+
+  private def sparkPlanInfosFromRDD(rdd: RDD[_]): Seq[SparkPlanInfo] = {
+    // Walk only driver-side RDD dependency metadata. Dedupe by RDD id so 
shared lineage does not
+    // duplicate the same internal SQL plan under a single RDDScanExec.
+    val visitedRDDs = scala.collection.mutable.HashSet.empty[Int]
+    val rddsToVisit = scala.collection.mutable.Queue.empty[RDD[_]]
+    val planInfos = scala.collection.mutable.ArrayBuffer.empty[SparkPlanInfo]
+
+    rddsToVisit.enqueue(rdd)
+    while (rddsToVisit.nonEmpty) {
+      val current = rddsToVisit.dequeue()
+      if (visitedRDDs.add(current.id)) {
+        current match {
+          case sqlRDD: SQLExecutionRDD if sqlRDD.sparkPlanInfo != EMPTY =>
+            planInfos += sqlRDD.sparkPlanInfo
+          case _ =>
+            try {
+              current.dependencies.foreach(dep => rddsToVisit.enqueue(dep.rdd))
+            } catch {
+              case NonFatal(_) =>
+            }
+          }
+      }
+    }
+
+    planInfos.toSeq
+  }
 }
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala
index 1ef07bf9ebc1..2fc070e8f763 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/ui/SparkPlanInfoSuite.scala
@@ -17,6 +17,9 @@
 
 package org.apache.spark.sql.execution.ui
 
+import scala.concurrent.duration._
+
+import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.execution.SparkPlanInfo
 import org.apache.spark.sql.test.SharedSparkSession
 
@@ -24,6 +27,39 @@ class SparkPlanInfoSuite extends SharedSparkSession {
 
   import testImplicits._
 
+  private def collectSparkPlanInfo(sparkPlanInfo: SparkPlanInfo): 
Seq[SparkPlanInfo] = {
+    sparkPlanInfo +: sparkPlanInfo.children.flatMap(collectSparkPlanInfo)
+  }
+
+  private def findSparkPlanInfo(sparkPlanInfo: SparkPlanInfo, nodeName: 
String): SparkPlanInfo = {
+    collectSparkPlanInfo(sparkPlanInfo)
+      .find(_.nodeName == nodeName)
+      .getOrElse(fail(s"Could not find $nodeName in 
${sparkPlanInfo.simpleString}"))
+  }
+
+  private def collectSparkPlanGraphMetrics(
+      df: DataFrame): (SparkPlanGraph, Map[Long, String]) = {
+    val statusStore = spark.sharedState.statusStore
+    spark.sparkContext.listenerBus.waitUntilEmpty(10000)
+    val previousExecutionIds = 
statusStore.executionsList().map(_.executionId).toSet
+
+    df.collect()
+    spark.sparkContext.listenerBus.waitUntilEmpty(10000)
+
+    eventually(timeout(10.seconds), interval(10.milliseconds)) {
+      assert(statusStore.executionsList().map(_.executionId).toSet
+        .diff(previousExecutionIds).size === 1)
+    }
+    val executionIds = statusStore.executionsList().map(_.executionId).toSet
+      .diff(previousExecutionIds)
+    val executionId = executionIds.head
+    eventually(timeout(10.seconds), interval(10.milliseconds)) {
+      assert(statusStore.execution(executionId).exists(_.metricValues != null))
+    }
+
+    (statusStore.planGraph(executionId), 
statusStore.executionMetrics(executionId))
+  }
+
   def validateSparkPlanInfo(sparkPlanInfo: SparkPlanInfo): Unit = {
     sparkPlanInfo.nodeName match {
       case "InMemoryTableScan" => assert(sparkPlanInfo.children.length == 1)
@@ -41,4 +77,48 @@ class SparkPlanInfoSuite extends SharedSparkSession {
 
     validateSparkPlanInfo(planInfoResult)
   }
+
+  test("SPARK-47017: SparkPlanInfo and SQL UI include SQL plan inside 
RDDScanExec") {
+    val source = spark.range(10).where($"id" > 3).select($"id".as("age"))
+    val recreated = spark.createDataFrame(source.rdd, source.schema)
+
+    val planInfo = 
SparkPlanInfo.fromSparkPlan(recreated.queryExecution.executedPlan)
+    val rddScanInfo = findSparkPlanInfo(planInfo, "Scan ExistingRDD")
+    val internalRDDPlanInfos = 
rddScanInfo.children.flatMap(collectSparkPlanInfo)
+    val filterInfo = internalRDDPlanInfos
+      .find(_.nodeName == "Filter")
+      .getOrElse(fail(s"Could not find Filter under Scan ExistingRDD in 
${planInfo.simpleString}"))
+
+    assert(rddScanInfo.children.nonEmpty)
+    assert(filterInfo.metrics.exists(_.name == "number of output rows"))
+
+    val unionRDD = spark.sparkContext.union(source.rdd, source.rdd)
+    val unionPlanInfo = SparkPlanInfo.fromSparkPlan(
+      spark.createDataFrame(unionRDD, 
source.schema).queryExecution.executedPlan)
+    val unionRDDScanInfo = findSparkPlanInfo(unionPlanInfo, "Scan ExistingRDD")
+
+    assert(unionRDDScanInfo.children.size === 1)
+
+    val nested = spark.createDataFrame(recreated.rdd, recreated.schema)
+    val nestedPlanInfo = 
SparkPlanInfo.fromSparkPlan(nested.queryExecution.executedPlan)
+    val nestedRDDScanInfo = findSparkPlanInfo(nestedPlanInfo, "Scan 
ExistingRDD")
+
+    assert(nestedRDDScanInfo.children.size === 1)
+    
assert(collectSparkPlanInfo(nestedRDDScanInfo.children.head).exists(_.nodeName 
== "Filter"))
+
+    val (planGraph, metricValues) = collectSparkPlanGraphMetrics(recreated)
+    val filterNode = planGraph.allNodes
+      .find(_.name == "Filter")
+      .getOrElse(fail(s"Could not find Filter in 
${recreated.queryExecution.executedPlan}"))
+    val filterMetric = filterNode.metrics
+      .find(_.name == "number of output rows")
+      .getOrElse(fail("Could not find number of output rows metric for 
Filter"))
+    val filterMetricValue = metricValues
+      .getOrElse(filterMetric.accumulatorId, fail("Could not find Filter 
metric value"))
+    val outputRows = "\\d+".r.findFirstIn(filterMetricValue.replace(",", ""))
+      .map(_.toLong)
+      .getOrElse(fail(s"Could not parse Filter metric value 
$filterMetricValue"))
+
+    assert(outputRows === 6L)
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to