This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 7dbfd9ae7ebc [SPARK-40259][SQL][FOLLOWUP] Add cross-table DSv2 scan
merge regression test
7dbfd9ae7ebc is described below
commit 7dbfd9ae7ebc6c2764abf9197f72a4a340d65a99
Author: Wenchen Fan <[email protected]>
AuthorDate: Mon Aug 24 14:39:18 2026 +0800
[SPARK-40259][SQL][FOLLOWUP] Add cross-table DSv2 scan merge regression test
### What changes were proposed in this pull request?
Followup to https://github.com/apache/spark/pull/57360.
Add an end-to-end regression test verifying that Data Source V2 scans of
two different tables are
not merged, even when both tables opt into scan merging and the query
shapes are otherwise similar.
### Why are the changes needed?
Merging scans from different tables could corrupt the optimized plan and
return incorrect query
results. The existing implementation compares canonical relations, and this
test protects that
table-identity safety boundary.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added coverage to `DSv2PlanMergingSuite` and ran the full suite.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #58202 from cloud-fan/SPARK-40259-followup-248101.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit f7910853e5d71a2dc797c3de1fc4e61ea3d12e3a)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../planmerging/DSv2PlanMergingSuite.scala | 24 ++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/planmerging/DSv2PlanMergingSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/planmerging/DSv2PlanMergingSuite.scala
index 977e115d16b0..86bbe951ab9b 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/planmerging/DSv2PlanMergingSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/planmerging/DSv2PlanMergingSuite.scala
@@ -41,6 +41,7 @@ class DSv2PlanMergingSuite extends QueryTest with
SharedSparkSession
private val v2Source = classOf[FakeV2ProviderWithCustomSchema].getName
private val tbl = "scanmerge.t"
+ private val tbl2 = "scanmerge.t2"
before {
spark.conf.set("spark.sql.catalog.scanmerge",
@@ -174,6 +175,29 @@ class DSv2PlanMergingSuite extends QueryTest with
SharedSparkSession
}
}
+ test("SPARK-40259: do not merge DSv2 scans from different tables") {
+ withTable(tbl, tbl2) {
+ sql(s"CREATE TABLE $tbl (part_col string, c1 int) USING $v2Source " +
+ "PARTITIONED BY (part_col)")
+ sql(s"CREATE TABLE $tbl2 (part_col string, c2 int) USING $v2Source " +
+ "PARTITIONED BY (part_col)")
+ sql(s"INSERT INTO $tbl VALUES ('a', 1), ('a', 2)")
+ sql(s"INSERT INTO $tbl2 VALUES ('a', 10), ('a', 30)")
+
+ val df = sql(
+ s"""
+ |SELECT
+ | (SELECT max(c1) FROM $tbl WHERE part_col IN ('a')) AS m1,
+ | (SELECT max(c2) FROM $tbl2 WHERE part_col IN ('a')) AS m2
+ |""".stripMargin)
+
+ checkAnswer(df, Row(2, 30))
+ val scans = v2Scans(df)
+ assert(scans.map(_.canonicalized).distinct.length == 2,
+ s"scans from different tables must remain
separate:\n${df.queryExecution.optimizedPlan}")
+ }
+ }
+
test("SPARK-58549: a scan merge preserves the sources' reported key-grouped
partitioning") {
val t = "scanmergereport.t2"
withTable(t) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
