This is an automated email from the ASF dual-hosted git repository.

gengliangwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 39652d88d4cf [SPARK-60525][SQL] Wrap RelationChanges directly in 
NamedStreamingRelation for streaming CHANGES queries
39652d88d4cf is described below

commit 39652d88d4cf6b7387de48807dbcf6ee6268230f
Author: AnishMahto <[email protected]>
AuthorDate: Thu Nov 5 12:07:43 2026 -0800

    [SPARK-60525][SQL] Wrap RelationChanges directly in NamedStreamingRelation 
for streaming CHANGES queries
    
    ### What changes were proposed in this pull request?
    When `spark.sql.streaming.queryEvolution.enableSourceEvolution=true`, every 
streaming source must be named, which SQL users do with `IDENTIFIED BY`. For 
streaming `CHANGES` queries, combining `IDENTIFIED BY` with an alias or 
watermark fails analysis:
    
    ```sql
    SELECT * FROM STREAM cat.t CHANGES FROM VERSION 1 IDENTIFIED BY src AS c;
    SELECT * FROM STREAM cat.t CHANGES FROM VERSION 1 IDENTIFIED BY src
    WATERMARK _commit_timestamp DELAY OF INTERVAL 1 MINUTE;
    ```
    
    For a `CHANGES` query, `AstBuilder.visitStreamTableName` builds the 
`NamedStreamingRelation` over the aliased and watermarked plan, not over 
`RelationChanges`. With both clauses, the plan looks roughly like:
    
    ```
    NamedStreamingRelation(src)
    +- EventTimeWatermark
       +- SubqueryAlias(c)
          +- RelationChanges
    ```
    
    `RelationChanges` resolves to `SubqueryAlias(cat.t, StreamingRelationV2)`. 
The `NameStreamingSources` analysis rule only removes `NamedStreamingRelation` 
and passes the name down when its child is a streaming relation, or a single 
`SubqueryAlias` directly over one. Here the watermark, the alias, or both sit 
in between, so the rule never matches. The `NamedStreamingRelation` is left in 
the plan unresolved, and analysis fails even though the statement is valid. For 
`SELECT *`, the error [...]
    
    The fix is to build the `NamedStreamingRelation` directly over 
`RelationChanges`, which is what the non-`CHANGES` branch and 
`DataStreamReader.changes()` already do:
    ```
    EventTimeWatermark
    +- SubqueryAlias(c)
       +- NamedStreamingRelation(src)
          +- RelationChanges
    ```
    
    ### Why are the changes needed?
    Bug fix; today users cannot specify `IDENTIFIED BY` along with an alias or 
watermark for a SQL `STREAM` statement, even though both should be supported.
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Unit tests.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Co-authored with Opus 5.5.
    
    Closes #59761 from 
AnishMahto/SPARK-60525-wrap-relation-changes-in-named-streaming-relation.
    
    Authored-by: AnishMahto <[email protected]>
    Signed-off-by: Gengliang Wang <[email protected]>
---
 .../spark/sql/catalyst/parser/AstBuilder.scala     |  8 +++---
 .../parser/StreamRelationParserSuite.scala         | 28 ++++++++++++++++++
 .../sql/connector/ChangelogEndToEndSuite.scala     | 33 +++++++++++++++++++++-
 3 files changed, 64 insertions(+), 5 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 400526af790d..7883e2e97a30 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -3413,11 +3413,11 @@ class AstBuilder extends DataTypeAstBuilder
         // Streaming CDC: wrap in RelationChanges and NamedStreamingRelation
         val options = resolveOptions(Option(ctx.optionsClause))
         val changelogContext = buildStreamChangelogContext(changesCtx, options)
-        val result = RelationChanges(relation, changelogContext)
-        val table = mayApplyAliasPlan(ctx.tableAlias, result)
-        val tableWithWatermark = 
table.optionalMap(ctx.watermarkClause)(withWatermark)
         val sourceNameOpt = extractSourceName(ctx.identifiedByClause)
-        NamedStreamingRelation.withUserProvidedName(tableWithWatermark, 
sourceNameOpt)
+        val namedChanges = NamedStreamingRelation.withUserProvidedName(
+          RelationChanges(relation, changelogContext), sourceNameOpt)
+        val table = mayApplyAliasPlan(ctx.tableAlias, namedChanges)
+        table.optionalMap(ctx.watermarkClause)(withWatermark)
       case None =>
         // Regular streaming relation
         val table = mayApplyAliasPlan(ctx.tableAlias, relation)
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
index e5c374a5130f..1d78baa72159 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
@@ -642,6 +642,34 @@ class StreamRelationParserSuite extends AnalysisTest {
     assert(relationChanges.size == 1)
   }
 
+  test("SPARK-60525: STREAM t CHANGES wraps RelationChanges directly in 
NamedStreamingRelation") {
+    val wm = "WATERMARK col DELAY OF INTERVAL 1 MINUTE"
+    val testCases = Seq(
+      // (trailing clauses, expectedSourceName)
+      ("", Unassigned),
+      ("AS tbl", Unassigned),
+      (wm, Unassigned),
+      (s"$wm AS tbl", Unassigned),
+      ("IDENTIFIED BY src1", UserProvided("src1")),
+      ("IDENTIFIED BY src1 AS tbl", UserProvided("src1")),
+      (s"IDENTIFIED BY src1 $wm", UserProvided("src1")),
+      (s"IDENTIFIED BY src1 $wm AS tbl", UserProvided("src1"))
+    )
+
+    testCases.foreach { case (clauses, expectedSourceName) =>
+      val query = s"SELECT * FROM STREAM t CHANGES FROM VERSION 1 $clauses"
+      val namedStreamingRelations = parsePlan(query).collect {
+        case n: NamedStreamingRelation => n
+      }
+      assert(namedStreamingRelations.size == 1,
+        s"Expected 1 NamedStreamingRelation node in: $query")
+      assert(namedStreamingRelations.head.sourceIdentifyingName == 
expectedSourceName,
+        s"Expected source name $expectedSourceName in: $query")
+      assert(namedStreamingRelations.head.child.isInstanceOf[RelationChanges],
+        s"Expected NamedStreamingRelation to directly wrap RelationChanges in: 
$query")
+    }
+  }
+
   test("STREAM t CHANGES with WITH options") {
     val plan = parsePlan(
       "SELECT * FROM STREAM t CHANGES FROM VERSION 1 " +
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
index bb40cd9874d2..31c16d245b8a 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
@@ -23,7 +23,7 @@ import java.util
 import org.apache.spark.sql.{AnalysisException, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.NamedStreamingRelation
-import org.apache.spark.sql.catalyst.streaming.UserProvided
+import org.apache.spark.sql.catalyst.streaming.{StreamingRelationV2, 
UserProvided}
 import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.connector.catalog.Changelog.{
   CHANGE_TYPE_DELETE, CHANGE_TYPE_INSERT, CHANGE_TYPE_UPDATE_POSTIMAGE, 
CHANGE_TYPE_UPDATE_PREIMAGE}
@@ -652,6 +652,37 @@ class ChangelogEndToEndSuite extends SharedSparkSession {
     }
   }
 
+  test("SPARK-60525: streaming SQL CHANGES supports IDENTIFIED BY with alias 
and watermark") {
+    catalog.addChangeRows(ident, Seq(
+      makeChangeRow(1L, "a", CHANGE_TYPE_INSERT, 1L, 1000000L)))
+
+    val expected = Seq(
+      Row(1L, "a", CHANGE_TYPE_INSERT, 1L, new Timestamp(1000L)))
+
+    val wm = "WATERMARK _commit_timestamp DELAY OF INTERVAL 1 MINUTE"
+    withSQLConf(SQLConf.ENABLE_STREAMING_SOURCE_EVOLUTION.key -> "true") {
+      Seq("AS c", wm, s"$wm AS c").zipWithIndex.foreach { case (clauses, i) =>
+        val stream = sql(
+          s"SELECT * FROM STREAM $fullTableName CHANGES FROM VERSION 1 " +
+            s"IDENTIFIED BY my_cdc_source $clauses")
+
+        val sourceNames = stream.queryExecution.analyzed.collect {
+          case r: StreamingRelationV2 => r.sourceIdentifyingName
+        }
+        assert(sourceNames === Seq(UserProvided("my_cdc_source")), clauses)
+
+        val queryName = s"cdc_stream_identified_by_$i"
+        val q = 
stream.writeStream.format("memory").queryName(queryName).start()
+        try {
+          q.processAllAvailable()
+          checkAnswer(spark.sql(s"SELECT * FROM $queryName"), expected)
+        } finally {
+          q.stop()
+        }
+      }
+    }
+  }
+
   // ---------- Streaming: error cases ----------
 
   test("streaming changes() rejects user-specified schema") {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to