This is an automated email from the ASF dual-hosted git repository.
gengliangwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 39652d88d4cf [SPARK-60525][SQL] Wrap RelationChanges directly in
NamedStreamingRelation for streaming CHANGES queries
39652d88d4cf is described below
commit 39652d88d4cf6b7387de48807dbcf6ee6268230f
Author: AnishMahto <[email protected]>
AuthorDate: Thu Nov 5 12:07:43 2026 -0800
[SPARK-60525][SQL] Wrap RelationChanges directly in NamedStreamingRelation
for streaming CHANGES queries
### What changes were proposed in this pull request?
When `spark.sql.streaming.queryEvolution.enableSourceEvolution=true`, every
streaming source must be named, which SQL users do with `IDENTIFIED BY`. For
streaming `CHANGES` queries, combining `IDENTIFIED BY` with an alias or
watermark fails analysis:
```sql
SELECT * FROM STREAM cat.t CHANGES FROM VERSION 1 IDENTIFIED BY src AS c;
SELECT * FROM STREAM cat.t CHANGES FROM VERSION 1 IDENTIFIED BY src
WATERMARK _commit_timestamp DELAY OF INTERVAL 1 MINUTE;
```
For a `CHANGES` query, `AstBuilder.visitStreamTableName` builds the
`NamedStreamingRelation` over the aliased and watermarked plan, not over
`RelationChanges`. With both clauses, the plan looks roughly like:
```
NamedStreamingRelation(src)
+- EventTimeWatermark
+- SubqueryAlias(c)
+- RelationChanges
```
`RelationChanges` resolves to `SubqueryAlias(cat.t, StreamingRelationV2)`.
The `NameStreamingSources` analysis rule only removes `NamedStreamingRelation`
and passes the name down when its child is a streaming relation, or a single
`SubqueryAlias` directly over one. Here the watermark, the alias, or both sit
in between, so the rule never matches. The `NamedStreamingRelation` is left in
the plan unresolved, and analysis fails even though the statement is valid. For
`SELECT *`, the error [...]
The fix is to build the `NamedStreamingRelation` directly over
`RelationChanges`, which is what the non-`CHANGES` branch and
`DataStreamReader.changes()` already do:
```
EventTimeWatermark
+- SubqueryAlias(c)
+- NamedStreamingRelation(src)
+- RelationChanges
```
### Why are the changes needed?
Bug fix; today users cannot specify `IDENTIFIED BY` along with an alias or
watermark for a SQL `STREAM` statement, even though both should be supported.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit tests.
### Was this patch authored or co-authored using generative AI tooling?
Co-authored with Opus 5.5.
Closes #59761 from
AnishMahto/SPARK-60525-wrap-relation-changes-in-named-streaming-relation.
Authored-by: AnishMahto <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
---
.../spark/sql/catalyst/parser/AstBuilder.scala | 8 +++---
.../parser/StreamRelationParserSuite.scala | 28 ++++++++++++++++++
.../sql/connector/ChangelogEndToEndSuite.scala | 33 +++++++++++++++++++++-
3 files changed, 64 insertions(+), 5 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 400526af790d..7883e2e97a30 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -3413,11 +3413,11 @@ class AstBuilder extends DataTypeAstBuilder
// Streaming CDC: wrap in RelationChanges and NamedStreamingRelation
val options = resolveOptions(Option(ctx.optionsClause))
val changelogContext = buildStreamChangelogContext(changesCtx, options)
- val result = RelationChanges(relation, changelogContext)
- val table = mayApplyAliasPlan(ctx.tableAlias, result)
- val tableWithWatermark =
table.optionalMap(ctx.watermarkClause)(withWatermark)
val sourceNameOpt = extractSourceName(ctx.identifiedByClause)
- NamedStreamingRelation.withUserProvidedName(tableWithWatermark,
sourceNameOpt)
+ val namedChanges = NamedStreamingRelation.withUserProvidedName(
+ RelationChanges(relation, changelogContext), sourceNameOpt)
+ val table = mayApplyAliasPlan(ctx.tableAlias, namedChanges)
+ table.optionalMap(ctx.watermarkClause)(withWatermark)
case None =>
// Regular streaming relation
val table = mayApplyAliasPlan(ctx.tableAlias, relation)
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
index e5c374a5130f..1d78baa72159 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/StreamRelationParserSuite.scala
@@ -642,6 +642,34 @@ class StreamRelationParserSuite extends AnalysisTest {
assert(relationChanges.size == 1)
}
+ test("SPARK-60525: STREAM t CHANGES wraps RelationChanges directly in
NamedStreamingRelation") {
+ val wm = "WATERMARK col DELAY OF INTERVAL 1 MINUTE"
+ val testCases = Seq(
+ // (trailing clauses, expectedSourceName)
+ ("", Unassigned),
+ ("AS tbl", Unassigned),
+ (wm, Unassigned),
+ (s"$wm AS tbl", Unassigned),
+ ("IDENTIFIED BY src1", UserProvided("src1")),
+ ("IDENTIFIED BY src1 AS tbl", UserProvided("src1")),
+ (s"IDENTIFIED BY src1 $wm", UserProvided("src1")),
+ (s"IDENTIFIED BY src1 $wm AS tbl", UserProvided("src1"))
+ )
+
+ testCases.foreach { case (clauses, expectedSourceName) =>
+ val query = s"SELECT * FROM STREAM t CHANGES FROM VERSION 1 $clauses"
+ val namedStreamingRelations = parsePlan(query).collect {
+ case n: NamedStreamingRelation => n
+ }
+ assert(namedStreamingRelations.size == 1,
+ s"Expected 1 NamedStreamingRelation node in: $query")
+ assert(namedStreamingRelations.head.sourceIdentifyingName ==
expectedSourceName,
+ s"Expected source name $expectedSourceName in: $query")
+ assert(namedStreamingRelations.head.child.isInstanceOf[RelationChanges],
+ s"Expected NamedStreamingRelation to directly wrap RelationChanges in:
$query")
+ }
+ }
+
test("STREAM t CHANGES with WITH options") {
val plan = parsePlan(
"SELECT * FROM STREAM t CHANGES FROM VERSION 1 " +
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
index bb40cd9874d2..31c16d245b8a 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/ChangelogEndToEndSuite.scala
@@ -23,7 +23,7 @@ import java.util
import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.NamedStreamingRelation
-import org.apache.spark.sql.catalyst.streaming.UserProvided
+import org.apache.spark.sql.catalyst.streaming.{StreamingRelationV2,
UserProvided}
import org.apache.spark.sql.connector.catalog._
import org.apache.spark.sql.connector.catalog.Changelog.{
CHANGE_TYPE_DELETE, CHANGE_TYPE_INSERT, CHANGE_TYPE_UPDATE_POSTIMAGE,
CHANGE_TYPE_UPDATE_PREIMAGE}
@@ -652,6 +652,37 @@ class ChangelogEndToEndSuite extends SharedSparkSession {
}
}
+ test("SPARK-60525: streaming SQL CHANGES supports IDENTIFIED BY with alias
and watermark") {
+ catalog.addChangeRows(ident, Seq(
+ makeChangeRow(1L, "a", CHANGE_TYPE_INSERT, 1L, 1000000L)))
+
+ val expected = Seq(
+ Row(1L, "a", CHANGE_TYPE_INSERT, 1L, new Timestamp(1000L)))
+
+ val wm = "WATERMARK _commit_timestamp DELAY OF INTERVAL 1 MINUTE"
+ withSQLConf(SQLConf.ENABLE_STREAMING_SOURCE_EVOLUTION.key -> "true") {
+ Seq("AS c", wm, s"$wm AS c").zipWithIndex.foreach { case (clauses, i) =>
+ val stream = sql(
+ s"SELECT * FROM STREAM $fullTableName CHANGES FROM VERSION 1 " +
+ s"IDENTIFIED BY my_cdc_source $clauses")
+
+ val sourceNames = stream.queryExecution.analyzed.collect {
+ case r: StreamingRelationV2 => r.sourceIdentifyingName
+ }
+ assert(sourceNames === Seq(UserProvided("my_cdc_source")), clauses)
+
+ val queryName = s"cdc_stream_identified_by_$i"
+ val q =
stream.writeStream.format("memory").queryName(queryName).start()
+ try {
+ q.processAllAvailable()
+ checkAnswer(spark.sql(s"SELECT * FROM $queryName"), expected)
+ } finally {
+ q.stop()
+ }
+ }
+ }
+ }
+
// ---------- Streaming: error cases ----------
test("streaming changes() rejects user-specified schema") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]