This is an automated email from the ASF dual-hosted git repository.
jose-torres pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 030f20a4fb07 [SPARK-59695][SDP] SCD1 Ignore-null support; Introduce
version map in `cdc_metadata`
030f20a4fb07 is described below
commit 030f20a4fb07b43d3ec3f9400dec036efd2c7635
Author: AnishMahto <[email protected]>
AuthorDate: Mon Nov 2 08:25:50 2026 -0800
[SPARK-59695][SDP] SCD1 Ignore-null support; Introduce version map in
`cdc_metadata`
### Why are the changes needed?
In order to support ignore-null in SCD1, we need to persist additional
metadata per key that tells us the last event that authored each key, if any so
far.
This is effectively per-leaf level versioning, rather than the typical
row/key-level versioning that SCD1 uses when ignore-null is off (the default
case). The latter relies on the single upsert/delete version per row, also in
the CDC metadata column.
This is the SCD1 equivalent change of
https://github.com/apache/spark/pull/58483.
### Does this PR introduce _any_ user-facing change?
Yes. After this change existing AutoCDC tables will undergo nested schema
evolution to gain a null version map field in their `cdc_metadata` column, when
they are not using ignore-null (the default case).
### How was this patch tested?
Added schema evolution test to `AutoCdcScd1SchemaEvolutionSuite`. Updated
existing tests to reflect the additional version map field in `cdc_metadata`.
### Was this patch authored or co-authored using generative AI tooling?
Co-authored with GPT 5.6 Sol.
Closes #58973 from AnishMahto/scd1-per-leaf-metadata-schema.
Authored-by: AnishMahto <[email protected]>
Signed-off-by: Jose Torres <[email protected]>
---
.../sql/pipelines/autocdc/Scd1BatchProcessor.scala | 21 +++++-
.../autocdc/Scd1ReconciliationStrategy.scala | 1 +
.../sql/pipelines/autocdc/Scd1VersionMap.scala | 40 +++++++++++
.../apache/spark/sql/pipelines/graph/Flow.scala | 1 +
.../autocdc/AutoCdcCatalogExecutionTestBase.scala | 8 +--
.../sql/pipelines/autocdc/AutoCdcFlowSuite.scala | 3 +
.../autocdc/Scd1BatchProcessorMergeSuite.scala | 61 ++++++++---------
.../autocdc/Scd1ForeachBatchHandlerSuite.scala | 42 ++++++------
.../autocdc/Scd1RowLevelReconciliationSuite.scala | 78 +++++++++++-----------
.../graph/AutoCdcGraphExecutionTestMixin.scala | 9 ++-
.../graph/AutoCdcScd1SchemaEvolutionSuite.scala | 74 ++++++++++++++++++++
.../sql/pipelines/graph/SqlPipelineSuite.scala | 4 +-
12 files changed, 237 insertions(+), 105 deletions(-)
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
index 4c93e688886e..87e92f6b2b8b 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
@@ -215,6 +215,7 @@ case class Scd1BatchProcessor(
object Scd1BatchProcessor {
private[pipelines] val cdcDeleteSequenceFieldName: String = "deleteSequence"
private[pipelines] val cdcUpsertSequenceFieldName: String = "upsertSequence"
+ private[pipelines] val versionMapFieldName: String = "__VERSION_MAP"
/** Project the delete sequence out of the CDC metadata column. */
private[autocdc] def deleteSequenceOf(cdcMetadataCol: Column): Column =
@@ -226,6 +227,10 @@ object Scd1BatchProcessor {
/**
* Schema of the CDC metadata struct column for SCD1.
+ *
+ * The CDC metadata of every microbatch, target, and auxiliary-table row
represents either a
+ * delete or an upsert, never both: exactly one of its delete and upsert
sequences is non-null,
+ * and only an upsert may carry a version map.
*/
private[pipelines] def cdcMetadataColSchema(sequencingType: DataType):
StructType =
StructType(
@@ -233,22 +238,34 @@ object Scd1BatchProcessor {
// The sequencing of the event if it represents a delete, null
otherwise.
StructField(cdcDeleteSequenceFieldName, sequencingType, nullable =
true),
// The sequencing of the event if it represents an upsert, null
otherwise.
- StructField(cdcUpsertSequenceFieldName, sequencingType, nullable =
true)
+ StructField(cdcUpsertSequenceFieldName, sequencingType, nullable =
true),
+ // On target rows, a null version map indicates row-level
reconciliation; a non-null map
+ // indicates per-leaf reconciliation. Delete rows, including
auxiliary-table tombstones,
+ // always use a null map: a delete's sequence applies to every leaf,
so a map would only
+ // repeat it.
+ StructField(
+ versionMapFieldName,
+ Scd1VersionMap.mapType(sequencingType),
+ nullable = true
+ )
)
)
/**
* Construct the CDC metadata struct column for SCD1, following the exact
schema and field
- * ordering defined by [[cdcMetadataColSchema]].
+ * ordering defined by [[cdcMetadataColSchema]]. Values for microbatch,
target, and
+ * auxiliary-table rows must uphold its contract.
*/
private[pipelines] def constructCdcMetadataCol(
deleteSequence: Column,
upsertSequence: Column,
+ versionMap: Column,
sequencingType: DataType): Column = {
val cdcMetadataFieldsInOrder =
cdcMetadataColSchema(sequencingType).fields.map { field =>
val value = field.name match {
case `cdcDeleteSequenceFieldName` => deleteSequence
case `cdcUpsertSequenceFieldName` => upsertSequence
+ case `versionMapFieldName` => versionMap
case other =>
throw SparkException.internalError(
s"Unable to construct SCD1 CDC metadata column due to unknown
`${other}` field."
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
index 8dbbdb51630e..7444bccd72c8 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
@@ -74,6 +74,7 @@ private[pipelines] trait Scd1ReconciliationStrategy {
Scd1BatchProcessor.constructCdcMetadataCol(
deleteSequence = rowDeleteSequence,
upsertSequence = rowUpsertSequence,
+ versionMap = F.lit(null),
sequencingType = resolvedSequencingType
)
)
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
new file mode 100644
index 000000000000..e2df5155ee96
--- /dev/null
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.pipelines.autocdc
+
+import org.apache.spark.sql.types.{DataType, MapType, StringType}
+
+/**
+ * Per-leaf authorship sequences for SCD1 reconciliation.
+ *
+ * An SCD1 target row can combine values authored by different CDC events. The
version map records
+ * which event currently authors each non-key user-data leaf.
+ */
+private[pipelines] object Scd1VersionMap {
+
+ /**
+ * Schema of the version map: `Map(String, sequencingType)`.
+ *
+ * Each key is the field path of a user-data leaf. Each value is the leaf's
authorship sequence:
+ * the sequencing value of the event that authored the leaf, or null if no
event has authored it.
+ *
+ * @param sequencingType The resolved data type of the flow's sequencing
expression.
+ */
+ def mapType(sequencingType: DataType): MapType =
+ MapType(StringType, sequencingType, valueContainsNull = true)
+}
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
index 98eba8a78bd4..1442afd6c432 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
@@ -391,6 +391,7 @@ class AutoCdcMergeFlow(
val emptyCdcMetadataCol: Column =
Scd1BatchProcessor.constructCdcMetadataCol(
deleteSequence = F.lit(null),
upsertSequence = F.lit(null),
+ versionMap = F.lit(null),
sequencingType = sequencingType
).as(AutoCdcReservedNames.cdcMetadataColName)
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
index 8688df071113..a73eba8ee721 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
@@ -87,16 +87,14 @@ trait AutoCdcCatalogExecutionTestBase {
* `Long` sequencing.
*/
protected def cdcMetadataColSchemaType(sequencingType: DataType = LongType):
StructType =
- new StructType()
- .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, sequencingType)
- .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, sequencingType)
+ Scd1BatchProcessor.cdcMetadataColSchema(sequencingType)
/**
- * Build a [[Row]] matching the [[AutoCdcReservedNames.cdcMetadataColName]]
struct's two fields,
+ * Build a [[Row]] matching the [[AutoCdcReservedNames.cdcMetadataColName]]
struct's three fields,
* in the order produced by [[Scd1BatchProcessor.constructCdcMetadataCol]]:
*/
protected def cdcMetadataRow[T](deleteSeq: Option[T], upsertSeq: Option[T]):
Row =
- Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null))
+ Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null), null)
/**
* Create a table in the test catalog under the given DSv2 [[Identifier]]
using `schema`,
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
index c20dca119eb8..05b838996d9b 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
@@ -283,6 +283,9 @@ class AutoCdcFlowSuite extends QueryTest with
SharedSparkSession {
val metaStruct = metaField.dataType.asInstanceOf[StructType]
assert(metaStruct(Scd1BatchProcessor.cdcDeleteSequenceFieldName).nullable)
assert(metaStruct(Scd1BatchProcessor.cdcUpsertSequenceFieldName).nullable)
+ val versionMapField = metaStruct(Scd1BatchProcessor.versionMapFieldName)
+ assert(versionMapField.nullable)
+ assert(versionMapField.dataType == Scd1VersionMap.mapType(LongType))
}
test("AutoCdcMergeFlow.schema is stable across reads") {
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
index 9cfe42267f8b..3e381a7297dd 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
@@ -115,12 +115,7 @@ class Scd1BatchProcessorMergeSuite
val microbatchSchema = new StructType()
.add("id", IntegerType)
.add("value", StringType)
- .add(
- AutoCdcReservedNames.cdcMetadataColName,
- new StructType()
- .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, LongType)
- .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, LongType)
- )
+ .add(AutoCdcReservedNames.cdcMetadataColName, cdcMetadataColSchemaType())
val microbatch = microbatchOf(microbatchSchema)(
Row(1, "data-leak", cdcMetadataRow(deleteSeq = Some(20L), upsertSeq =
None))
)
@@ -129,7 +124,7 @@ class Scd1BatchProcessorMergeSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
// Tombstone advanced to delete=20, with exactly one row per key (no
duplicate tombstones).
- checkAnswer(resultAuxTable, Row(1, Row(20L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(20L, null, null)))
// Schema strictly matches minimalSchema; the `value` column was dropped,
not smuggled in.
assert(columnNamesAndDataTypes(resultAuxTable.schema) ==
columnNamesAndDataTypes(minimalSchema))
}
@@ -159,7 +154,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoAuxiliaryTable(microbatch,
defaultAuxTableIdentifier)
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
- checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
}
test("mergeMicrobatchOntoAuxiliaryTable leaves rows for unrelated keys
untouched") {
@@ -175,8 +170,8 @@ class Scd1BatchProcessorMergeSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
checkAnswer(resultAuxTable, Seq(
- Row(1, Row(10L, null)),
- Row(2, Row(100L, null))
+ Row(1, Row(10L, null, null)),
+ Row(2, Row(100L, null, null))
))
}
@@ -194,7 +189,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoAuxiliaryTable(microbatch,
defaultAuxTableIdentifier)
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
- checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
}
test("mergeMicrobatchOntoAuxiliaryTable ignores microbatch upserts whose
sequence is older " +
@@ -208,7 +203,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoAuxiliaryTable(microbatch,
defaultAuxTableIdentifier)
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
- checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
}
test("mergeMicrobatchOntoAuxiliaryTable applies the tied-sequence asymmetry:
equal deletes " +
@@ -232,7 +227,7 @@ class Scd1BatchProcessorMergeSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
// Row 1's tombstone remains the same, but row 2's tombstone should be
marked as stale and
// deleted.
- checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
}
test("mergeMicrobatchOntoAuxiliaryTable upsert event for different key does
not affect " +
@@ -247,7 +242,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoAuxiliaryTable(microbatch,
defaultAuxTableIdentifier)
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
- checkAnswer(resultAuxTable, Row(2, Row(5L, null)))
+ checkAnswer(resultAuxTable, Row(2, Row(5L, null, null)))
}
test("mergeMicrobatchOntoAuxiliaryTable is idempotent across a microbatch
that exercises " +
@@ -271,9 +266,9 @@ class Scd1BatchProcessorMergeSuite
)
val expectedAfterMerge = Seq(
- Row(2, Row(25L, null)),
- Row(3, Row(30L, null)),
- Row(4, Row(40L, null))
+ Row(2, Row(25L, null, null)),
+ Row(3, Row(30L, null, null)),
+ Row(4, Row(40L, null, null))
)
// First merge applies all three clauses exactly once.
@@ -319,8 +314,8 @@ class Scd1BatchProcessorMergeSuite
compositeKeyProcessor.mergeMicrobatchOntoAuxiliaryTable(microbatch,
defaultAuxTableIdentifier)
checkAnswer(spark.read.table(defaultAuxTableIdentifier.quotedString), Seq(
- Row("US", 99, Row(50L, null)),
- Row("US", 1, Row(10L, null))
+ Row("US", 99, Row(50L, null, null)),
+ Row("US", 1, Row(10L, null, null))
))
}
@@ -352,7 +347,9 @@ class Scd1BatchProcessorMergeSuite
dottedKeyProcessor.mergeMicrobatchOntoAuxiliaryTable(microbatch,
defaultAuxTableIdentifier)
- checkAnswer(spark.read.table(defaultAuxTableIdentifier.quotedString),
Row(1, Row(20L, null)))
+ checkAnswer(
+ spark.read.table(defaultAuxTableIdentifier.quotedString),
+ Row(1, Row(20L, null, null)))
}
// =============== mergeMicrobatchOntoTarget tests ===============
@@ -367,7 +364,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoTarget(microbatch,
defaultTargetTableIdentifier)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L)))
+ checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L, null)))
assert(columnNamesAndDataTypes(resultTargetTable.schema) ==
columnNamesAndDataTypes(targetSchema))
}
@@ -385,7 +382,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoTarget(microbatch,
defaultTargetTableIdentifier)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(2, "keep-me", Row(null, 20L)))
+ checkAnswer(resultTargetTable, Row(2, "keep-me", Row(null, 20L, null)))
}
test("mergeMicrobatchOntoTarget inserts new upserts but not new (tombstone)
deletes") {
@@ -400,8 +397,8 @@ class Scd1BatchProcessorMergeSuite
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
checkAnswer(resultTargetTable, Seq(
- Row(1, "existing", Row(null, 10L)),
- Row(2, "insert-me", Row(null, 20L))
+ Row(1, "existing", Row(null, 10L, null)),
+ Row(2, "insert-me", Row(null, 20L, null))
))
}
@@ -420,8 +417,8 @@ class Scd1BatchProcessorMergeSuite
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
checkAnswer(resultTargetTable, Seq(
- Row(1, "target-delete-tie", Row(null, 10L)),
- Row(2, "target-newer", Row(null, 20L))
+ Row(1, "target-delete-tie", Row(null, 10L, null)),
+ Row(2, "target-newer", Row(null, 20L, null))
))
}
@@ -435,7 +432,7 @@ class Scd1BatchProcessorMergeSuite
processor.mergeMicrobatchOntoTarget(microbatch,
defaultTargetTableIdentifier)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "same-sequence-upsert", Row(null,
10L)))
+ checkAnswer(resultTargetTable, Row(1, "same-sequence-upsert", Row(null,
10L, null)))
}
test("mergeMicrobatchOntoTarget correctly matches escaped key column names")
{
@@ -470,7 +467,7 @@ class Scd1BatchProcessorMergeSuite
)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L)))
+ checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L, null)))
}
gridTest(
@@ -505,7 +502,7 @@ class Scd1BatchProcessorMergeSuite
defaultTargetTableIdentifier
)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L)))
+ checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L, null)))
}
}
}
@@ -525,9 +522,9 @@ class Scd1BatchProcessorMergeSuite
)
val expectedAfterMerge = Seq(
- Row(2, "updated", Row(null, 25L)),
- Row(3, "untouched", Row(null, 30L)),
- Row(4, "inserted", Row(null, 40L))
+ Row(2, "updated", Row(null, 25L, null)),
+ Row(3, "untouched", Row(null, 30L, null)),
+ Row(4, "inserted", Row(null, 40L, null))
)
processor.mergeMicrobatchOntoTarget(microbatch,
defaultTargetTableIdentifier)
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
index bb8043e720c6..ebc2dcd77651 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
@@ -107,7 +107,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
assert(resultAuxTable.collect().isEmpty)
- checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L)))
+ checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L, null)))
}
test(
@@ -139,7 +139,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
assert(resultAuxTable.collect().isEmpty)
- checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L)))
+ checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L, null)))
}
test(
@@ -260,7 +260,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
assert(resultAuxTable.collect().isEmpty)
- checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L)))
+ checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L, null)))
}
//
===========================================================================================
@@ -283,8 +283,8 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
- checkAnswer(resultTargetTable, Row(2, "fresh", Row(null, 20L)))
+ checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
+ checkAnswer(resultTargetTable, Row(2, "fresh", Row(null, 20L, null)))
}
test(
@@ -301,7 +301,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultAuxTable, Row(1, Row(20L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(20L, null, null)))
assert(resultTargetTable.collect().isEmpty)
}
@@ -321,7 +321,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
assert(resultAuxTable.collect().isEmpty)
- checkAnswer(resultTargetTable, Row(1, "newer", Row(null, 20L)))
+ checkAnswer(resultTargetTable, Row(1, "newer", Row(null, 20L, null)))
}
test(
@@ -374,7 +374,7 @@ class Scd1ForeachBatchHandlerSuite
excludeExec.execute(microbatchOf(sourceSchema)(Row(1, null, 2L, true)),
batchId = 3L)
val targetTableAfterBatch4 =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(targetTableAfterBatch4, Row(1, "dave", Row(null, 4L)))
+ checkAnswer(targetTableAfterBatch4, Row(1, "dave", Row(null, 4L, null)))
// Batch 5: newer delete (seq=5) wipes the row from the target.
excludeExec.execute(microbatchOf(sourceSchema)(Row(1, null, 5L, true)),
batchId = 4L)
@@ -385,7 +385,7 @@ class Scd1ForeachBatchHandlerSuite
val auxTableAfterBatch6 =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val targetTableAfterBatch6 =
spark.read.table(defaultTargetTableIdentifier.quotedString)
assert(targetTableAfterBatch6.collect().isEmpty)
- checkAnswer(auxTableAfterBatch6, Row(1, Row(5L, null)))
+ checkAnswer(auxTableAfterBatch6, Row(1, Row(5L, null, null)))
}
test(
@@ -398,7 +398,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L)))
+ checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L, null)))
assert(resultAuxTable.collect().isEmpty)
}
@@ -417,7 +417,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
assert(resultTargetTable.collect().isEmpty)
- checkAnswer(resultAuxTable, Row(99, Row(1L, null)))
+ checkAnswer(resultAuxTable, Row(99, Row(1L, null, null)))
}
test(
@@ -431,7 +431,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L)))
+ checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L, null)))
assert(resultAuxTable.collect().isEmpty)
}
@@ -449,8 +449,8 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L)))
- checkAnswer(resultAuxTable, Row(1, Row(2L, null)))
+ checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L, null)))
+ checkAnswer(resultAuxTable, Row(1, Row(2L, null, null)))
}
test(
@@ -473,7 +473,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 3L)))
+ checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 3L, null)))
assert(resultAuxTable.collect().isEmpty)
}
@@ -536,9 +536,9 @@ class Scd1ForeachBatchHandlerSuite
checkAnswer(
resultTargetTable.orderBy("country", "city"),
Seq(
- Row("UK", "London", 9000000L, Row(null, 1L)),
- Row("US", "Los Angeles", 4000000L, Row(null, 1L)),
- Row("US", "New York", 8000000L, Row(null, 1L))
+ Row("UK", "London", 9000000L, Row(null, 1L, null)),
+ Row("US", "Los Angeles", 4000000L, Row(null, 1L, null)),
+ Row("US", "New York", 8000000L, Row(null, 1L, null))
)
)
assert(resultAuxTable.collect().isEmpty)
@@ -560,8 +560,8 @@ class Scd1ForeachBatchHandlerSuite
checkAnswer(
resultTargetTable.orderBy("id"),
Seq(
- Row(1, "alice-updated", Row(null, 2L)),
- Row(2, "bob", Row(null, 1L))
+ Row(1, "alice-updated", Row(null, 2L, null)),
+ Row(2, "bob", Row(null, 1L, null))
)
)
assert(resultAuxTable.collect().isEmpty)
@@ -613,7 +613,7 @@ class Scd1ForeachBatchHandlerSuite
val resultAuxTable =
spark.read.table(defaultAuxTableIdentifier.quotedString)
val resultTargetTable =
spark.read.table(defaultTargetTableIdentifier.quotedString)
- checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L)))
+ checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L, null)))
assert(resultAuxTable.collect().isEmpty)
}
}
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
index 0ae3b2f595e1..cf0f3af6c153 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
@@ -40,21 +40,18 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
.add("age", IntegerType)
.add(
AutoCdcReservedNames.cdcMetadataColName,
- new StructType()
- .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, LongType)
- .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, LongType)
+ Scd1BatchProcessor.cdcMetadataColSchema(LongType)
)
/** DataType for the CDC metadata column, where sequencing type is Long. */
- private val cdcMetadataColSchemaType: DataType = new StructType()
- .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, LongType)
- .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, LongType)
+ private val cdcMetadataColSchemaType: DataType =
+ Scd1BatchProcessor.cdcMetadataColSchema(LongType)
/**
* Helper to construct a CDC metadata column row, following
[[cdcMetadataColSchemaType]].
*/
private def cdcMetadataRow(deleteSeq: Option[Long], upsertSeq:
Option[Long]): Row =
- Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null))
+ Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null), null)
/** Build a microbatch [[DataFrame]] from explicit rows and an explicit
schema. */
private def microbatchOf(schema: StructType)(rows: Row*): DataFrame =
@@ -476,10 +473,10 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = extendMicrobatchRowsWithCdcMetadata(changeArgs, batch),
expectedAnswer = Seq(
- Row(1, 10L, false, Row(null, 10L)),
- Row(2, 20L, true, Row(20L, null)),
- Row(3, 30L, false, Row(null, 30L)),
- Row(4, 40L, true, Row(40L, null))
+ Row(1, 10L, false, Row(null, 10L, null)),
+ Row(2, 20L, true, Row(20L, null, null)),
+ Row(3, 30L, false, Row(null, 30L, null)),
+ Row(4, 40L, true, Row(40L, null, null))
)
)
}
@@ -503,7 +500,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = extendMicrobatchRowsWithCdcMetadata(changeArgs, batch),
- expectedAnswer = Row(1, 10L, null, Row(null, 10L))
+ expectedAnswer = Row(1, 10L, null, Row(null, 10L, null))
)
}
@@ -529,8 +526,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = extendMicrobatchRowsWithCdcMetadata(changeArgs, batch),
expectedAnswer = Seq(
- Row(1, 10L, "a", Row(null, 10L)),
- Row(2, 20L, "b", Row(null, 20L))
+ Row(1, 10L, "a", Row(null, 10L, null)),
+ Row(2, 20L, "b", Row(null, 20L, null))
)
)
}
@@ -559,7 +556,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
schema.fieldNames.toSeq :+ AutoCdcReservedNames.cdcMetadataColName)
}
- test("extendMicrobatchRowsWithCdcMetadata casts delete / upsert sequence
fields to " +
+ test("extendMicrobatchRowsWithCdcMetadata casts CDC metadata clocks to " +
"resolvedSequencingType") {
val schema = new StructType()
.add("id", IntegerType)
@@ -584,13 +581,14 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
resultDf.schema(AutoCdcReservedNames.cdcMetadataColName).dataType.asInstanceOf[StructType]
assert(columnNamesAndDataTypes(cdcMetadataDataType) == Seq(
Scd1BatchProcessor.cdcDeleteSequenceFieldName -> LongType,
- Scd1BatchProcessor.cdcUpsertSequenceFieldName -> LongType))
+ Scd1BatchProcessor.cdcUpsertSequenceFieldName -> LongType,
+ Scd1BatchProcessor.versionMapFieldName ->
Scd1VersionMap.mapType(LongType)))
// The cast must also succeed at runtime: upsertSequence is materialized
as a Long value, not
// an Int.
checkAnswer(
df = resultDf,
- expectedAnswer = Row(1, 10, "a", Row(null, 10L))
+ expectedAnswer = Row(1, 10, "a", Row(null, 10L, null))
)
}
@@ -626,8 +624,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
test("projectTargetColumnsOntoMicrobatch keeps every user column and the CDC
metadata column " +
"when columnSelection is None") {
val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
- Row(1, "alice", 30, Row(null, 10L)),
- Row(2, "bob", 25, Row(20L, null))
+ Row(1, "alice", 30, Row(null, 10L, null)),
+ Row(2, "bob", 25, Row(20L, null, null))
)
val changeArgs = ChangeArgs(
@@ -645,8 +643,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = result,
expectedAnswer = Seq(
- Row(1, "alice", 30, Row(null, 10L)),
- Row(2, "bob", 25, Row(20L, null))
+ Row(1, "alice", 30, Row(null, 10L, null)),
+ Row(2, "bob", 25, Row(20L, null, null))
)
)
}
@@ -654,7 +652,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
test("projectTargetColumnsOntoMicrobatch retains the CDC metadata column
even when " +
"IncludeColumns does not contain it") {
val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
- Row(1, "alice", 30, Row(null, 10L))
+ Row(1, "alice", 30, Row(null, 10L, null))
)
val changeArgs = ChangeArgs(
@@ -674,13 +672,13 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
Seq("id", "age", AutoCdcReservedNames.cdcMetadataColName))
checkAnswer(
df = result,
- expectedAnswer = Row(1, 30, Row(null, 10L))
+ expectedAnswer = Row(1, 30, Row(null, 10L, null))
)
}
test("projectTargetColumnsOntoMicrobatch respects exclude column") {
val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
- Row(1, "alice", 30, Row(null, 10L))
+ Row(1, "alice", 30, Row(null, 10L, null))
)
val changeArgs = ChangeArgs(
@@ -702,13 +700,13 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
)
checkAnswer(
df = result,
- expectedAnswer = Row(1, "alice", Row(null, 10L))
+ expectedAnswer = Row(1, "alice", Row(null, 10L, null))
)
}
test("projectTargetColumnsOntoMicrobatch preserves the microbatch schema
order") {
val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
- Row(1, "alice", 30, Row(null, 10L))
+ Row(1, "alice", 30, Row(null, 10L, null))
)
val changeArgs = ChangeArgs(
@@ -731,7 +729,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = result,
- expectedAnswer = Row(1, 30, Row(null, 10L))
+ expectedAnswer = Row(1, 30, Row(null, 10L, null))
)
}
@@ -745,7 +743,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
.add(AutoCdcReservedNames.cdcMetadataColName, cdcMetadataColSchemaType)
val batch = microbatchOf(schema)(
- Row(1, "u-100", Row(null, 10L))
+ Row(1, "u-100", Row(null, 10L, null))
)
val changeArgs = ChangeArgs(
@@ -768,7 +766,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
Seq("id", "user.id", AutoCdcReservedNames.cdcMetadataColName))
checkAnswer(
df = result,
- expectedAnswer = Row(1, "u-100", Row(null, 10L))
+ expectedAnswer = Row(1, "u-100", Row(null, 10L, null))
)
}
@@ -776,7 +774,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
"when SQLConf.CASE_SENSITIVE=false") {
withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
- Row(1, "alice", 30, Row(null, 10L))
+ Row(1, "alice", 30, Row(null, 10L, null))
)
val changeArgs = ChangeArgs(
@@ -799,7 +797,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
Seq("id", "age", AutoCdcReservedNames.cdcMetadataColName))
checkAnswer(
df = result,
- expectedAnswer = Row(1, 30, Row(null, 10L))
+ expectedAnswer = Row(1, 30, Row(null, 10L, null))
)
}
}
@@ -881,7 +879,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
- expectedAnswer = Row(1, "tied-upsert", Row(null, 10L))
+ expectedAnswer = Row(1, "tied-upsert", Row(null, 10L, null))
)
}
@@ -906,8 +904,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
expectedAnswer = Seq(
- Row(1, "fresher-upsert", Row(null, 15L)),
- Row(1, "fresher-delete", Row(20L, null))
+ Row(1, "fresher-upsert", Row(null, 15L, null)),
+ Row(1, "fresher-delete", Row(20L, null, null))
)
)
}
@@ -933,7 +931,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
- expectedAnswer = Row(1, "stays", Row(null, 5L))
+ expectedAnswer = Row(1, "stays", Row(null, 5L, null))
)
}
@@ -965,8 +963,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
expectedAnswer = Seq(
- Row("US", 1, Row(null, 5L)),
- Row("US", 2, Row(null, 5L))
+ Row("US", 1, Row(null, 5L, null)),
+ Row("US", 2, Row(null, 5L, null))
)
)
}
@@ -1019,8 +1017,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
expectedAnswer = Seq(
- Row(1, "kept-upsert", Row(null, 5L)),
- Row(2, "kept-delete", Row(7L, null))
+ Row(1, "kept-upsert", Row(null, 5L, null)),
+ Row(2, "kept-delete", Row(7L, null, null))
)
)
}
@@ -1048,7 +1046,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest
with SharedSparkSession
checkAnswer(
df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
- expectedAnswer = Row(1, "kept-upsert", Row(null, 5L))
+ expectedAnswer = Row(1, "kept-upsert", Row(null, 5L, null))
)
}
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
index 9eeba36b7352..4bf25794ae7b 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
@@ -151,7 +151,8 @@ trait AutoCdcGraphExecutionTestMixin extends
BeforeAndAfterEach {
val col = AutoCdcReservedNames.cdcMetadataColName
val del = Scd1BatchProcessor.cdcDeleteSequenceFieldName
val ups = Scd1BatchProcessor.cdcUpsertSequenceFieldName
- s"$col STRUCT<$del:BIGINT,$ups:BIGINT> NOT NULL"
+ val versionMap = Scd1BatchProcessor.versionMapFieldName
+ s"$col STRUCT<$del:BIGINT,$ups:BIGINT,$versionMap:MAP<STRING,BIGINT>> NOT
NULL"
}
/**
@@ -186,9 +187,11 @@ trait AutoCdcGraphExecutionTestMixin extends
BeforeAndAfterEach {
protected def insertPreloadedRow(table: String, colValues: String, sequence:
Long): Unit = {
val del = Scd1BatchProcessor.cdcDeleteSequenceFieldName
val ups = Scd1BatchProcessor.cdcUpsertSequenceFieldName
+ val versionMap = Scd1BatchProcessor.versionMapFieldName
spark.sql(
s"INSERT INTO $table SELECT $colValues, " +
- s"named_struct('$del', CAST(NULL AS BIGINT), '$ups', CAST($sequence AS
BIGINT))"
+ s"named_struct('$del', CAST(NULL AS BIGINT), '$ups', CAST($sequence AS
BIGINT), " +
+ s"'$versionMap', CAST(NULL AS MAP<STRING, BIGINT>))"
)
}
@@ -264,5 +267,5 @@ trait AutoCdcGraphExecutionTestMixin extends
BeforeAndAfterEach {
/** Build a target row's `_cdc_metadata` struct value. */
protected def cdcMeta(deleteSeq: Option[Long], upsertSeq: Option[Long]): Row
=
- Row(deleteSeq.orNull, upsertSeq.orNull)
+ Row(deleteSeq.orNull, upsertSeq.orNull, null)
}
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
index 448820443d13..dbb8e340807e 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
@@ -26,10 +26,13 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.pipelines.autocdc.{
AutoCdcReservedNames,
ColumnSelection,
+ Scd1BatchProcessor,
+ ScdType,
UnqualifiedColumnName
}
import org.apache.spark.sql.pipelines.utils.{ExecutionTest,
TestGraphRegistrationContext}
import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.LongType
/**
* Tests covering AutoCDC's interaction with non-key schema evolution across
pipeline runs. The
@@ -50,6 +53,77 @@ class AutoCdcScd1SchemaEvolutionSuite
import testImplicits._
+ test("legacy SCD1 CDC metadata schema evolves to include the version map") {
+ val targetName = s"$catalog.$namespace.target"
+ val auxiliaryName = auxTableNameFor("target")
+ val cdcMetadataCol = AutoCdcReservedNames.cdcMetadataColName
+ val deleteSequence = Scd1BatchProcessor.cdcDeleteSequenceFieldName
+ val upsertSequence = Scd1BatchProcessor.cdcUpsertSequenceFieldName
+ val legacyMetadataDdl =
+ s"$cdcMetadataCol STRUCT<$deleteSequence:BIGINT,$upsertSequence:BIGINT>
NOT NULL"
+
+ spark.sql(
+ s"CREATE TABLE $targetName " +
+ s"(id INT NOT NULL, name STRING, version BIGINT NOT NULL,
$legacyMetadataDdl)"
+ )
+ spark.sql(
+ s"""CREATE TABLE $auxiliaryName (id INT NOT NULL, $legacyMetadataDdl)
""" +
+ s"""TBLPROPERTIES (""" +
+ s"""'${AutoCdcAuxiliaryTable.scdTypePropertyKey}' =
'${ScdType.Type1.label}', """ +
+ s"""'${AutoCdcAuxiliaryTable.keyColumnNamesProperty}' = '["id"]')"""
+ )
+
+ spark.sql(
+ s"INSERT INTO $targetName VALUES " +
+ s"(1, 'alice', CAST(5 AS BIGINT), " +
+ s"named_struct('$deleteSequence', CAST(NULL AS BIGINT), " +
+ s"'$upsertSequence', CAST(5 AS BIGINT))), " +
+ s"(3, 'carol', CAST(5 AS BIGINT), " +
+ s"named_struct('$deleteSequence', CAST(NULL AS BIGINT), " +
+ s"'$upsertSequence', CAST(5 AS BIGINT)))"
+ )
+ spark.sql(
+ s"INSERT INTO $auxiliaryName VALUES " +
+ s"(2, named_struct('$deleteSequence', CAST(5 AS BIGINT), " +
+ s"'$upsertSequence', CAST(NULL AS BIGINT))), " +
+ s"(4, named_struct('$deleteSequence', CAST(5 AS BIGINT), " +
+ s"'$upsertSequence', CAST(NULL AS BIGINT)))"
+ )
+
+ val stream = MemoryStream[(Int, String, Long)]
+ // Update one legacy target row and one legacy tombstone. The other rows
remain untouched,
+ // demonstrating that both rewritten and existing metadata values acquire
a null version map.
+ stream.addData((1, "alicia", 6L), (2, "ignored-delete-value", 6L))
+ runPipeline(singleAutoCdcFlowPipeline(
+ flowName = "auto_cdc_flow",
+ target = "target",
+ sourceDf = stream.toDF().toDF("id", "name", "version"),
+ keys = Seq("id"),
+ sequencing = functions.col("version"),
+ deleteCondition = Some(functions.col("id") === 2)))
+
+ val expectedMetadataSchema =
Scd1BatchProcessor.cdcMetadataColSchema(LongType)
+ assert(
+ spark.table(targetName).schema(cdcMetadataCol).dataType ===
+ expectedMetadataSchema.asNullable)
+ assert(
+ spark.table(auxiliaryName).schema(cdcMetadataCol).dataType ===
expectedMetadataSchema)
+ checkAnswer(
+ spark.table(targetName),
+ Seq(
+ Row(1, "alicia", 6L, cdcMeta(None, Some(6L))),
+ Row(3, "carol", 5L, cdcMeta(None, Some(5L)))
+ )
+ )
+ checkAnswer(
+ spark.table(auxiliaryName),
+ Seq(
+ Row(2, cdcMeta(Some(6L), None)),
+ Row(4, cdcMeta(Some(5L), None))
+ )
+ )
+ }
+
test("a nullable non-key column merges correctly with mixed NULL and
non-NULL values") {
// Single MemoryStream with `email` as nullable from the start. Run #1
emits a row with
// a NULL email; run #2 emits an upsert with a non-NULL email.
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
index 1a924cf82464..6921d702e092 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
@@ -1433,9 +1433,9 @@ class SqlPipelineSuite extends PipelineTest with
SharedSparkSession {
}
}
- /** Build a target row's `_cdc_metadata` struct value (deleteSequence,
upsertSequence). */
+ /** Build a target row's `_cdc_metadata` struct value. */
private def cdcMeta(deleteSeq: Option[Long], upsertSeq: Option[Long]): Row =
- Row(deleteSeq.orNull, upsertSeq.orNull)
+ Row(deleteSeq.orNull, upsertSeq.orNull, null)
test("CREATE STREAMING TABLE FLOW AUTO CDC upserts rows into the target
end-to-end") {
withRowLevelAutoCdcCatalog {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]