This is an automated email from the ASF dual-hosted git repository.

jose-torres pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 425795e8f031 [SPARK-59695][SDP] SCD1 Ignore-null support; Introduce 
version map in `cdc_metadata`
425795e8f031 is described below

commit 425795e8f031a989fbdab46d0939a481c08cd893
Author: AnishMahto <[email protected]>
AuthorDate: Mon Nov 2 08:25:50 2026 -0800

    [SPARK-59695][SDP] SCD1 Ignore-null support; Introduce version map in 
`cdc_metadata`
    
    ### Why are the changes needed?
    In order to support ignore-null in SCD1, we need to persist additional 
metadata per key that tells us the last event that authored each key, if any so 
far.
    
    This is effectively per-leaf level versioning, rather than the typical 
row/key-level versioning that SCD1 uses when ignore-null is off (the default 
case). The latter relies on the single upsert/delete version per row, also in 
the CDC metadata column.
    
    This is the SCD1 equivalent change of 
https://github.com/apache/spark/pull/58483.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes. After this change existing AutoCDC tables will undergo nested schema 
evolution to gain a null version map field in their `cdc_metadata` column, when 
they are not using ignore-null (the default case).
    
    ### How was this patch tested?
    Added schema evolution test to `AutoCdcScd1SchemaEvolutionSuite`. Updated 
existing tests to reflect the additional version map field in `cdc_metadata`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Co-authored with GPT 5.6 Sol.
    
    Closes #58973 from AnishMahto/scd1-per-leaf-metadata-schema.
    
    Authored-by: AnishMahto <[email protected]>
    Signed-off-by: Jose Torres <[email protected]>
    (cherry picked from commit 030f20a4fb07b43d3ec3f9400dec036efd2c7635)
    Signed-off-by: Jose Torres <[email protected]>
---
 .../sql/pipelines/autocdc/Scd1BatchProcessor.scala | 21 +++++-
 .../autocdc/Scd1ReconciliationStrategy.scala       |  1 +
 .../sql/pipelines/autocdc/Scd1VersionMap.scala     | 40 +++++++++++
 .../apache/spark/sql/pipelines/graph/Flow.scala    |  1 +
 .../autocdc/AutoCdcCatalogExecutionTestBase.scala  |  8 +--
 .../sql/pipelines/autocdc/AutoCdcFlowSuite.scala   |  3 +
 .../autocdc/Scd1BatchProcessorMergeSuite.scala     | 61 ++++++++---------
 .../autocdc/Scd1ForeachBatchHandlerSuite.scala     | 42 ++++++------
 .../autocdc/Scd1RowLevelReconciliationSuite.scala  | 78 +++++++++++-----------
 .../graph/AutoCdcGraphExecutionTestMixin.scala     |  9 ++-
 .../graph/AutoCdcScd1SchemaEvolutionSuite.scala    | 74 ++++++++++++++++++++
 .../sql/pipelines/graph/SqlPipelineSuite.scala     |  4 +-
 12 files changed, 237 insertions(+), 105 deletions(-)

diff --git 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
index 4c93e688886e..87e92f6b2b8b 100644
--- 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
+++ 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessor.scala
@@ -215,6 +215,7 @@ case class Scd1BatchProcessor(
 object Scd1BatchProcessor {
   private[pipelines] val cdcDeleteSequenceFieldName: String = "deleteSequence"
   private[pipelines] val cdcUpsertSequenceFieldName: String = "upsertSequence"
+  private[pipelines] val versionMapFieldName: String = "__VERSION_MAP"
 
   /** Project the delete sequence out of the CDC metadata column. */
   private[autocdc] def deleteSequenceOf(cdcMetadataCol: Column): Column =
@@ -226,6 +227,10 @@ object Scd1BatchProcessor {
 
   /**
    * Schema of the CDC metadata struct column for SCD1.
+   *
+   * The CDC metadata of every microbatch, target, and auxiliary-table row 
represents either a
+   * delete or an upsert, never both: exactly one of its delete and upsert 
sequences is non-null,
+   * and only an upsert may carry a version map.
    */
   private[pipelines] def cdcMetadataColSchema(sequencingType: DataType): 
StructType =
     StructType(
@@ -233,22 +238,34 @@ object Scd1BatchProcessor {
         // The sequencing of the event if it represents a delete, null 
otherwise.
         StructField(cdcDeleteSequenceFieldName, sequencingType, nullable = 
true),
         // The sequencing of the event if it represents an upsert, null 
otherwise.
-        StructField(cdcUpsertSequenceFieldName, sequencingType, nullable = 
true)
+        StructField(cdcUpsertSequenceFieldName, sequencingType, nullable = 
true),
+        // On target rows, a null version map indicates row-level 
reconciliation; a non-null map
+        // indicates per-leaf reconciliation. Delete rows, including 
auxiliary-table tombstones,
+        // always use a null map: a delete's sequence applies to every leaf, 
so a map would only
+        // repeat it.
+        StructField(
+          versionMapFieldName,
+          Scd1VersionMap.mapType(sequencingType),
+          nullable = true
+        )
       )
     )
 
   /**
    * Construct the CDC metadata struct column for SCD1, following the exact 
schema and field
-   * ordering defined by [[cdcMetadataColSchema]].
+   * ordering defined by [[cdcMetadataColSchema]]. Values for microbatch, 
target, and
+   * auxiliary-table rows must uphold its contract.
    */
   private[pipelines] def constructCdcMetadataCol(
       deleteSequence: Column,
       upsertSequence: Column,
+      versionMap: Column,
       sequencingType: DataType): Column = {
     val cdcMetadataFieldsInOrder = 
cdcMetadataColSchema(sequencingType).fields.map { field =>
       val value = field.name match {
         case `cdcDeleteSequenceFieldName` => deleteSequence
         case `cdcUpsertSequenceFieldName` => upsertSequence
+        case `versionMapFieldName` => versionMap
         case other =>
           throw SparkException.internalError(
             s"Unable to construct SCD1 CDC metadata column due to unknown 
`${other}` field."
diff --git 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
index 8dbbdb51630e..7444bccd72c8 100644
--- 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
+++ 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ReconciliationStrategy.scala
@@ -74,6 +74,7 @@ private[pipelines] trait Scd1ReconciliationStrategy {
       Scd1BatchProcessor.constructCdcMetadataCol(
         deleteSequence = rowDeleteSequence,
         upsertSequence = rowUpsertSequence,
+        versionMap = F.lit(null),
         sequencingType = resolvedSequencingType
       )
     )
diff --git 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
new file mode 100644
index 000000000000..e2df5155ee96
--- /dev/null
+++ 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.pipelines.autocdc
+
+import org.apache.spark.sql.types.{DataType, MapType, StringType}
+
+/**
+ * Per-leaf authorship sequences for SCD1 reconciliation.
+ *
+ * An SCD1 target row can combine values authored by different CDC events. The 
version map records
+ * which event currently authors each non-key user-data leaf.
+ */
+private[pipelines] object Scd1VersionMap {
+
+  /**
+   * Schema of the version map: `Map(String, sequencingType)`.
+   *
+   * Each key is the field path of a user-data leaf. Each value is the leaf's 
authorship sequence:
+   * the sequencing value of the event that authored the leaf, or null if no 
event has authored it.
+   *
+   * @param sequencingType The resolved data type of the flow's sequencing 
expression.
+   */
+  def mapType(sequencingType: DataType): MapType =
+    MapType(StringType, sequencingType, valueContainsNull = true)
+}
diff --git 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
index 98eba8a78bd4..1442afd6c432 100644
--- 
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
+++ 
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/Flow.scala
@@ -391,6 +391,7 @@ class AutoCdcMergeFlow(
         val emptyCdcMetadataCol: Column = 
Scd1BatchProcessor.constructCdcMetadataCol(
           deleteSequence = F.lit(null),
           upsertSequence = F.lit(null),
+          versionMap = F.lit(null),
           sequencingType = sequencingType
         ).as(AutoCdcReservedNames.cdcMetadataColName)
 
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
index 8688df071113..a73eba8ee721 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcCatalogExecutionTestBase.scala
@@ -87,16 +87,14 @@ trait AutoCdcCatalogExecutionTestBase {
    * `Long` sequencing.
    */
   protected def cdcMetadataColSchemaType(sequencingType: DataType = LongType): 
StructType =
-    new StructType()
-      .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, sequencingType)
-      .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, sequencingType)
+    Scd1BatchProcessor.cdcMetadataColSchema(sequencingType)
 
   /**
-   * Build a [[Row]] matching the [[AutoCdcReservedNames.cdcMetadataColName]] 
struct's two fields,
+   * Build a [[Row]] matching the [[AutoCdcReservedNames.cdcMetadataColName]] 
struct's three fields,
    * in the order produced by [[Scd1BatchProcessor.constructCdcMetadataCol]]:
    */
   protected def cdcMetadataRow[T](deleteSeq: Option[T], upsertSeq: Option[T]): 
Row =
-    Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null))
+    Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null), null)
 
   /**
    * Create a table in the test catalog under the given DSv2 [[Identifier]] 
using `schema`,
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
index c20dca119eb8..05b838996d9b 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcFlowSuite.scala
@@ -283,6 +283,9 @@ class AutoCdcFlowSuite extends QueryTest with 
SharedSparkSession {
     val metaStruct = metaField.dataType.asInstanceOf[StructType]
     assert(metaStruct(Scd1BatchProcessor.cdcDeleteSequenceFieldName).nullable)
     assert(metaStruct(Scd1BatchProcessor.cdcUpsertSequenceFieldName).nullable)
+    val versionMapField = metaStruct(Scd1BatchProcessor.versionMapFieldName)
+    assert(versionMapField.nullable)
+    assert(versionMapField.dataType == Scd1VersionMap.mapType(LongType))
   }
 
   test("AutoCdcMergeFlow.schema is stable across reads") {
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
index 9cfe42267f8b..3e381a7297dd 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1BatchProcessorMergeSuite.scala
@@ -115,12 +115,7 @@ class Scd1BatchProcessorMergeSuite
     val microbatchSchema = new StructType()
       .add("id", IntegerType)
       .add("value", StringType)
-      .add(
-        AutoCdcReservedNames.cdcMetadataColName,
-        new StructType()
-          .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, LongType)
-          .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, LongType)
-      )
+      .add(AutoCdcReservedNames.cdcMetadataColName, cdcMetadataColSchemaType())
     val microbatch = microbatchOf(microbatchSchema)(
       Row(1, "data-leak", cdcMetadataRow(deleteSeq = Some(20L), upsertSeq = 
None))
     )
@@ -129,7 +124,7 @@ class Scd1BatchProcessorMergeSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     // Tombstone advanced to delete=20, with exactly one row per key (no 
duplicate tombstones).
-    checkAnswer(resultAuxTable, Row(1, Row(20L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(20L, null, null)))
     // Schema strictly matches minimalSchema; the `value` column was dropped, 
not smuggled in.
     assert(columnNamesAndDataTypes(resultAuxTable.schema) == 
columnNamesAndDataTypes(minimalSchema))
   }
@@ -159,7 +154,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoAuxiliaryTable(microbatch, 
defaultAuxTableIdentifier)
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
-    checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
   }
 
   test("mergeMicrobatchOntoAuxiliaryTable leaves rows for unrelated keys 
untouched") {
@@ -175,8 +170,8 @@ class Scd1BatchProcessorMergeSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     checkAnswer(resultAuxTable, Seq(
-      Row(1, Row(10L, null)),
-      Row(2, Row(100L, null))
+      Row(1, Row(10L, null, null)),
+      Row(2, Row(100L, null, null))
     ))
   }
 
@@ -194,7 +189,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoAuxiliaryTable(microbatch, 
defaultAuxTableIdentifier)
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
-    checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
   }
 
   test("mergeMicrobatchOntoAuxiliaryTable ignores microbatch upserts whose 
sequence is older " +
@@ -208,7 +203,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoAuxiliaryTable(microbatch, 
defaultAuxTableIdentifier)
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
-    checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
   }
 
   test("mergeMicrobatchOntoAuxiliaryTable applies the tied-sequence asymmetry: 
equal deletes " +
@@ -232,7 +227,7 @@ class Scd1BatchProcessorMergeSuite
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     // Row 1's tombstone remains the same, but row 2's tombstone should be 
marked as stale and
     // deleted.
-    checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
   }
 
   test("mergeMicrobatchOntoAuxiliaryTable upsert event for different key does 
not affect " +
@@ -247,7 +242,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoAuxiliaryTable(microbatch, 
defaultAuxTableIdentifier)
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
-    checkAnswer(resultAuxTable, Row(2, Row(5L, null)))
+    checkAnswer(resultAuxTable, Row(2, Row(5L, null, null)))
   }
 
   test("mergeMicrobatchOntoAuxiliaryTable is idempotent across a microbatch 
that exercises " +
@@ -271,9 +266,9 @@ class Scd1BatchProcessorMergeSuite
     )
 
     val expectedAfterMerge = Seq(
-      Row(2, Row(25L, null)),
-      Row(3, Row(30L, null)),
-      Row(4, Row(40L, null))
+      Row(2, Row(25L, null, null)),
+      Row(3, Row(30L, null, null)),
+      Row(4, Row(40L, null, null))
     )
 
     // First merge applies all three clauses exactly once.
@@ -319,8 +314,8 @@ class Scd1BatchProcessorMergeSuite
     compositeKeyProcessor.mergeMicrobatchOntoAuxiliaryTable(microbatch, 
defaultAuxTableIdentifier)
 
     checkAnswer(spark.read.table(defaultAuxTableIdentifier.quotedString), Seq(
-      Row("US", 99, Row(50L, null)),
-      Row("US", 1, Row(10L, null))
+      Row("US", 99, Row(50L, null, null)),
+      Row("US", 1, Row(10L, null, null))
     ))
   }
 
@@ -352,7 +347,9 @@ class Scd1BatchProcessorMergeSuite
 
     dottedKeyProcessor.mergeMicrobatchOntoAuxiliaryTable(microbatch, 
defaultAuxTableIdentifier)
 
-    checkAnswer(spark.read.table(defaultAuxTableIdentifier.quotedString), 
Row(1, Row(20L, null)))
+    checkAnswer(
+      spark.read.table(defaultAuxTableIdentifier.quotedString),
+      Row(1, Row(20L, null, null)))
   }
 
   // =============== mergeMicrobatchOntoTarget tests ===============
@@ -367,7 +364,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoTarget(microbatch, 
defaultTargetTableIdentifier)
 
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L)))
+    checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L, null)))
     assert(columnNamesAndDataTypes(resultTargetTable.schema) ==
       columnNamesAndDataTypes(targetSchema))
   }
@@ -385,7 +382,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoTarget(microbatch, 
defaultTargetTableIdentifier)
 
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(2, "keep-me", Row(null, 20L)))
+    checkAnswer(resultTargetTable, Row(2, "keep-me", Row(null, 20L, null)))
   }
 
   test("mergeMicrobatchOntoTarget inserts new upserts but not new (tombstone) 
deletes") {
@@ -400,8 +397,8 @@ class Scd1BatchProcessorMergeSuite
 
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     checkAnswer(resultTargetTable, Seq(
-      Row(1, "existing", Row(null, 10L)),
-      Row(2, "insert-me", Row(null, 20L))
+      Row(1, "existing", Row(null, 10L, null)),
+      Row(2, "insert-me", Row(null, 20L, null))
     ))
   }
 
@@ -420,8 +417,8 @@ class Scd1BatchProcessorMergeSuite
 
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     checkAnswer(resultTargetTable, Seq(
-      Row(1, "target-delete-tie", Row(null, 10L)),
-      Row(2, "target-newer", Row(null, 20L))
+      Row(1, "target-delete-tie", Row(null, 10L, null)),
+      Row(2, "target-newer", Row(null, 20L, null))
     ))
   }
 
@@ -435,7 +432,7 @@ class Scd1BatchProcessorMergeSuite
     processor.mergeMicrobatchOntoTarget(microbatch, 
defaultTargetTableIdentifier)
 
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "same-sequence-upsert", Row(null, 
10L)))
+    checkAnswer(resultTargetTable, Row(1, "same-sequence-upsert", Row(null, 
10L, null)))
   }
 
   test("mergeMicrobatchOntoTarget correctly matches escaped key column names") 
{
@@ -470,7 +467,7 @@ class Scd1BatchProcessorMergeSuite
     )
 
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L)))
+    checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L, null)))
   }
 
   gridTest(
@@ -505,7 +502,7 @@ class Scd1BatchProcessorMergeSuite
           defaultTargetTableIdentifier
         )
         val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-        checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L)))
+        checkAnswer(resultTargetTable, Row(1, "new", Row(null, 20L, null)))
       }
     }
   }
@@ -525,9 +522,9 @@ class Scd1BatchProcessorMergeSuite
     )
 
     val expectedAfterMerge = Seq(
-      Row(2, "updated", Row(null, 25L)),
-      Row(3, "untouched", Row(null, 30L)),
-      Row(4, "inserted", Row(null, 40L))
+      Row(2, "updated", Row(null, 25L, null)),
+      Row(3, "untouched", Row(null, 30L, null)),
+      Row(4, "inserted", Row(null, 40L, null))
     )
 
     processor.mergeMicrobatchOntoTarget(microbatch, 
defaultTargetTableIdentifier)
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
index bb8043e720c6..ebc2dcd77651 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1ForeachBatchHandlerSuite.scala
@@ -107,7 +107,7 @@ class Scd1ForeachBatchHandlerSuite
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     assert(resultAuxTable.collect().isEmpty)
-    checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L)))
+    checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L, null)))
   }
 
   test(
@@ -139,7 +139,7 @@ class Scd1ForeachBatchHandlerSuite
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     assert(resultAuxTable.collect().isEmpty)
-    checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L)))
+    checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L, null)))
   }
 
   test(
@@ -260,7 +260,7 @@ class Scd1ForeachBatchHandlerSuite
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     assert(resultAuxTable.collect().isEmpty)
-    checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L)))
+    checkAnswer(resultTargetTable, Row(1, "old", Row(null, 10L, null)))
   }
 
   // 
===========================================================================================
@@ -283,8 +283,8 @@ class Scd1ForeachBatchHandlerSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultAuxTable, Row(1, Row(10L, null)))
-    checkAnswer(resultTargetTable, Row(2, "fresh", Row(null, 20L)))
+    checkAnswer(resultAuxTable, Row(1, Row(10L, null, null)))
+    checkAnswer(resultTargetTable, Row(2, "fresh", Row(null, 20L, null)))
   }
 
   test(
@@ -301,7 +301,7 @@ class Scd1ForeachBatchHandlerSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultAuxTable, Row(1, Row(20L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(20L, null, null)))
     assert(resultTargetTable.collect().isEmpty)
   }
 
@@ -321,7 +321,7 @@ class Scd1ForeachBatchHandlerSuite
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     assert(resultAuxTable.collect().isEmpty)
-    checkAnswer(resultTargetTable, Row(1, "newer", Row(null, 20L)))
+    checkAnswer(resultTargetTable, Row(1, "newer", Row(null, 20L, null)))
   }
 
   test(
@@ -374,7 +374,7 @@ class Scd1ForeachBatchHandlerSuite
     excludeExec.execute(microbatchOf(sourceSchema)(Row(1, null, 2L, true)), 
batchId = 3L)
 
     val targetTableAfterBatch4 = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(targetTableAfterBatch4, Row(1, "dave", Row(null, 4L)))
+    checkAnswer(targetTableAfterBatch4, Row(1, "dave", Row(null, 4L, null)))
 
     // Batch 5: newer delete (seq=5) wipes the row from the target.
     excludeExec.execute(microbatchOf(sourceSchema)(Row(1, null, 5L, true)), 
batchId = 4L)
@@ -385,7 +385,7 @@ class Scd1ForeachBatchHandlerSuite
     val auxTableAfterBatch6 = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val targetTableAfterBatch6 = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     assert(targetTableAfterBatch6.collect().isEmpty)
-    checkAnswer(auxTableAfterBatch6, Row(1, Row(5L, null)))
+    checkAnswer(auxTableAfterBatch6, Row(1, Row(5L, null, null)))
   }
 
   test(
@@ -398,7 +398,7 @@ class Scd1ForeachBatchHandlerSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L)))
+    checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L, null)))
     assert(resultAuxTable.collect().isEmpty)
   }
 
@@ -417,7 +417,7 @@ class Scd1ForeachBatchHandlerSuite
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
     assert(resultTargetTable.collect().isEmpty)
-    checkAnswer(resultAuxTable, Row(99, Row(1L, null)))
+    checkAnswer(resultAuxTable, Row(99, Row(1L, null, null)))
   }
 
   test(
@@ -431,7 +431,7 @@ class Scd1ForeachBatchHandlerSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L)))
+    checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L, null)))
     assert(resultAuxTable.collect().isEmpty)
   }
 
@@ -449,8 +449,8 @@ class Scd1ForeachBatchHandlerSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L)))
-    checkAnswer(resultAuxTable, Row(1, Row(2L, null)))
+    checkAnswer(resultTargetTable, Row(1, "alice", Row(null, 5L, null)))
+    checkAnswer(resultAuxTable, Row(1, Row(2L, null, null)))
   }
 
   test(
@@ -473,7 +473,7 @@ class Scd1ForeachBatchHandlerSuite
 
     val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
     val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-    checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 3L)))
+    checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 3L, null)))
     assert(resultAuxTable.collect().isEmpty)
   }
 
@@ -536,9 +536,9 @@ class Scd1ForeachBatchHandlerSuite
     checkAnswer(
       resultTargetTable.orderBy("country", "city"),
       Seq(
-        Row("UK", "London", 9000000L, Row(null, 1L)),
-        Row("US", "Los Angeles", 4000000L, Row(null, 1L)),
-        Row("US", "New York", 8000000L, Row(null, 1L))
+        Row("UK", "London", 9000000L, Row(null, 1L, null)),
+        Row("US", "Los Angeles", 4000000L, Row(null, 1L, null)),
+        Row("US", "New York", 8000000L, Row(null, 1L, null))
       )
     )
     assert(resultAuxTable.collect().isEmpty)
@@ -560,8 +560,8 @@ class Scd1ForeachBatchHandlerSuite
     checkAnswer(
       resultTargetTable.orderBy("id"),
       Seq(
-        Row(1, "alice-updated", Row(null, 2L)),
-        Row(2, "bob", Row(null, 1L))
+        Row(1, "alice-updated", Row(null, 2L, null)),
+        Row(2, "bob", Row(null, 1L, null))
       )
     )
     assert(resultAuxTable.collect().isEmpty)
@@ -613,7 +613,7 @@ class Scd1ForeachBatchHandlerSuite
 
       val resultAuxTable = 
spark.read.table(defaultAuxTableIdentifier.quotedString)
       val resultTargetTable = 
spark.read.table(defaultTargetTableIdentifier.quotedString)
-      checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L)))
+      checkAnswer(resultTargetTable, Row(1, "bob", Row(null, 2L, null)))
       assert(resultAuxTable.collect().isEmpty)
     }
   }
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
index 0ae3b2f595e1..cf0f3af6c153 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1RowLevelReconciliationSuite.scala
@@ -40,21 +40,18 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     .add("age", IntegerType)
     .add(
       AutoCdcReservedNames.cdcMetadataColName,
-      new StructType()
-        .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, LongType)
-        .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, LongType)
+      Scd1BatchProcessor.cdcMetadataColSchema(LongType)
     )
 
   /** DataType for the CDC metadata column, where sequencing type is Long. */
-  private val cdcMetadataColSchemaType: DataType = new StructType()
-    .add(Scd1BatchProcessor.cdcDeleteSequenceFieldName, LongType)
-    .add(Scd1BatchProcessor.cdcUpsertSequenceFieldName, LongType)
+  private val cdcMetadataColSchemaType: DataType =
+    Scd1BatchProcessor.cdcMetadataColSchema(LongType)
 
   /**
    * Helper to construct a CDC metadata column row, following 
[[cdcMetadataColSchemaType]].
    */
   private def cdcMetadataRow(deleteSeq: Option[Long], upsertSeq: 
Option[Long]): Row =
-    Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null))
+    Row(deleteSeq.getOrElse(null), upsertSeq.getOrElse(null), null)
 
   /** Build a microbatch [[DataFrame]] from explicit rows and an explicit 
schema. */
   private def microbatchOf(schema: StructType)(rows: Row*): DataFrame =
@@ -476,10 +473,10 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     checkAnswer(
       df = extendMicrobatchRowsWithCdcMetadata(changeArgs, batch),
       expectedAnswer = Seq(
-        Row(1, 10L, false, Row(null, 10L)),
-        Row(2, 20L, true, Row(20L, null)),
-        Row(3, 30L, false, Row(null, 30L)),
-        Row(4, 40L, true, Row(40L, null))
+        Row(1, 10L, false, Row(null, 10L, null)),
+        Row(2, 20L, true, Row(20L, null, null)),
+        Row(3, 30L, false, Row(null, 30L, null)),
+        Row(4, 40L, true, Row(40L, null, null))
       )
     )
   }
@@ -503,7 +500,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
 
     checkAnswer(
       df = extendMicrobatchRowsWithCdcMetadata(changeArgs, batch),
-      expectedAnswer = Row(1, 10L, null, Row(null, 10L))
+      expectedAnswer = Row(1, 10L, null, Row(null, 10L, null))
     )
   }
 
@@ -529,8 +526,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     checkAnswer(
       df = extendMicrobatchRowsWithCdcMetadata(changeArgs, batch),
       expectedAnswer = Seq(
-        Row(1, 10L, "a", Row(null, 10L)),
-        Row(2, 20L, "b", Row(null, 20L))
+        Row(1, 10L, "a", Row(null, 10L, null)),
+        Row(2, 20L, "b", Row(null, 20L, null))
       )
     )
   }
@@ -559,7 +556,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
       schema.fieldNames.toSeq :+ AutoCdcReservedNames.cdcMetadataColName)
   }
 
-  test("extendMicrobatchRowsWithCdcMetadata casts delete / upsert sequence 
fields to " +
+  test("extendMicrobatchRowsWithCdcMetadata casts CDC metadata clocks to " +
     "resolvedSequencingType") {
     val schema = new StructType()
       .add("id", IntegerType)
@@ -584,13 +581,14 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
       
resultDf.schema(AutoCdcReservedNames.cdcMetadataColName).dataType.asInstanceOf[StructType]
     assert(columnNamesAndDataTypes(cdcMetadataDataType) == Seq(
       Scd1BatchProcessor.cdcDeleteSequenceFieldName -> LongType,
-      Scd1BatchProcessor.cdcUpsertSequenceFieldName -> LongType))
+      Scd1BatchProcessor.cdcUpsertSequenceFieldName -> LongType,
+      Scd1BatchProcessor.versionMapFieldName -> 
Scd1VersionMap.mapType(LongType)))
 
     // The cast must also succeed at runtime: upsertSequence is materialized 
as a Long value, not
     // an Int.
     checkAnswer(
       df = resultDf,
-      expectedAnswer = Row(1, 10, "a", Row(null, 10L))
+      expectedAnswer = Row(1, 10, "a", Row(null, 10L, null))
     )
   }
 
@@ -626,8 +624,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
   test("projectTargetColumnsOntoMicrobatch keeps every user column and the CDC 
metadata column " +
     "when columnSelection is None") {
     val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
-      Row(1, "alice", 30, Row(null, 10L)),
-      Row(2, "bob", 25, Row(20L, null))
+      Row(1, "alice", 30, Row(null, 10L, null)),
+      Row(2, "bob", 25, Row(20L, null, null))
     )
 
     val changeArgs = ChangeArgs(
@@ -645,8 +643,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     checkAnswer(
       df = result,
       expectedAnswer = Seq(
-        Row(1, "alice", 30, Row(null, 10L)),
-        Row(2, "bob", 25, Row(20L, null))
+        Row(1, "alice", 30, Row(null, 10L, null)),
+        Row(2, "bob", 25, Row(20L, null, null))
       )
     )
   }
@@ -654,7 +652,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
   test("projectTargetColumnsOntoMicrobatch retains the CDC metadata column 
even when " +
     "IncludeColumns does not contain it") {
     val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
-      Row(1, "alice", 30, Row(null, 10L))
+      Row(1, "alice", 30, Row(null, 10L, null))
     )
 
     val changeArgs = ChangeArgs(
@@ -674,13 +672,13 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
       Seq("id", "age", AutoCdcReservedNames.cdcMetadataColName))
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, 30, Row(null, 10L))
+      expectedAnswer = Row(1, 30, Row(null, 10L, null))
     )
   }
 
   test("projectTargetColumnsOntoMicrobatch respects exclude column") {
     val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
-      Row(1, "alice", 30, Row(null, 10L))
+      Row(1, "alice", 30, Row(null, 10L, null))
     )
 
     val changeArgs = ChangeArgs(
@@ -702,13 +700,13 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     )
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, "alice", Row(null, 10L))
+      expectedAnswer = Row(1, "alice", Row(null, 10L, null))
     )
   }
 
   test("projectTargetColumnsOntoMicrobatch preserves the microbatch schema 
order") {
     val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
-      Row(1, "alice", 30, Row(null, 10L))
+      Row(1, "alice", 30, Row(null, 10L, null))
     )
 
     val changeArgs = ChangeArgs(
@@ -731,7 +729,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
 
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, 30, Row(null, 10L))
+      expectedAnswer = Row(1, 30, Row(null, 10L, null))
     )
   }
 
@@ -745,7 +743,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
       .add(AutoCdcReservedNames.cdcMetadataColName, cdcMetadataColSchemaType)
 
     val batch = microbatchOf(schema)(
-      Row(1, "u-100", Row(null, 10L))
+      Row(1, "u-100", Row(null, 10L, null))
     )
 
     val changeArgs = ChangeArgs(
@@ -768,7 +766,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
       Seq("id", "user.id", AutoCdcReservedNames.cdcMetadataColName))
     checkAnswer(
       df = result,
-      expectedAnswer = Row(1, "u-100", Row(null, 10L))
+      expectedAnswer = Row(1, "u-100", Row(null, 10L, null))
     )
   }
 
@@ -776,7 +774,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     "when SQLConf.CASE_SENSITIVE=false") {
     withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
       val batch = microbatchOf(microbatchWithCdcMetadataSchema)(
-        Row(1, "alice", 30, Row(null, 10L))
+        Row(1, "alice", 30, Row(null, 10L, null))
       )
 
       val changeArgs = ChangeArgs(
@@ -799,7 +797,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
         Seq("id", "age", AutoCdcReservedNames.cdcMetadataColName))
       checkAnswer(
         df = result,
-        expectedAnswer = Row(1, 30, Row(null, 10L))
+        expectedAnswer = Row(1, 30, Row(null, 10L, null))
       )
     }
   }
@@ -881,7 +879,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
 
     checkAnswer(
       df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
-      expectedAnswer = Row(1, "tied-upsert", Row(null, 10L))
+      expectedAnswer = Row(1, "tied-upsert", Row(null, 10L, null))
     )
   }
 
@@ -906,8 +904,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     checkAnswer(
       df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
       expectedAnswer = Seq(
-        Row(1, "fresher-upsert", Row(null, 15L)),
-        Row(1, "fresher-delete", Row(20L, null))
+        Row(1, "fresher-upsert", Row(null, 15L, null)),
+        Row(1, "fresher-delete", Row(20L, null, null))
       )
     )
   }
@@ -933,7 +931,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
 
     checkAnswer(
       df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
-      expectedAnswer = Row(1, "stays", Row(null, 5L))
+      expectedAnswer = Row(1, "stays", Row(null, 5L, null))
     )
   }
 
@@ -965,8 +963,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     checkAnswer(
       df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
       expectedAnswer = Seq(
-        Row("US", 1, Row(null, 5L)),
-        Row("US", 2, Row(null, 5L))
+        Row("US", 1, Row(null, 5L, null)),
+        Row("US", 2, Row(null, 5L, null))
       )
     )
   }
@@ -1019,8 +1017,8 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
     checkAnswer(
       df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
       expectedAnswer = Seq(
-        Row(1, "kept-upsert", Row(null, 5L)),
-        Row(2, "kept-delete", Row(7L, null))
+        Row(1, "kept-upsert", Row(null, 5L, null)),
+        Row(2, "kept-delete", Row(7L, null, null))
       )
     )
   }
@@ -1048,7 +1046,7 @@ class Scd1RowLevelReconciliationSuite extends QueryTest 
with SharedSparkSession
 
     checkAnswer(
       df = applyTombstonesToMicrobatch(changeArgs, microbatch, auxiliary),
-      expectedAnswer = Row(1, "kept-upsert", Row(null, 5L))
+      expectedAnswer = Row(1, "kept-upsert", Row(null, 5L, null))
     )
   }
 
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
index 9eeba36b7352..4bf25794ae7b 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcGraphExecutionTestMixin.scala
@@ -151,7 +151,8 @@ trait AutoCdcGraphExecutionTestMixin extends 
BeforeAndAfterEach {
     val col = AutoCdcReservedNames.cdcMetadataColName
     val del = Scd1BatchProcessor.cdcDeleteSequenceFieldName
     val ups = Scd1BatchProcessor.cdcUpsertSequenceFieldName
-    s"$col STRUCT<$del:BIGINT,$ups:BIGINT> NOT NULL"
+    val versionMap = Scd1BatchProcessor.versionMapFieldName
+    s"$col STRUCT<$del:BIGINT,$ups:BIGINT,$versionMap:MAP<STRING,BIGINT>> NOT 
NULL"
   }
 
   /**
@@ -186,9 +187,11 @@ trait AutoCdcGraphExecutionTestMixin extends 
BeforeAndAfterEach {
   protected def insertPreloadedRow(table: String, colValues: String, sequence: 
Long): Unit = {
     val del = Scd1BatchProcessor.cdcDeleteSequenceFieldName
     val ups = Scd1BatchProcessor.cdcUpsertSequenceFieldName
+    val versionMap = Scd1BatchProcessor.versionMapFieldName
     spark.sql(
       s"INSERT INTO $table SELECT $colValues, " +
-      s"named_struct('$del', CAST(NULL AS BIGINT), '$ups', CAST($sequence AS 
BIGINT))"
+      s"named_struct('$del', CAST(NULL AS BIGINT), '$ups', CAST($sequence AS 
BIGINT), " +
+      s"'$versionMap', CAST(NULL AS MAP<STRING, BIGINT>))"
     )
   }
 
@@ -264,5 +267,5 @@ trait AutoCdcGraphExecutionTestMixin extends 
BeforeAndAfterEach {
 
   /** Build a target row's `_cdc_metadata` struct value. */
   protected def cdcMeta(deleteSeq: Option[Long], upsertSeq: Option[Long]): Row 
=
-    Row(deleteSeq.orNull, upsertSeq.orNull)
+    Row(deleteSeq.orNull, upsertSeq.orNull, null)
 }
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
index 448820443d13..dbb8e340807e 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/AutoCdcScd1SchemaEvolutionSuite.scala
@@ -26,10 +26,13 @@ import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.pipelines.autocdc.{
   AutoCdcReservedNames,
   ColumnSelection,
+  Scd1BatchProcessor,
+  ScdType,
   UnqualifiedColumnName
 }
 import org.apache.spark.sql.pipelines.utils.{ExecutionTest, 
TestGraphRegistrationContext}
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types.LongType
 
 /**
  * Tests covering AutoCDC's interaction with non-key schema evolution across 
pipeline runs. The
@@ -50,6 +53,77 @@ class AutoCdcScd1SchemaEvolutionSuite
 
   import testImplicits._
 
+  test("legacy SCD1 CDC metadata schema evolves to include the version map") {
+    val targetName = s"$catalog.$namespace.target"
+    val auxiliaryName = auxTableNameFor("target")
+    val cdcMetadataCol = AutoCdcReservedNames.cdcMetadataColName
+    val deleteSequence = Scd1BatchProcessor.cdcDeleteSequenceFieldName
+    val upsertSequence = Scd1BatchProcessor.cdcUpsertSequenceFieldName
+    val legacyMetadataDdl =
+      s"$cdcMetadataCol STRUCT<$deleteSequence:BIGINT,$upsertSequence:BIGINT> 
NOT NULL"
+
+    spark.sql(
+      s"CREATE TABLE $targetName " +
+      s"(id INT NOT NULL, name STRING, version BIGINT NOT NULL, 
$legacyMetadataDdl)"
+    )
+    spark.sql(
+      s"""CREATE TABLE $auxiliaryName (id INT NOT NULL, $legacyMetadataDdl) 
""" +
+      s"""TBLPROPERTIES (""" +
+      s"""'${AutoCdcAuxiliaryTable.scdTypePropertyKey}' = 
'${ScdType.Type1.label}', """ +
+      s"""'${AutoCdcAuxiliaryTable.keyColumnNamesProperty}' = '["id"]')"""
+    )
+
+    spark.sql(
+      s"INSERT INTO $targetName VALUES " +
+      s"(1, 'alice', CAST(5 AS BIGINT), " +
+      s"named_struct('$deleteSequence', CAST(NULL AS BIGINT), " +
+      s"'$upsertSequence', CAST(5 AS BIGINT))), " +
+      s"(3, 'carol', CAST(5 AS BIGINT), " +
+      s"named_struct('$deleteSequence', CAST(NULL AS BIGINT), " +
+      s"'$upsertSequence', CAST(5 AS BIGINT)))"
+    )
+    spark.sql(
+      s"INSERT INTO $auxiliaryName VALUES " +
+      s"(2, named_struct('$deleteSequence', CAST(5 AS BIGINT), " +
+      s"'$upsertSequence', CAST(NULL AS BIGINT))), " +
+      s"(4, named_struct('$deleteSequence', CAST(5 AS BIGINT), " +
+      s"'$upsertSequence', CAST(NULL AS BIGINT)))"
+    )
+
+    val stream = MemoryStream[(Int, String, Long)]
+    // Update one legacy target row and one legacy tombstone. The other rows 
remain untouched,
+    // demonstrating that both rewritten and existing metadata values acquire 
a null version map.
+    stream.addData((1, "alicia", 6L), (2, "ignored-delete-value", 6L))
+    runPipeline(singleAutoCdcFlowPipeline(
+      flowName = "auto_cdc_flow",
+      target = "target",
+      sourceDf = stream.toDF().toDF("id", "name", "version"),
+      keys = Seq("id"),
+      sequencing = functions.col("version"),
+      deleteCondition = Some(functions.col("id") === 2)))
+
+    val expectedMetadataSchema = 
Scd1BatchProcessor.cdcMetadataColSchema(LongType)
+    assert(
+      spark.table(targetName).schema(cdcMetadataCol).dataType ===
+        expectedMetadataSchema.asNullable)
+    assert(
+      spark.table(auxiliaryName).schema(cdcMetadataCol).dataType === 
expectedMetadataSchema)
+    checkAnswer(
+      spark.table(targetName),
+      Seq(
+        Row(1, "alicia", 6L, cdcMeta(None, Some(6L))),
+        Row(3, "carol", 5L, cdcMeta(None, Some(5L)))
+      )
+    )
+    checkAnswer(
+      spark.table(auxiliaryName),
+      Seq(
+        Row(2, cdcMeta(Some(6L), None)),
+        Row(4, cdcMeta(Some(5L), None))
+      )
+    )
+  }
+
   test("a nullable non-key column merges correctly with mixed NULL and 
non-NULL values") {
     // Single MemoryStream with `email` as nullable from the start. Run #1 
emits a row with
     // a NULL email; run #2 emits an upsert with a non-NULL email.
diff --git 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
index 1a924cf82464..6921d702e092 100644
--- 
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
+++ 
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/graph/SqlPipelineSuite.scala
@@ -1433,9 +1433,9 @@ class SqlPipelineSuite extends PipelineTest with 
SharedSparkSession {
     }
   }
 
-  /** Build a target row's `_cdc_metadata` struct value (deleteSequence, 
upsertSequence). */
+  /** Build a target row's `_cdc_metadata` struct value. */
   private def cdcMeta(deleteSeq: Option[Long], upsertSeq: Option[Long]): Row =
-    Row(deleteSeq.orNull, upsertSeq.orNull)
+    Row(deleteSeq.orNull, upsertSeq.orNull, null)
 
   test("CREATE STREAMING TABLE FLOW AUTO CDC upserts rows into the target 
end-to-end") {
     withRowLevelAutoCdcCatalog {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to