This is an automated email from the ASF dual-hosted git repository.
jose-torres pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 897c70271fbb [SPARK-60097][SDP] SCD1 Ignore-null support; Define
version map construction
897c70271fbb is described below
commit 897c70271fbb656d1c58233619b7f610fd0bf56b
Author: AnishMahto <[email protected]>
AuthorDate: Wed Nov 4 09:35:26 2026 -0800
[SPARK-60097][SDP] SCD1 Ignore-null support; Define version map construction
### What changes were proposed in this pull request?
Concretely define the contract for the contents of the version map; the
format the keys are written in, which leaves get an entry, and what the stored
sequence represents when null or not null.
Then implement the logic to construct a version map column expression as
per the contract, given an upsert-representing row. Delete-representing rows
need no equivalent operation, as they always hold null version maps
Populating incoming SCD1 microbatch rows with this logic will be done in
the next PR in the stack: https://github.com/apache/spark/pull/59044
As part of these changes, refactor `resolveIgnoreNullLeafPaths` into
`AutoCdcSchemaUtils`, as it is now used by both SCD1 and SCD2.
### Why are the changes needed?
AutoCDC SCD1 ignore-null support.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit tested in `Scd1VersionMapSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Co-authored with Opus 5.5
Closes #59311 from AnishMahto/scd1-version-map-construction.
Authored-by: AnishMahto <[email protected]>
Signed-off-by: Jose Torres <[email protected]>
(cherry picked from commit ce561d691bc251a49a20fc0e84f3d1b1de4f5742)
Signed-off-by: Jose Torres <[email protected]>
---
.../sql/pipelines/autocdc/AutoCdcSchemaUtils.scala | 23 ++
.../sql/pipelines/autocdc/Scd1VersionMap.scala | 91 ++++-
.../sql/pipelines/autocdc/Scd2BatchProcessor.scala | 2 +-
.../sql/pipelines/autocdc/Scd2VersionMap.scala | 28 +-
.../autocdc/AutoCdcSchemaUtilsSuite.scala | 72 ++++
.../pipelines/autocdc/Scd1VersionMapSuite.scala | 374 +++++++++++++++++++++
.../pipelines/autocdc/Scd2VersionMapSuite.scala | 63 ----
7 files changed, 563 insertions(+), 90 deletions(-)
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtils.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtils.scala
index 99d1698788b1..3a59acbf465c 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtils.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtils.scala
@@ -46,4 +46,27 @@ private[autocdc] object AutoCdcSchemaUtils {
case _ => Seq(Seq(field.name))
}
}
+
+ /**
+ * Resolves an ignore-null selection against the top-level fields of
`schema`, then flattens
+ * selected structs into leaf paths. Arrays and maps remain opaque leaves.
+ *
+ * @param schema Schema against which to resolve and canonicalize the
selection.
+ * @param ignoreNullSelection Top-level columns selected for ignore-null
handling.
+ * @param resolver Case-sensitivity resolver for column name matching.
+ * @return Raw, unquoted leaf-name parts in schema order, with spelling
taken exactly from
+ * `schema`.
+ */
+ def resolveIgnoreNullLeafPaths(
+ schema: StructType,
+ ignoreNullSelection: ColumnSelection,
+ resolver: Resolver): Seq[Seq[String]] = {
+ val ignoreNullColumns = ColumnSelection.applyToSchema(
+ schemaName = "ignoreNullSelection",
+ schema = schema,
+ columnSelection = Some(ignoreNullSelection),
+ resolver = resolver
+ )
+ flattenStructFieldPaths(ignoreNullColumns)
+ }
}
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
index e2df5155ee96..0a191cbfa5f5 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMap.scala
@@ -17,13 +17,37 @@
package org.apache.spark.sql.pipelines.autocdc
-import org.apache.spark.sql.types.{DataType, MapType, StringType}
+import org.apache.spark.sql.{functions => F, Column}
+import org.apache.spark.sql.catalyst.analysis.Resolver
+import org.apache.spark.sql.catalyst.expressions.Literal
+import org.apache.spark.sql.catalyst.util.QuotingUtils
+import org.apache.spark.sql.classic.ExpressionUtils
+import org.apache.spark.sql.types.{DataType, MapType, StringType, StructType}
/**
* Per-leaf authorship sequences for SCD1 reconciliation.
*
* An SCD1 target row can combine values authored by different CDC events. The
version map records
* which event currently authors each non-key user-data leaf.
+ *
+ * Concretely, the contract of the version map is as follows.
+ * 1. Every user-data leaf present in the row when the map is written receives
an entry.
+ * 2. A non-null entry is the sequencing clock of the event that authored the
leaf's current stored
+ * value. A delete authors a null value until a later upsert reauthors the
leaf.
+ * 3. A null entry means the leaf has no authored value.
+ * 4. A leaf absent from the map was not in the row when the map was written,
such as a leaf added
+ * through later schema evolution. It is treated as unauthored, as in (3),
until the map is next
+ * rewritten; then a null entry is explicitly materialized.
+ *
+ * Unlike the SCD2 version map, the SCD1 version map is mutable reconciliation
state. When a newer
+ * CDC event authors a leaf, reconciliation updates that leaf's stored value
and advances its
+ * sequencing clock in the map without changing the clocks of other leaves.
+ *
+ * A user-data leaf is a non-key, non-framework field obtained by recursively
expanding structs.
+ *
+ * Version-map keys are field paths rendered as fully quoted multipart
identifiers using
+ * [[org.apache.spark.sql.catalyst.util.QuotingUtils.quoteNameParts]]. Quoting
each name part
+ * distinguishes a nested path from a column whose name contains dots.
*/
private[pipelines] object Scd1VersionMap {
@@ -37,4 +61,69 @@ private[pipelines] object Scd1VersionMap {
*/
def mapType(sequencingType: DataType): MapType =
MapType(StringType, sequencingType, valueContainsNull = true)
+
+ /** Serializes a leaf field path into its version-map key. */
+ def serializeKey(path: Seq[String]): String =
QuotingUtils.quoteNameParts(path)
+
+ /**
+ * Builds a version map of type `mapType(sequencingType)` with one entry per
leaf.
+ *
+ * @param entries Each leaf's raw, unquoted name parts, paired with its
authorship sequence. Keys
+ * are serialized with [[serializeKey]], and the leaf paths
must be distinct.
+ * @param sequencingType The data type the sequences are cast to.
+ */
+ private def fromEntries(entries: Seq[(Seq[String], Column)], sequencingType:
DataType): Column = {
+ if (entries.isEmpty) {
+ // Without arguments, `map()` is `map<string, string>` under
+ // spark.sql.legacy.createEmptyCollectionUsingStringType, which cannot
be cast to struct or
+ // array sequence types.
+ ExpressionUtils.column(Literal.create(Map.empty,
mapType(sequencingType)))
+ } else {
+ val keyValueColumns = entries.flatMap { case (leafPath, leafSequence) =>
+ Seq(F.lit(serializeKey(leafPath)), leafSequence)
+ }
+ F.map(keyValueColumns: _*).cast(mapType(sequencingType))
+ }
+ }
+
+ /**
+ * Builds a column that computes a row's version map from that row's column
values.
+ *
+ * Every leaf in `schema` receives an entry of `authorshipSequence`, except
that a leaf selected
+ * by `ignoreNullSelection` is read by path from the row and receives a null
entry when its value
+ * is null.
+ *
+ * @param schema The user-data schema, without keys or framework columns,
whose leaves receive
+ * version-map entries. Keys take each name part's spelling
verbatim from this
+ * schema, so callers that persist the map must supply a
spelling that stays the
+ * same across runs. Its leaves selected by
`ignoreNullSelection` must resolve by
+ * path in the DataFrame on which the returned column is
evaluated.
+ * @param ignoreNullSelection The selection identifying leaves whose null
values are unauthored.
+ * @param authorshipSequence The sequence at which the leaves of `schema`
are considered authored.
+ * @param sequencingType The data type of `authorshipSequence`.
+ * @param resolver The resolver used for column-name matching.
+ * @return A column of type `mapType(sequencingType)` holding the row's
version map.
+ */
+ def buildVersionMap(
+ schema: StructType,
+ ignoreNullSelection: ColumnSelection,
+ authorshipSequence: Column,
+ sequencingType: DataType,
+ resolver: Resolver): Column = {
+ val ignoreNullLeafPaths = AutoCdcSchemaUtils
+ .resolveIgnoreNullLeafPaths(schema, ignoreNullSelection, resolver)
+ .toSet
+
+ val entries = AutoCdcSchemaUtils.flattenStructFieldPaths(schema).map {
path =>
+ val leafSequence =
+ if (ignoreNullLeafPaths.contains(path)) {
+ F.when(F.col(QuotingUtils.quoteNameParts(path)).isNotNull,
authorshipSequence)
+ } else {
+ authorshipSequence
+ }
+ path -> leafSequence
+ }
+
+ fromEntries(entries, sequencingType)
+ }
}
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
index bec882cce398..36db8c5c9696 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2BatchProcessor.scala
@@ -1024,7 +1024,7 @@ case class Scd2BatchProcessor(
resolver = resolver
)
val activeIgnoreNullLeafPaths =
- Scd2VersionMap.resolveIgnoreNullLeafPaths(
+ AutoCdcSchemaUtils.resolveIgnoreNullLeafPaths(
userDataColumnSchema, ignoreNullSelection, resolver)
val cdcMetadataCol = F.col(AutoCdcReservedNames.cdcMetadataColName)
diff --git
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala
index 61aed955710e..2daf90c66cbf 100644
---
a/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala
+++
b/sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMap.scala
@@ -93,8 +93,9 @@ private[pipelines] object Scd2VersionMap {
schema: StructType,
ignoreNullSelection: ColumnSelection,
resolver: Resolver): Column = {
- val ignoreNullLeafPaths =
- resolveIgnoreNullLeafPaths(schema, ignoreNullSelection, resolver).toSet
+ val ignoreNullLeafPaths = AutoCdcSchemaUtils
+ .resolveIgnoreNullLeafPaths(schema, ignoreNullSelection, resolver)
+ .toSet
// For each leaf, build a nullable struct (key, value). The struct is
non-null only when
// the leaf column's runtime value is null (meaning the leaf needs a
version map entry).
@@ -119,29 +120,6 @@ private[pipelines] object Scd2VersionMap {
}
}
- /**
- * Resolves an ignore-null selection against the top-level fields of
`schema`, then flattens
- * selected structs into leaf paths. Arrays and maps remain opaque leaves.
- *
- * @param schema Schema against which to resolve and canonicalize the
selection.
- * @param ignoreNullSelection Top-level columns selected for ignore-null
handling.
- * @param resolver Case-sensitivity resolver for column name matching.
- * @return Raw, unquoted leaf-name parts in schema order, with spelling
taken exactly from
- * `schema`.
- */
- private[autocdc] def resolveIgnoreNullLeafPaths(
- schema: StructType,
- ignoreNullSelection: ColumnSelection,
- resolver: Resolver): Seq[Seq[String]] = {
- val ignoreNullColumns = ColumnSelection.applyToSchema(
- schemaName = "ignoreNullSelection",
- schema = schema,
- columnSelection = Some(ignoreNullSelection),
- resolver = resolver
- )
- AutoCdcSchemaUtils.flattenStructFieldPaths(ignoreNullColumns)
- }
-
/**
* Returns the authorship value for `columnPath`, or null if the map or
entry is absent.
*
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtilsSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtilsSuite.scala
index 6b5c6a8d5074..1faee50af41d 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtilsSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/AutoCdcSchemaUtilsSuite.scala
@@ -18,7 +18,9 @@
package org.apache.spark.sql.pipelines.autocdc
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution,
caseSensitiveResolution}
+import org.apache.spark.sql.catalyst.util.QuotingUtils
import org.apache.spark.sql.types._
class AutoCdcSchemaUtilsSuite extends SparkFunSuite {
@@ -97,4 +99,74 @@ class AutoCdcSchemaUtilsSuite extends SparkFunSuite {
test("flattenStructFieldPaths returns an empty sequence for an empty
schema") {
assert(AutoCdcSchemaUtils.flattenStructFieldPaths(new StructType()) ===
Seq.empty)
}
+
+ test("resolveIgnoreNullLeafPaths returns canonical leaves in schema order") {
+ val schema = new StructType()
+ .add("id", IntegerType)
+ .add("Profile", new StructType()
+ .add("Display Name", StringType)
+ .add("Contact", new StructType().add("e.mail", StringType)))
+ .add("Events", ArrayType(new StructType().add("kind", StringType)))
+ .add("Lookup", MapType(StringType, new StructType().add("value",
StringType)))
+
+ val includeSelection = ColumnSelection.IncludeColumns(Seq(
+ UnqualifiedColumnName("lookup"),
+ UnqualifiedColumnName("profile"),
+ UnqualifiedColumnName("events")))
+ assert(AutoCdcSchemaUtils.resolveIgnoreNullLeafPaths(
+ schema, includeSelection, caseInsensitiveResolution) === Seq(
+ Seq("Profile", "Display Name"),
+ Seq("Profile", "Contact", "e.mail"),
+ Seq("Events"),
+ Seq("Lookup")))
+
+ val excludeSelection =
+ ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("profile")))
+ assert(AutoCdcSchemaUtils.resolveIgnoreNullLeafPaths(
+ schema, excludeSelection, caseInsensitiveResolution) === Seq(
+ Seq("id"),
+ Seq("Events"),
+ Seq("Lookup")))
+ }
+
+ test("resolveIgnoreNullLeafPaths respects case sensitivity") {
+ val schema = new StructType()
+ .add("Profile", new StructType().add("Name", StringType))
+
+ val exactSelection =
+ ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("Profile")))
+ assert(AutoCdcSchemaUtils.resolveIgnoreNullLeafPaths(
+ schema, exactSelection, caseSensitiveResolution) === Seq(Seq("Profile",
"Name")))
+
+ val mismatchedSelection =
+ ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("profile")))
+ checkError(
+ exception = intercept[AnalysisException] {
+ AutoCdcSchemaUtils.resolveIgnoreNullLeafPaths(
+ schema, mismatchedSelection, caseSensitiveResolution)
+ },
+ condition = "AUTOCDC_COLUMNS_NOT_FOUND_IN_SCHEMA",
+ sqlState = "42703",
+ parameters = Map(
+ "caseSensitivity" -> "case-sensitive",
+ "schemaName" -> "ignoreNullSelection",
+ "missingColumns" -> "profile",
+ "availableColumns" -> "Profile"
+ )
+ )
+ }
+
+ test("resolveIgnoreNullLeafPaths preserves raw special-character names") {
+ val schema = new StructType()
+ .add("wrapper.with.dot", new StructType()
+ .add("back`tick", StringType)
+ .add("space name", StringType))
+ val selection = ColumnSelection.IncludeColumns(Seq(
+ UnqualifiedColumnName(QuotingUtils.quoteIdentifier("wrapper.with.dot"))))
+
+ assert(AutoCdcSchemaUtils.resolveIgnoreNullLeafPaths(
+ schema, selection, caseInsensitiveResolution) === Seq(
+ Seq("wrapper.with.dot", "back`tick"),
+ Seq("wrapper.with.dot", "space name")))
+ }
}
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMapSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMapSuite.scala
new file mode 100644
index 000000000000..eb2c774bf40a
--- /dev/null
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd1VersionMapSuite.scala
@@ -0,0 +1,374 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.pipelines.autocdc
+
+import org.apache.spark.sql.{functions => F, QueryTest, Row}
+import org.apache.spark.sql.catalyst.analysis.{caseInsensitiveResolution,
caseSensitiveResolution}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
+import org.apache.spark.sql.classic.DataFrame
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+
+class Scd1VersionMapSuite extends QueryTest with SharedSparkSession {
+
+ private def encodedPath(parts: String*): String =
Scd1VersionMap.serializeKey(parts)
+
+ private def dataFrameOf(schema: StructType)(rows: Row*): DataFrame =
+ spark.createDataFrame(spark.sparkContext.parallelize(rows), schema)
+
+ test("mapType uses the supplied sequencing type and permits null values") {
+ val sequencingType = new StructType()
+ .add("timestamp", LongType)
+ .add("tieBreaker", IntegerType)
+
+ assert(Scd1VersionMap.mapType(sequencingType) ==
+ MapType(StringType, sequencingType, valueContainsNull = true))
+ }
+
+ test("buildVersionMap records every leaf with the expected sequencing
clock") {
+ val userDataSchema = new StructType()
+ .add("selectedNull", StringType)
+ .add("selectedValue", StringType)
+ .add("unselectedNull", StringType)
+ .add("unselectedValue", StringType)
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
IntegerType))
+ val input = dataFrameOf(inputSchema)(
+ Row(null, "selected", null, "unselected", 10)
+ )
+ val ignoreNullSelection = ColumnSelection.IncludeColumns(Seq(
+ UnqualifiedColumnName("selectedNull"),
+ UnqualifiedColumnName("selectedValue")
+ ))
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = IntegerType,
+ resolver = caseSensitiveResolution
+ ).as("versionMap")
+ )
+
+ assert(result.schema("versionMap").dataType ==
Scd1VersionMap.mapType(IntegerType))
+ checkAnswer(result, Row(Map[String, Any](
+ encodedPath("selectedNull") -> null,
+ encodedPath("selectedValue") -> 10,
+ encodedPath("unselectedNull") -> 10,
+ encodedPath("unselectedValue") -> 10
+ )))
+ }
+
+ test("buildVersionMap supports exclude-list ignore-null selections") {
+ val userDataSchema = new StructType()
+ .add("ignoredNull", StringType)
+ .add("authoredNull", StringType)
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(Row(null, null, 10L))
+ val ignoreNullSelection = ColumnSelection.ExcludeColumns(
+ Seq(UnqualifiedColumnName("authoredNull"))
+ )
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ ).as("versionMap")
+ )
+
+ checkAnswer(result, Row(Map[String, Any](
+ encodedPath("ignoredNull") -> null,
+ encodedPath("authoredNull") -> 10L
+ )))
+ }
+
+ test("buildVersionMap recursively expands structs and treats other complex
types as leaves") {
+ val userDataSchema = new StructType()
+ .add("profile", new StructType()
+ .add("city", StringType)
+ .add("contact", new StructType().add("zip", StringType)))
+ .add("profile.city", StringType)
+ .add("tags", ArrayType(StringType))
+ .add("properties", MapType(StringType, StringType))
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(
+ Row(Row(null, Row(null)), null, null, null, 10L)
+ )
+ val ignoreNullSelection = ColumnSelection.IncludeColumns(Seq(
+ UnqualifiedColumnName("profile"),
+ UnqualifiedColumnName("tags")
+ ))
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ ).as("versionMap")
+ )
+
+ checkAnswer(result, Row(Map[String, Any](
+ encodedPath("profile", "city") -> null,
+ encodedPath("profile", "contact", "zip") -> null,
+ encodedPath("profile.city") -> 10L,
+ encodedPath("tags") -> null,
+ encodedPath("properties") -> 10L
+ )))
+ }
+
+ test("buildVersionMap reads selected struct leaves independently and under
null structs") {
+ val userDataSchema = new StructType()
+ .add("profile", new StructType()
+ .add("city", StringType)
+ .add("contact", new StructType().add("zip", StringType)))
+ .add("billing", new StructType().add("city", StringType))
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(
+ Row(null, null, 10L),
+ Row(Row("SF", null), Row(null), 20L),
+ Row(Row(null, Row("94105")), Row("NYC"), 30L)
+ )
+ val ignoreNullSelection =
+ ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("profile")))
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ )
+ )
+
+ checkAnswer(result, Seq(
+ Row(Map[String, Any](
+ encodedPath("profile", "city") -> null,
+ encodedPath("profile", "contact", "zip") -> null,
+ encodedPath("billing", "city") -> 10L)),
+ Row(Map[String, Any](
+ encodedPath("profile", "city") -> 20L,
+ encodedPath("profile", "contact", "zip") -> null,
+ encodedPath("billing", "city") -> 20L)),
+ Row(Map[String, Any](
+ encodedPath("profile", "city") -> null,
+ encodedPath("profile", "contact", "zip") -> 30L,
+ encodedPath("billing", "city") -> 30L))
+ ))
+ }
+
+ test("buildVersionMap respects the supplied resolver for ignore-null
selection") {
+ val userDataSchema = new StructType().add("Value", StringType)
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(Row(null, 10L))
+ val ignoreNullSelection = ColumnSelection.IncludeColumns(
+ Seq(UnqualifiedColumnName("value"))
+ )
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseInsensitiveResolution
+ ).as("versionMap")
+ )
+
+ checkAnswer(result, Row(Map[String, Any](encodedPath("Value") -> null)))
+ }
+
+ test("buildVersionMap produces a typed empty map for an empty schema") {
+ val input = dataFrameOf(new StructType().add("seq", LongType))(Row(10L))
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = new StructType(),
+ ignoreNullSelection = ColumnSelection.ExcludeColumns(Seq.empty),
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ ).as("versionMap")
+ )
+
+ assert(result.schema("versionMap").dataType ==
Scd1VersionMap.mapType(LongType))
+ checkAnswer(result, Row(Map.empty[String, Long]))
+ }
+
+ test("buildVersionMap produces a typed empty map without leaves under legacy
collection types") {
+ val sequencingType = new StructType()
+ .add("timestamp", LongType)
+ .add("tieBreaker", IntegerType)
+ val userDataSchema = new StructType().add("empty", new StructType())
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
sequencingType))
+ val input = dataFrameOf(inputSchema)(Row(Row(), Row(10L, 1)))
+
+ withSQLConf(SQLConf.LEGACY_CREATE_EMPTY_COLLECTION_USING_STRING_TYPE.key
-> "true") {
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ColumnSelection.ExcludeColumns(Seq.empty),
+ authorshipSequence = F.col("seq"),
+ sequencingType = sequencingType,
+ resolver = caseSensitiveResolution
+ ).as("versionMap")
+ )
+
+ assert(result.schema("versionMap").dataType ==
Scd1VersionMap.mapType(sequencingType))
+ checkAnswer(result, Row(Map.empty[String, Row]))
+ }
+ }
+
+ // Version-map keys are persisted in target tables, so their format must not
change.
+ test("version map keys have a stable persisted serialization") {
+ val leafPathsAndExpectedKeys = Seq(
+ Seq("simple") -> "`simple`",
+ Seq("a.b") -> "`a.b`",
+ Seq("a", "b") -> "`a`.`b`",
+ Seq("wrapper", "leaf") -> "`wrapper`.`leaf`",
+ Seq("wrapper", "a.b") -> "`wrapper`.`a.b`",
+ Seq("wrapper", "has space") -> "`wrapper`.`has space`",
+ Seq("wrapper", "col-one") -> "`wrapper`.`col-one`",
+ Seq("wrapper", "back`tick") -> "`wrapper`.`back``tick`",
+ Seq("wrapper", "`already quoted`") -> "`wrapper`.```already quoted```",
+ Seq("wrapper", "single'quote") -> "`wrapper`.`single'quote`",
+ Seq("wrapper", "double\"quote") -> "`wrapper`.`double\"quote`",
+ Seq("wrapper", "back\\slash") -> "`wrapper`.`back\\slash`",
+ Seq("wrapper", "") -> "`wrapper`.``",
+ Seq("wrapper", "null" + 0.toChar + "byte") -> ("`wrapper`.`null" +
0.toChar + "byte`")
+ )
+ val wrapperLeafNames = leafPathsAndExpectedKeys.collect {
+ case (Seq("wrapper", name), _) => name
+ }
+ val userDataSchema = new StructType()
+ .add("simple", StringType)
+ .add("a.b", StringType)
+ .add("a", new StructType().add("b", StringType))
+ .add("wrapper", StructType(wrapperLeafNames.map(StructField(_,
StringType))))
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(
+ Row(null, null, Row(null),
Row.fromSeq(Seq.fill[Any](wrapperLeafNames.size)(null)), 10L))
+
+ leafPathsAndExpectedKeys.foreach { case (path, key) =>
+ assert(Scd1VersionMap.serializeKey(path) === key)
+ }
+
+ // Selecting every leaf makes buildVersionMap resolve each leaf's column
by its path.
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ColumnSelection.ExcludeColumns(Seq.empty),
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ )
+ )
+
+ checkAnswer(result, Row(leafPathsAndExpectedKeys.map { case (_, key) =>
key -> null }.toMap))
+ }
+
+ test("buildVersionMap reads a dotted column and the nested leaf it resembles
separately") {
+ val userDataSchema = new StructType()
+ .add("a.b", StringType)
+ .add("a", new StructType().add("b", StringType))
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(
+ Row(null, Row("nested"), 10L),
+ Row("dotted", Row(null), 20L)
+ )
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ColumnSelection.ExcludeColumns(Seq.empty),
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ )
+ )
+
+ checkAnswer(result, Seq(
+ Row(Map[String, Any](encodedPath("a.b") -> null, encodedPath("a", "b")
-> 10L)),
+ Row(Map[String, Any](encodedPath("a.b") -> 20L, encodedPath("a", "b") ->
null))
+ ))
+ }
+
+ test("buildVersionMap keys round-trip and resolve special-character leaf
names") {
+ val specialNames =
+ Seq("a.b", "has space", "back`tick", "quote\"", "back\\slash", "null" +
0.toChar + "byte")
+ val userDataSchema = new StructType()
+ .add("wrapper", StructType(specialNames.map(StructField(_, StringType))))
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
LongType))
+ val input = dataFrameOf(inputSchema)(Row(Row(specialNames: _*), 10L))
+ val ignoreNullSelection =
+ ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("wrapper")))
+
+ val versionMap = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = LongType,
+ resolver = caseSensitiveResolution
+ )
+ ).head().getMap[String, Long](0)
+
+
assert(versionMap.keys.map(CatalystSqlParser.parseMultipartIdentifier).toSet ==
+ specialNames.map(name => Seq("wrapper", name)).toSet)
+ assert(versionMap.values.toSet == Set(10L))
+ }
+
+ test("buildVersionMap supports struct-valued sequencing") {
+ val sequencingType = new StructType()
+ .add("timestamp", LongType)
+ .add("tieBreaker", IntegerType)
+ val userDataSchema = new StructType()
+ .add("selectedNull", StringType)
+ .add("selectedValue", StringType)
+ .add("unselectedNull", StringType)
+ val inputSchema = StructType(userDataSchema.fields :+ StructField("seq",
sequencingType))
+ val input = dataFrameOf(inputSchema)(Row(null, "selected", null, Row(10L,
1)))
+ val ignoreNullSelection = ColumnSelection.IncludeColumns(Seq(
+ UnqualifiedColumnName("selectedNull"),
+ UnqualifiedColumnName("selectedValue")
+ ))
+
+ val result = input.select(
+ Scd1VersionMap.buildVersionMap(
+ schema = userDataSchema,
+ ignoreNullSelection = ignoreNullSelection,
+ authorshipSequence = F.col("seq"),
+ sequencingType = sequencingType,
+ resolver = caseSensitiveResolution
+ ).as("versionMap")
+ )
+
+ assert(result.schema("versionMap").dataType ==
Scd1VersionMap.mapType(sequencingType))
+ checkAnswer(result, Row(Map[String, Any](
+ encodedPath("selectedNull") -> null,
+ encodedPath("selectedValue") -> Row(10L, 1),
+ encodedPath("unselectedNull") -> Row(10L, 1)
+ )))
+ }
+}
diff --git
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMapSuite.scala
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMapSuite.scala
index 4b8dc58a4872..bf9b25828281 100644
---
a/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMapSuite.scala
+++
b/sql/pipelines/src/test/scala/org/apache/spark/sql/pipelines/autocdc/Scd2VersionMapSuite.scala
@@ -486,69 +486,6 @@ class Scd2VersionMapSuite extends QueryTest with
SharedSparkSession {
// Version map authorship operations
// =========================================================================
- test("resolveIgnoreNullLeafPaths returns canonical leaves in schema order") {
- val schema = new StructType()
- .add("id", IntegerType)
- .add("Profile", new StructType()
- .add("Display Name", StringType)
- .add("Contact", new StructType().add("e.mail", StringType)))
- .add("Events", ArrayType(new StructType().add("kind", StringType)))
- .add("Lookup", MapType(StringType, new StructType().add("value",
StringType)))
-
- withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
- val includeSelection = ColumnSelection.IncludeColumns(Seq(
- UnqualifiedColumnName("lookup"),
- UnqualifiedColumnName("profile"),
- UnqualifiedColumnName("events")))
- assert(Scd2VersionMap.resolveIgnoreNullLeafPaths(
- schema, includeSelection, resolver) === Seq(
- Seq("Profile", "Display Name"),
- Seq("Profile", "Contact", "e.mail"),
- Seq("Events"),
- Seq("Lookup")))
-
- val excludeSelection =
- ColumnSelection.ExcludeColumns(Seq(UnqualifiedColumnName("profile")))
- assert(Scd2VersionMap.resolveIgnoreNullLeafPaths(
- schema, excludeSelection, resolver) === Seq(
- Seq("id"),
- Seq("Events"),
- Seq("Lookup")))
- }
- }
-
- test("resolveIgnoreNullLeafPaths respects case sensitivity") {
- val schema = new StructType()
- .add("Profile", new StructType().add("Name", StringType))
-
- withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
- val exactSelection =
- ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("Profile")))
- assert(Scd2VersionMap.resolveIgnoreNullLeafPaths(
- schema, exactSelection, resolver) === Seq(Seq("Profile", "Name")))
-
- val mismatchedSelection =
- ColumnSelection.IncludeColumns(Seq(UnqualifiedColumnName("profile")))
- intercept[AnalysisException] {
- Scd2VersionMap.resolveIgnoreNullLeafPaths(schema, mismatchedSelection,
resolver)
- }
- }
- }
-
- test("resolveIgnoreNullLeafPaths preserves raw special-character names") {
- val schema = new StructType()
- .add("wrapper.with.dot", new StructType()
- .add("back`tick", StringType)
- .add("space name", StringType))
- val selection = ColumnSelection.IncludeColumns(Seq(
- UnqualifiedColumnName(QuotingUtils.quoteIdentifier("wrapper.with.dot"))))
-
- assert(Scd2VersionMap.resolveIgnoreNullLeafPaths(
- schema, selection, resolver) === Seq(
- Seq("wrapper.with.dot", "back`tick"),
- Seq("wrapper.with.dot", "space name")))
- }
-
test("entryValue distinguishes values from absent entries") {
val path = Seq("wrapper", "a.b")
val key = encodedPath(path: _*)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]