anew commented on code in PR #57625:
URL: https://github.com/apache/spark/pull/57625#discussion_r3693059428
##########
sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/AutoCdcAuxiliaryTable.scala:
##########
@@ -227,13 +239,27 @@ object AutoCdcAuxiliaryTable {
StructField(Scd2BatchProcessor.deletedByBatchIdColName, LongType,
nullable = true)
val scd2AuxiliaryTableSchema = StructType(targetTableSchema.fields :+
deletedByBatchIdField)
+ // The effective track-history column set, resolved by the flow from its
user-selected source
+ // schema (see [[AutoCdcMergeFlow.trackHistoryColumnNames]]) -- NOT
recomputed here from the
+ // evolved target schema, which would keep tracking columns the flow no
longer selects and would
+ // miss implicit (default / `* EXCEPT`) tracked-set changes. A change in
this set reinterprets
+ // which transitions open a new historical record, so it is drift-checked.
+ val trackHistoryColumnNames =
inputAutoCdcFlow.trackHistoryColumnNames.getOrElse(
+ throw SparkException.internalError(
+ "SCD2 AutoCDC flow is missing its resolved track-history column set."
+ )
+ )
+
val scd2AuxiliaryTableProperties =
// Record which SCD strategy this auxiliary table serves so downstream
readers can identify it
// without inspecting the schema.
Map(scdTypePropertyKey -> ScdType.Type2.label) ++
// Persist the AutoCDC key column names as a JSON list; immutable
post-creation (full-refresh
// is the only way to change it).
Map(keyColumnNamesProperty ->
serializeKeyColumnNames(keyFields.map(_.name))) ++
+ // Persist the resolved track-history column names; a change
reinterprets already-reconciled
+ // history, so it is immutable post-creation (full-refresh is the only
way to change it).
+ Map(trackHistoryColumnNamesProperty ->
serializeKeyColumnNames(trackHistoryColumnNames)) ++
Review Comment:
renamed!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
