anew commented on code in PR #56686:
URL: https://github.com/apache/spark/pull/56686#discussion_r3553096963
##########
sql/pipelines/src/main/scala/org/apache/spark/sql/pipelines/graph/DatasetManager.scala:
##########
@@ -104,12 +106,38 @@ object DatasetManager extends Logging {
transformer.transformTables { table =>
if (tablesToMaterialize.keySet.contains(table.identifier)) {
try {
- materializeTable(
+ val isFullRefresh =
tablesToMaterialize(table.identifier).isFullRefresh
+ val (tableWithMaterializationMetadata, catalogTableEntity) =
materializeTable(
resolvedDataflowGraph = resolvedDataflowGraph,
table = table,
- isFullRefresh =
tablesToMaterialize(table.identifier).isFullRefresh,
+ isFullRefresh = isFullRefresh,
context = context
)
+ // Auxiliary tables' lifecycle should follow the table that it
is complimentary to.
+ // If this table has any auxiliary tables, validate the target
can host them and
+ // materialize/full-refresh them accordingly.
+
resolvedDataflowGraph.auxiliaryTableSpecs.get(table.identifier).foreach {
+ auxiliaryTableSpec =>
+ // If this table is an AutoCDC target table, as identified
by being
+ // accompanied by an AutoCDC auxiliary table, additionally
validate that the
+ // target table supports row level mutations. This is a
relevant validation for
Review Comment:
I guess this comment was on an outdated commit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
