yunchipang commented on code in PR #7100:
URL: https://github.com/apache/datafusion-comet/pull/7100#discussion_r4429609351
##########
spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala:
##########
@@ -277,24 +277,50 @@ class CometIcebergWriteDetectionSuite extends
CometTestBase with CometIcebergTes
}
}
- test("fall-back: unknown column in the write schema") {
+ test("fall-back: unknown field nested in a struct") {
assume(icebergVersionAtLeast(1, 10), "The unknown type requires Iceberg
1.10+")
withDetectionCatalog { _ =>
- // Spark DDL cannot declare `unknown` (Iceberg plans it as Spark's
NullType), so evolve the
- // schema through the Iceberg API.
+ // Spark DDL cannot declare `unknown`, so add it through the Iceberg API.
createTable("unknown_col", partitionSpec = "", properties =
Some("'format-version'='3'"))
+ spark.sql(s"ALTER TABLE $catalog.$ns.unknown_col ADD COLUMN s STRUCT<a:
INT>")
addIcebergColumn(
loadIcebergTable(spark, catalog, ns, "unknown_col"),
"u",
- icebergUnknownType())
+ icebergUnknownType(),
+ parent = Some("s"))
spark.sql(s"REFRESH TABLE $catalog.$ns.unknown_col")
val writeExec = captureWriteExec("unknown_col", allowWriteFailure =
true) {
- spark.sql(s"INSERT INTO $catalog.$ns.unknown_col VALUES (1, 'us', 1.0,
NULL)")
+ spark.sql(
+ s"INSERT INTO $catalog.$ns.unknown_col " +
+ "VALUES (1, 'us', 1.0, named_struct('a', 1, 'u', NULL))")
}
assertUnsupportedContains(writeExec, "unknown_col", "column u has
Iceberg type unknown")
}
}
+ test("fall-back: every column is unknown") {
Review Comment:
added `"fall-back: partition field sourced from an unknown column"`
detection case that asserts this reason.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
