voonhous commented on code in PR #19163:
URL: https://github.com/apache/hudi/pull/19163#discussion_r3871077737
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/insert/TestInsertTable2.scala:
##########
@@ -693,6 +695,164 @@ class TestInsertTable2 extends HoodieSparkSqlTestBase {
}
}
+ test("Test bulk insert with insert overwrite partition in dynamic mode") {
+ // The static-mode test above resolves the partitions to replace from the
PARTITION clause;
+ // dynamic mode resolves them from the incoming rows, which on the
row-writer path is
+ // DatasetBulkInsertOverwriteCommitActionExecutor reading
_hoodie_partition_path off the
+ // prepared dataset.
+ withSQLConf(SPARK_SQL_INSERT_INTO_OPERATION.key ->
WriteOperationType.BULK_INSERT.value(),
+ "hoodie.datasource.overwrite.mode" -> "dynamic") {
+ withTempDir { tmp =>
+ withTable(generateTableName) { tableName =>
+ val tablePath = s"${tmp.getCanonicalPath}/$tableName"
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | dt string
+ |) using hudi
+ | tblproperties (
+ | type = 'cow',
+ | primaryKey = 'id'
+ | )
+ | partitioned by (dt)
+ | location '$tablePath'
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values(1, 'a1', 10,
'2021-07-18'), (2, 'a2', 20, '2021-07-19')")
+
+ // Only the partition present in the incoming rows is replaced;
2021-07-19 survives.
+ spark.sql(s"insert overwrite table $tableName partition (dt)
values(3, 'b1', 11, '2021-07-18')")
+ checkAnswer(s"select id, name, price, dt from $tableName order by
id")(
+ Seq(2, "a2", 20.0, "2021-07-19"),
+ Seq(3, "b1", 11.0, "2021-07-18")
+ )
+ assertResult(WriteOperationType.INSERT_OVERWRITE) {
+ getLastCommitMetadata(spark, tablePath).getOperationType
+ }
+ assertResult(Set("dt=2021-07-18"))(getReplacedPartitions(tablePath))
+ }
+ }
+ }
+ }
+
+ test("Test bulk insert with insert overwrite against pending clustering") {
+ // The only coverage of rejectIfOverlappingPendingClustering on the
row-writer path; the RDD
+ // path is covered by TestInsertOverwriteWithClustering. Clustering is
pending on one of two
+ // partitions: overwriting the other must succeed and leave the plan
pending, overwriting the
+ // clustered one must be rejected before anything is written. Static mode
resolves the target
+ // partition from the PARTITION clause and dynamic mode from the rows, so
both arms of
+ // DatasetBulkInsertOverwriteCommitActionExecutor.resolveTargetPartitions
are driven.
Review Comment:
The dynamic arm only proves the populated-meta-fields case. With
`hoodie.populate.meta.fields=false` the same overlapping overwrite is not
rejected: `prepareForBulkInsert` stubs `_hoodie_partition_path` as null,
`resolveTargetPartitions` returns `[null]`, the view lookup comes back empty
and the check returns before the update strategy runs. Reproduced on master and
filed as #19770; latent since #18829, not introduced here.
Should this comment say the coverage is populated-meta-fields only, and
would a `populate.meta.fields=false` leg be worth adding once #19770 lands?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
