nsivabalan commented on code in PR #8899:
URL: https://github.com/apache/hudi/pull/8899#discussion_r1227328424
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala:
##########
@@ -1465,8 +1466,36 @@ class TestCOWDataSource extends
HoodieSparkClientTestBase with ScalaAssertionSup
}
}
-}
+ @Test
+ def testHiveStyleDelete(): Unit = {
+ val df = spark.sql("select cast(1 as bigint) as list_id, 1 as
_hudi_last_update, '123' as _hudi_partition")
+ var hudiOptions = Map[String, String](
+ HoodieWriteConfig.TBL_NAME.key() -> "tbl",
+ DataSourceWriteOptions.OPERATION.key() -> "insert",
+ DataSourceWriteOptions.TABLE_TYPE.key() -> "COPY_ON_WRITE",
+ DataSourceWriteOptions.RECORDKEY_FIELD.key() -> "list_id",
+ DataSourceWriteOptions.PARTITIONPATH_FIELD.key() -> "_hudi_partition",
+ DataSourceWriteOptions.PRECOMBINE_FIELD.key() -> "_hudi_last_update",
+ DataSourceWriteOptions.HIVE_STYLE_PARTITIONING.key() -> "true"
+ )
+
+
df.write.format("org.apache.hudi").options(hudiOptions).mode(SaveMode.Overwrite).save(basePath)
+ hudiOptions = Map[String, String](
+ HoodieWriteConfig.TBL_NAME.key() -> "tbl",
+ DataSourceWriteOptions.OPERATION.key() -> "delete",
+ DataSourceWriteOptions.TABLE_TYPE.key() -> "COPY_ON_WRITE",
+ DataSourceWriteOptions.RECORDKEY_FIELD.key() -> "list_id",
+ DataSourceWriteOptions.PARTITIONPATH_FIELD.key() -> "_hudi_partition",
+ DataSourceWriteOptions.PRECOMBINE_FIELD.key() -> "_hudi_last_update"
+ )
+
+ df.select("list_id", "_hudi_partition").
+ write.format("org.apache.hudi").options(hudiOptions).
+ mode(SaveMode.Append).save(basePath)
+ assertEquals(0, spark.read.format("hudi").load(basePath).count())
Review Comment:
can we also delete a subset of records and ensure deletion only deletes the
intended records.
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala:
##########
@@ -1075,18 +1075,18 @@ object HoodieSparkSqlWriter {
// for missing write configs corresponding to table configs, fill them
up.
fetchMissingWriteConfigsFromTableConfig(tableConfig,
optParams).foreach((kv) => translatedOptsWithMappedTableConfig += (kv._1 ->
kv._2))
}
+ if (null != tableConfig && mode != SaveMode.Overwrite) {
Review Comment:
good find
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
