boneanxs commented on code in PR #10242:
URL: https://github.com/apache/hudi/pull/10242#discussion_r1415062623
##########
hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java:
##########
@@ -164,4 +164,8 @@ public static boolean isUpsert(WriteOperationType
operation) {
public static boolean isDelete(WriteOperationType operation) {
return operation == DELETE || operation == DELETE_PREPPED;
}
+
+ public static boolean isDeletePartition(WriteOperationType operation) {
Review Comment:
Better move this to `TimelineUtils`, and `INSERT_OVERWRITE` also could
delete partitions
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestShowPartitions.scala:
##########
@@ -202,8 +202,63 @@ class TestShowPartitions extends HoodieSparkSqlTestBase {
// Lazily drop that partition
spark.sql(s"alter table $tableName drop partition(year='2023',
month='06', day='06')")
checkAnswer(s"show partitions $tableName")(Seq.empty: _*)
+ // rewrite data to the dropped partition
+ spark.sql(s"insert into $tableName values (1, 'a1', 10, 1000,
'2023', '06', '06')")
+ checkAnswer(s"show partitions $tableName")(
+ Seq("year=2023/month=06/day=06")
+ )
}
}
}
}
+
+ test("Test show partitions after table being overwritten") {
+ withTable(generateTableName) { tableName =>
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long,
+ | year string,
+ | month string,
+ | day string
+ | ) using hudi
+ | partitioned by (year, month, day)
+ | tblproperties (
+ | primaryKey = 'id',
+ | preCombineField = 'ts'
+ | )
+ """.stripMargin)
+
+ // Insert into dynamic partition
+ spark.sql(
+ s"""
+ | insert into $tableName
+ | values
+ | (1, 'a1', 10, 1000, '2023', '12', '01'),
+ | (2, 'a2', 10, 1000, '2023', '12', '02'),
+ | (3, 'a3', 10, 1000, '2023', '12', '03')
+ """.stripMargin)
+ checkAnswer(s"show partitions $tableName")(
+ Seq("year=2023/month=12/day=01"),
+ Seq("year=2023/month=12/day=02"),
+ Seq("year=2023/month=12/day=03")
+ )
+
+ // Insert overwrite table
+ spark.sql(
+ s"""
+ | insert overwrite table $tableName
+ | values
+ | (4, 'a4', 10, 1000, '2023', '12', '01'),
+ | (2, 'a2', 10, 1000, '2023', '12', '04')
+ """.stripMargin)
+ checkAnswer(s"show partitions $tableName")(
+ Seq("year=2023/month=12/day=01"),
+ Seq("year=2023/month=12/day=04")
+ )
Review Comment:
Need to add tests for static partition overwrite
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
