Zouxxyy commented on code in PR #6949:
URL: https://github.com/apache/hudi/pull/6949#discussion_r998857447
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala:
##########
@@ -901,4 +901,47 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Test enable hoodie.merge.allow.duplicate.on.inserts when write") {
+ spark.sql("set hoodie.datasource.write.operation = insert")
+ Seq("mor", "cow").foreach { tableType =>
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long,
+ | dt string
+ | ) using hudi
+ | partitioned by (dt)
+ | location '${tmp.getCanonicalPath}/$tableName'
+ | tblproperties (
+ | primaryKey = 'id',
+ | preCombineField = 'ts',
+ | type = '$tableType'
+ | )
+ """.stripMargin)
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values
(1, 'a1', 10, 1000)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a1", 10, 1000, "2021-12-25")
+ )
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values
(1, 'a2', 20, 1001)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a2", 20, 1001, "2021-12-25")
+ )
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true")
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values
(1, 'a3', 30, 1002)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a2", 20, 1001, "2021-12-25"),
+ Seq(1, "a3", 30, 1002, "2021-12-25")
+ )
+ }
+ }
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
Review Comment:
> We should reset it at the beginning of the test
This is done to ensure that other test cases are not affected by this case,
just like line 945, 535 and so on
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestMergeIntoTable2.scala:
##########
@@ -609,7 +609,6 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
| preCombineField = 'ts'
| )
""".stripMargin)
- spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true")
// Insert data
Review Comment:
> Why drop this line ?
Because this configuration does not work here and can be misleading
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
