TengHuo commented on code in PR #9123:
URL: https://github.com/apache/hudi/pull/9123#discussion_r1502195393
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala:
##########
@@ -1578,5 +1578,333 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
)
}
}
+ spark.sessionState.conf.unsetConf("hoodie.datasource.write.operation")
+ }
+
+ /**
+ * When neither of strict mode nor sql.write.operation is set, sql write
operation takes precedence and default value is chosen.
+ */
+ test("Test sql write operation with INSERT_INTO No explicit configs") {
+ spark.sessionState.conf.unsetConf("hoodie.sql.write.operation")
+ spark.sessionState.conf.unsetConf("hoodie.sql.insert.mode")
+ spark.sessionState.conf.unsetConf("hoodie.datasource.insert.dup.policy")
+ spark.sessionState.conf.unsetConf("hoodie.datasource.write.operation")
+ withRecordType()(withTempDir { tmp =>
+ Seq("cow","mor").foreach {tableType =>
+ withTable(generateTableName) { tableName =>
+ ingestAndValidateData(tableType, tableName, tmp)
+ }
+ }
+ })
+ }
+
+ test("Test sql write operation with INSERT_INTO override both strict mode
and sql write operation") {
+ withRecordType()(withTempDir { tmp =>
+ Seq("cow","mor").foreach { tableType =>
+ Seq(WriteOperationType.INSERT, WriteOperationType.BULK_INSERT,
WriteOperationType.UPSERT).foreach { operation =>
+ withTable(generateTableName) { tableName =>
+ ingestAndValidateData(tableType, tableName, tmp, operation,
+ List("set hoodie.sql.write.operation = " + operation.value(),
"set hoodie.sql.insert.mode = upsert"))
+ }
+ }
+ }
+ })
+ }
+
+ test("Test sql write operation with INSERT_INTO override only sql write
operation") {
+ withRecordType()(withTempDir { tmp =>
+ Seq("cow","mor").foreach {tableType =>
+ Seq(WriteOperationType.INSERT, WriteOperationType.BULK_INSERT,
WriteOperationType.UPSERT).foreach { operation =>
+ withTable(generateTableName) { tableName =>
+ ingestAndValidateData(tableType, tableName, tmp, operation,
+ List("set hoodie.sql.write.operation = " + operation.value()))
+ }
+ }
+ }
+ })
+ }
+
+ test("Test sql write operation with INSERT_INTO override only strict mode") {
+ spark.sessionState.conf.unsetConf("hoodie.sql.write.operation")
+ spark.sessionState.conf.unsetConf("hoodie.sql.insert.mode")
+ spark.sessionState.conf.unsetConf("hoodie.datasource.insert.dup.policy")
+ spark.sessionState.conf.unsetConf("hoodie.datasource.write.operation")
+ spark.sessionState.conf.unsetConf("hoodie.sql.bulk.insert.enable")
+ withRecordType()(withTempDir { tmp =>
+ Seq("cow","mor").foreach {tableType =>
+ withTable(generateTableName) { tableName =>
+ ingestAndValidateData(tableType, tableName, tmp,
WriteOperationType.UPSERT,
+ List("set hoodie.sql.insert.mode = upsert"))
+ }
+ }
+ })
+ }
+
+ def ingestAndValidateData(tableType: String, tableName: String, tmp: File,
+ expectedOperationtype: WriteOperationType =
WriteOperationType.INSERT,
+ setOptions: List[String] = List.empty) : Unit = {
+ setOptions.foreach(entry => {
+ spark.sql(entry)
+ })
+
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | dt string
+ |) using hudi
+ | tblproperties (
+ | type = '$tableType',
+ | primaryKey = 'id',
+ | preCombine = 'name'
Review Comment:
The name of this key should be `preCombineField` here.
`keyTableConfigMapping` in `HoodieOptionConfig` ->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
