TheR1sing3un commented on code in PR #14198:
URL: https://github.com/apache/hudi/pull/14198#discussion_r2496919180
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala:
##########
@@ -34,7 +34,7 @@ import org.junit.jupiter.api.Assertions.{assertDoesNotThrow,
assertFalse}
import org.junit.jupiter.api.function.Executable
@Tag("functional")
-class TestIncrementalQueryWithArchivedInstants extends
SparkClientFunctionalTestHarness {
+class TestIncrementalQueryWithArchivedInstants extends
SparkClientFunctionalTestHarnessScala {
Review Comment:
> No need to update the class if there is no usage of `withSQLConf` (same
for a few other classes)?
Nice suggestion! The modification has been completed as you said
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala:
##########
@@ -251,36 +247,39 @@ class TestSparkSqlCoreFlow extends HoodieSparkSqlTestBase
{
isMetadataEnabledOnWrite: Boolean, count: Int): Unit = {
inputDf.select("timestamp", "_row_key", "rider", "driver", "begin_lat",
"begin_lon", "end_lat", "end_lon", "fare",
"_hoodie_is_deleted",
"partition_path").createOrReplaceTempView("insert_temp_table")
- try {
- spark.sql(s"set
hoodie.metadata.enable=${String.valueOf(isMetadataEnabledOnWrite)}")
- if (writeOp.equals(UPSERT)) {
- spark.sql("set hoodie.sql.bulk.insert.enable=false")
- spark.sql("set hoodie.sql.insert.mode=upsert")
- spark.sql(
- s"""
- | merge into $tableName as target
- | using insert_temp_table as source
- | on target._row_key = source._row_key and
- | target.partition_path = source.partition_path
- | when matched then update set *
- | when not matched then insert *
- | """.stripMargin)
- } else if (writeOp.equals(BULK_INSERT)) {
- spark.sql("set hoodie.sql.bulk.insert.enable=true")
- spark.sql("set hoodie.sql.insert.mode=non-strict")
- spark.sql(s"insert into $tableName select * from insert_temp_table")
- } else if (writeOp.equals(INSERT)) {
- spark.sql("set hoodie.sql.bulk.insert.enable=false")
- spark.sql("set hoodie.sql.insert.mode=non-strict")
- spark.sql(s"insert into $tableName select * from insert_temp_table")
+ withSQLConf(HoodieMetadataConfig.ENABLE.key() ->
isMetadataEnabledOnWrite.toString) {
+ if (writeOp.equals(UPSERT)) {
+ withSQLConf(
+ "hoodie.sql.bulk.insert.enable" -> "false",
+ "hoodie.sql.insert.mode" -> "upsert"
+ ) {
+ spark.sql(
+ s"""
+ | merge into $tableName as target
+ | using insert_temp_table as source
+ | on target._row_key = source._row_key and
+ | target.partition_path = source.partition_path
+ | when matched then update set *
+ | when not matched then insert *
+ | """.stripMargin)
+ }
+ } else if (writeOp.equals(BULK_INSERT)) {
+ withSQLConf(
+ "hoodie.sql.bulk.insert.enable" -> "true",
+ "hoodie.sql.insert.mode" -> "non-strict"
+ ) {
+ spark.sql(s"insert into $tableName select * from
insert_temp_table")
+ }
+ } else if (writeOp.equals(INSERT)) {
+ withSQLConf(
+ "hoodie.sql.bulk.insert.enable" -> "false",
+ "hoodie.sql.insert.mode" -> "non-strict"
+ ) {
+ spark.sql(s"insert into $tableName select * from
insert_temp_table")
+ }
+ }
Review Comment:
> Add `HoodieMetadataConfig.ENABLE.key() ->
isMetadataEnabledOnWrite.toString` to `if` and `else if` branches to reduce one
level of nesting?
Nice suggestion! The modification has been completed as you said
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
