YannByron commented on a change in pull request #4270:
URL: https://github.com/apache/hudi/pull/4270#discussion_r767131307
##########
File path:
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
##########
@@ -293,28 +293,26 @@ class TestHoodieSparkSqlWriter {
*/
@Test
def testDisableAndEnableMetaFields(): Unit = {
- try {
- testBulkInsertWithSortMode(BulkInsertSortMode.NONE, populateMetaFields =
false)
- //create a new table
- val fooTableModifier =
commonTableModifier.updated("hoodie.bulkinsert.shuffle.parallelism", "4")
- .updated(DataSourceWriteOptions.OPERATION.key,
DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL)
- .updated(DataSourceWriteOptions.ENABLE_ROW_WRITER.key, "true")
- .updated(HoodieWriteConfig.BULK_INSERT_SORT_MODE.key(),
BulkInsertSortMode.NONE.name())
- .updated(HoodieTableConfig.POPULATE_META_FIELDS.key(), "true")
+ testBulkInsertWithSortMode(BulkInsertSortMode.NONE, populateMetaFields =
false)
Review comment:
solve the warning while build.
##########
File path:
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
##########
@@ -711,51 +709,49 @@ class TestHoodieSparkSqlWriter {
DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "",
DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key ->
"org.apache.hudi.keygen.NonpartitionedKeyGenerator",
HoodieWriteConfig.TBL_NAME.key -> "hoodie_test")
- try {
- val df = spark.range(0, 1000).toDF("keyid")
- .withColumn("col3", expr("keyid"))
- .withColumn("age", lit(1))
- .withColumn("p", lit(2))
-
- df.write.format("hudi")
- .options(options)
- .option(DataSourceWriteOptions.OPERATION.key, "insert")
- .option("hoodie.insert.shuffle.parallelism", "4")
- .mode(SaveMode.Overwrite).save(tempBasePath)
-
- df.write.format("hudi")
- .options(options)
- .option(DataSourceWriteOptions.OPERATION.key,
"insert_overwrite_table")
- .option("hoodie.insert.shuffle.parallelism", "4")
- .mode(SaveMode.Append).save(tempBasePath)
-
- val currentCommits =
spark.read.format("hudi").load(tempBasePath).select("_hoodie_commit_time").take(1).map(_.getString(0))
- val incrementalKeyIdNum = spark.read.format("hudi")
- .option(DataSourceReadOptions.QUERY_TYPE.key,
DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL)
- .option(DataSourceReadOptions.BEGIN_INSTANTTIME.key, "0000")
- .option(DataSourceReadOptions.END_INSTANTTIME.key, currentCommits(0))
- .load(tempBasePath).select("keyid").orderBy("keyid").count
- assert(incrementalKeyIdNum == 1000)
-
- df.write.mode(SaveMode.Overwrite).save(baseBootStrapPath)
- spark.emptyDataFrame.write.format("hudi")
- .options(options)
- .option(HoodieBootstrapConfig.BASE_PATH.key, baseBootStrapPath)
- .option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key,
classOf[NonpartitionedKeyGenerator].getCanonicalName)
- .option(DataSourceWriteOptions.OPERATION.key,
DataSourceWriteOptions.BOOTSTRAP_OPERATION_OPT_VAL)
- .option(HoodieBootstrapConfig.PARALLELISM_VALUE.key, "4")
- .mode(SaveMode.Overwrite).save(tempBasePath)
- df.write.format("hudi").options(options)
- .option(DataSourceWriteOptions.OPERATION.key,
"insert_overwrite_table")
- .option("hoodie.insert.shuffle.parallelism",
"4").mode(SaveMode.Append).save(tempBasePath)
- val currentCommitsBootstrap =
spark.read.format("hudi").load(tempBasePath).select("_hoodie_commit_time").take(1).map(_.getString(0))
- val incrementalKeyIdNumBootstrap = spark.read.format("hudi")
- .option(DataSourceReadOptions.QUERY_TYPE.key,
DataSourceReadOptions.QUERY_TYPE_INCREMENTAL_OPT_VAL)
- .option(DataSourceReadOptions.BEGIN_INSTANTTIME.key, "0000")
- .option(DataSourceReadOptions.END_INSTANTTIME.key,
currentCommitsBootstrap(0))
- .load(tempBasePath).select("keyid").orderBy("keyid").count
- assert(incrementalKeyIdNumBootstrap == 1000)
- }
+ val df = spark.range(0, 1000).toDF("keyid")
Review comment:
ditto
##########
File path:
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestHoodieSqlBase.scala
##########
@@ -59,14 +59,18 @@ class TestHoodieSqlBase extends FunSuite with
BeforeAndAfterAll {
}
override protected def test(testName: String, testTags: Tag*)(testFun: =>
Any /* Assertion */)(implicit pos: source.Position): Unit = {
- try super.test(testName, testTags: _*)(try testFun finally {
- val catalog = spark.sessionState.catalog
- catalog.listDatabases().foreach{db =>
- catalog.listTables(db).foreach {table =>
- catalog.dropTable(table, true, true)
+ super.test(testName, testTags: _*)(
Review comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
