[
https://issues.apache.org/jira/browse/HUDI-5419?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17649907#comment-17649907
]
Jonathan Vexler commented on HUDI-5419:
---------------------------------------
Here is my modified test function from HoodieSparkSqlTestBase incase something
happens to the closed pr:
{code:java}
override protected def test(testName: String, testTags: Tag*)(testFun: => Any
/* Assertion */)(implicit pos: source.Position): Unit = {
val conf = spark.sessionState.conf.getAllConfs
println(s"Before $testName")
for ((k, v) <- conf) {
println(s"k: $k, v: $v")
}
super.test(testName, testTags: _*)(
try {
testFun
} finally {
val catalog = spark.sessionState.catalog
catalog.listDatabases().foreach { db =>
catalog.listTables(db).foreach { table =>
catalog.dropTable(table, true, true)
}
}
println(s"After $testName")
for ((k, v) <- spark.sessionState.conf.getAllConfs) {
//some configs like spark.driver.port or spark.app.startTime may change
if (k.startsWith("hoodie")) {
if (!conf.contains(k)) {
println(s"unsetting k: $k, v: $v")
spark.sessionState.conf.unsetConf(k)
} else if (!conf(k).equals(v)) {
println(s"overwriting k: $k, v: $v with ${conf(k)}")
spark.sessionState.conf.setConfString(k, conf(k))
} else {
println(s"keeping k: $k, v: $v")
}
} else {
println(s"Doesn't start with hoodie k: $k, v: $v")
}
}
}
)
} {code}
> Spark-SQL tests persist configs
> --------------------------------
>
> Key: HUDI-5419
> URL: https://issues.apache.org/jira/browse/HUDI-5419
> Project: Apache Hudi
> Issue Type: Test
> Components: spark-sql
> Reporter: Jonathan Vexler
> Priority: Minor
> Labels: pull-request-available
> Attachments: logs_24157.zip
>
>
> Spark configs are persisted between tests. This can cause interaction between
> tests
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
