hudi-agent commented on code in PR #19921:
URL: https://github.com/apache/hudi/pull/19921#discussion_r3996427919
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala:
##########
@@ -71,25 +76,56 @@ class HoodieSparkSqlTestBase extends FunSuite with
BeforeAndAfterAll {
// is consistent with the fixtures
DateTimeZone.setDefault(DateTimeZone.UTC)
TimeZone.setDefault(DateTimeUtils.getTimeZone("UTC"))
- protected lazy val spark: SparkSession = SparkSession.builder()
- .config("spark.sql.warehouse.dir", sparkWareHouse.getCanonicalPath)
- .config("spark.sql.session.timeZone", "UTC")
- .config("hoodie.insert.shuffle.parallelism", "4")
- .config("hoodie.upsert.shuffle.parallelism", "4")
- .config("hoodie.delete.shuffle.parallelism", "4")
- .config(sparkConf())
- .getOrCreate()
+ protected lazy val spark: SparkSession = if (sharedSessionEnabled) {
+ val session = HoodieSparkSqlTestBase.sharedBaseSession().newSession()
+ SparkSession.setActiveSession(session)
+ applySuiteConfToSharedSession(session)
+ session
+ } else {
+ HoodieSparkSqlTestBase.sessionBuilder(sparkWareHouse,
sparkConf()).getOrCreate()
+ }
private var tableId = new AtomicInteger(0)
private var extraConf = Map[String, String]()
+ // Shared mode: spark.hadoop.* keys this suite set on the shared Hadoop
conf, with the value they replaced.
+ private var hadoopConfOverrides: Seq[(String, String)] = Seq.empty
+
def sparkConf(): SparkConf = {
val conf = getSparkConfForTest("Hoodie SQL Test")
conf.setAll(extraConf)
conf
}
+ /**
+ * Shared mode: the context-level SparkConf is fixed, so the deltas a suite
adds through extraConf or a
+ * sparkConf() override go to its session conf (hoodie.* and spark.sql.*
keys), except spark.hadoop.*
+ * keys, which the write client reads from sparkContext.hadoopConfiguration
and which are restored in
+ * afterAll. Any other spark.* key, and any static SQL conf, is a setting a
child session cannot change,
+ * so it is rejected before anything is mutated: a partial apply would leave
the shared Hadoop conf
+ * changed for every later suite in the JVM.
+ */
+ private def applySuiteConfToSharedSession(session: SparkSession): Unit = {
+ val defaults = getSparkConfForTest("Hoodie SQL Test").getAll.toMap
Review Comment:
🤖 nit: `"Hoodie SQL Test"` now appears in three places (here, `sparkConf()`,
`sharedBaseSession()`); could you pull it into a constant in the companion
object so the delta computation here can't silently drift from the base
session's conf?
<sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag
quality.</i></sub>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
