HyukjinKwon commented on a change in pull request #34698:
URL: https://github.com/apache/spark/pull/34698#discussion_r756045187
##########
File path:
sql/core/src/test/scala/org/apache/spark/sql/TPCDSQueryTestSuite.scala
##########
@@ -171,22 +173,35 @@ class TPCDSQueryTestSuite extends QueryTest with
TPCDSBase with SQLQueryTestHelp
SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
"spark.sql.join.forceApplyShuffledHashJoin" -> "true")
- val joinConfSet: Set[Map[String, String]] =
- Set(sortMergeJoinConf, broadcastHashJoinConf, shuffledHashJoinConf);
+ val allJoinConfCombinations = Seq(
+ sortMergeJoinConf, broadcastHashJoinConf, shuffledHashJoinConf)
+
+ val joinConfs: Seq[Map[String, String]] = if (regenerateGoldenFiles) {
+ require(
+ !sys.env.contains("SPARK_TPCDS_CONF"),
+ "'SPARK_TPCDS_CONF' cannot be set together with
'SPARK_GENERATE_GOLDEN_FILES'")
+ Seq(sortMergeJoinConf)
+ } else {
+ sys.env.get("SPARK_TPCDS_CONF").map { s =>
+ val p = new java.util.Properties()
+ p.load(new java.io.StringReader(s))
+ Seq(p.asScala.toMap)
+ }.getOrElse(allJoinConfCombinations)
+ }
+
+ assert(joinConfs.nonEmpty)
+ joinConfs.foreach(conf => require(
+ allJoinConfCombinations.contains(conf), s"$conf should be one of
$allJoinConfCombinations"))
if (tpcdsDataPath.nonEmpty) {
tpcdsQueries.foreach { name =>
val queryString = resourceToString(s"tpcds/$name.sql",
classLoader = Thread.currentThread().getContextClassLoader)
test(name) {
val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out")
- System.gc() // Workaround for GitHub Actions memory limitation, see
also SPARK-37368
- runQuery(queryString, goldenFile, joinConfSet.head.toSeq, false)
- if (!regenerateGoldenFiles) {
- joinConfSet.tail.foreach { conf =>
- System.gc() // SPARK-37368
- runQuery(queryString, goldenFile, conf.toSeq, true)
- }
Review comment:
While I am here, I ended up with refactoring this code part here. No
behavior change. Everything is as was.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
