HyukjinKwon commented on a change in pull request #26612: [SPARK-29975][SQL]
introduce --CONFIG_DIM directive
URL: https://github.com/apache/spark/pull/26612#discussion_r349855185
##########
File path: sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
##########
@@ -271,36 +259,36 @@ class SQLQueryTestSuite extends QueryTest with
SharedSparkSession {
// Fix misplacement when comment is at the end of the query.
.map(_.split("\n").filterNot(_.startsWith("--")).mkString("\n")).map(_.trim).filter(_
!= "")
- // When we are regenerating the golden files for test cases without
'--IMPORT' specified, or
- // running test cases against [[ThriftServerQueryTestSuite], we don't need
to set any config as
- // they all need to return the same result.
- // When we use '--SET' and '--IMPORT' together for those import queries,
we want to run the
- // same queries from the original file but with different settings and
save the answers. So the
- // `--SET` will be respected in this case.
- if ((regenerateGoldenFiles && importedTestCaseName.isEmpty) ||
!isTestWithConfigSets) {
- runQueries(queries, testCase, None)
+ val settingLines = comments.filter(_.startsWith("--SET
")).map(_.substring(6))
+ val settings = settingLines.flatMap(_.split(",").map { kv =>
+ val (conf, value) = kv.span(_ != '=')
+ conf.trim -> value.substring(1).trim
+ })
+
+ if (regenerateGoldenFiles) {
+ runQueries(queries, testCase, Some(settings))
} else {
- val configSets = {
- val configLines = comments.filter(_.startsWith("--SET
")).map(_.substring(6))
- val configs = configLines.map(_.split(",").map { confAndValue =>
- val (conf, value) = confAndValue.span(_ != '=')
+ // A config dimension has multiple config sets, and a config set has
multiple configs.
+ // - config dim: Seq[Seq[(String, String)]]
+ // - config set: Seq[(String, String)]
+ // - config: (String, String))
+ // We need to do cartesian product for all the config dimensions, to get
a list of
+ // config sets, and run the query once for each config set.
+ val configDimLines =
comments.filter(_.startsWith("--CONFIG_DIM")).map(_.substring(12))
Review comment:
Yeah we should document this ...
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
