This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 40407df0 build: Re-enable TPCDS q72 for broadcast and hash join
configs (#781)
40407df0 is described below
commit 40407df07ec50d0c9aca37fc789de22c478cbe78
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Tue Aug 6 06:28:12 2024 -0700
build: Re-enable TPCDS q72 for broadcast and hash join configs (#781)
* build: Re-enable TPCDS q72
* Increase heap size
* Move q72
* Skip q72 for sort merge join
---
.../org/apache/spark/sql/CometTPCDSQuerySuite.scala | 8 ++------
.../apache/spark/sql/CometTPCDSQueryTestSuite.scala | 18 ++++++++++++++----
2 files changed, 16 insertions(+), 10 deletions(-)
diff --git
a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
index 6eeb7e33..864f2c03 100644
--- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQuerySuite.scala
@@ -108,9 +108,7 @@ class CometTPCDSQuerySuite
"q69",
"q70",
"q71",
- // TODO: unknown failure (seems memory usage over Github action
runner) in CI with q72 in
- // https://github.com/apache/datafusion-comet/pull/613.
- // "q72",
+ "q72",
"q73",
"q74",
"q75",
@@ -164,9 +162,7 @@ class CometTPCDSQuerySuite
"q64",
"q67a",
"q70a",
- // TODO: unknown failure (seems memory usage over Github action
runner) in CI with q72-v2.7
- // in https://github.com/apache/datafusion-comet/pull/613.
- // "q72",
+ "q72",
"q74",
"q75",
"q77a",
diff --git
a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala
b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala
index 6aec042a..448277ba 100644
--- a/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCDSQueryTestSuite.scala
@@ -208,8 +208,13 @@ class CometTPCDSQueryTestSuite extends QueryTest with
TPCDSBase with CometSQLQue
test(name) {
val goldenFile = new File(s"$baseResourcePath/v1_4", s"$name.sql.out")
joinConfs.foreach { conf =>
- System.gc() // Workaround for GitHub Actions memory limitation, see
also SPARK-37368
- runQuery(queryString, goldenFile, conf)
+ val sortMergeJoin = sortMergeJoinConf == conf
+ // Skip q72 for sort-merge join because it uses too many resources
+ // that can cause OOM in GitHub Actions
+ if (!(sortMergeJoin && name == "q72")) {
+ System.gc() // Workaround for GitHub Actions memory limitation,
see also SPARK-37368
+ runQuery(queryString, goldenFile, conf)
+ }
}
}
}
@@ -221,8 +226,13 @@ class CometTPCDSQueryTestSuite extends QueryTest with
TPCDSBase with CometSQLQue
test(s"$name-v2.7") {
val goldenFile = new File(s"$baseResourcePath/v2_7", s"$name.sql.out")
joinConfs.foreach { conf =>
- System.gc() // SPARK-37368
- runQuery(queryString, goldenFile, conf)
+ val sortMergeJoin = sortMergeJoinConf == conf
+ // Skip q72 for sort-merge join because it uses too many resources
+ // that can cause OOM in GitHub Actions
+ if (!(sortMergeJoin && name == "q72")) {
+ System.gc() // SPARK-37368
+ runQuery(queryString, goldenFile, conf)
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
