This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new cc125cc1 chore: Enable shuffle in micro benchmarks (#806)
cc125cc1 is described below
commit cc125cc1c48aac31fc92640c952057e2e85016d7
Author: Andy Grove <agr...@apache.org>
AuthorDate: Mon Aug 12 07:53:52 2024 -0600
chore: Enable shuffle in micro benchmarks (#806)
* Enable shuffle in benchmarks
* format
* Revert remove SPARK_GENERATE_BENCHMARK_FILES=1
---
.../test/resources/tpcds-micro-benchmarks/join_exploding_output.sql | 2 +-
spark/src/test/resources/tpcds-micro-benchmarks/join_inner.sql | 2 +-
.../org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala | 4 ++++
.../org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala | 5 ++++-
4 files changed, 10 insertions(+), 3 deletions(-)
diff --git
a/spark/src/test/resources/tpcds-micro-benchmarks/join_exploding_output.sql
b/spark/src/test/resources/tpcds-micro-benchmarks/join_exploding_output.sql
index b8ee8916..941d13a9 100644
--- a/spark/src/test/resources/tpcds-micro-benchmarks/join_exploding_output.sql
+++ b/spark/src/test/resources/tpcds-micro-benchmarks/join_exploding_output.sql
@@ -17,6 +17,6 @@
-- This is based on the first join in q72 when there is no join reordering
-select count(*)
+select cs_order_number, cs_quantity, inv_quantity_on_hand
from catalog_sales join inventory on cs_item_sk = inv_item_sk
where cs_warehouse_sk = 1 and inv_quantity_on_hand = 666;
\ No newline at end of file
diff --git a/spark/src/test/resources/tpcds-micro-benchmarks/join_inner.sql
b/spark/src/test/resources/tpcds-micro-benchmarks/join_inner.sql
index 8e7c2412..1e501725 100644
--- a/spark/src/test/resources/tpcds-micro-benchmarks/join_inner.sql
+++ b/spark/src/test/resources/tpcds-micro-benchmarks/join_inner.sql
@@ -15,6 +15,6 @@
-- specific language governing permissions and limitations
-- under the License.
-select count(*)
+select ss_sold_date_sk, ss_sold_time_sk, ss_quantity, d_year, d_moy, d_dom
from date_dim join store_sales on d_date_sk = ss_sold_date_sk
where d_year = 2000;
diff --git
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
index aa0c9115..86f8b486 100644
---
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
+++
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
@@ -114,7 +114,11 @@ object CometTPCDSMicroBenchmark extends
CometTPCQueryBenchmarkBase {
benchmark.addCase(s"$name$nameSuffix: Comet (Scan, Exec)") { _ =>
withSQLConf(
CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
+ CometConf.COMET_SHUFFLE_ENFORCE_MODE_ENABLED.key -> "true",
+ CometConf.COMET_SHUFFLE_MODE.key -> "auto",
CometConf.COMET_REGEXP_ALLOW_INCOMPATIBLE.key -> "true",
+ CometConf.COMET_EXPLAIN_NATIVE_ENABLED.key -> "true",
CometConf.COMET_EXEC_ENABLED.key -> "true",
CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true") {
cometSpark.sql(queryString).noop()
diff --git
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
index db4e8ca5..2d78d03a 100644
---
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
+++
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
@@ -74,7 +74,10 @@ trait CometTPCQueryBenchmarkBase extends SqlBasedBenchmark
with CometTPCQueryBas
withSQLConf(
CometConf.COMET_ENABLED.key -> "true",
CometConf.COMET_EXEC_ENABLED.key -> "true",
- CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true") {
+ CometConf.COMET_EXEC_ALL_OPERATOR_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
+ CometConf.COMET_SHUFFLE_ENFORCE_MODE_ENABLED.key -> "true",
+ CometConf.COMET_SHUFFLE_MODE.key -> "auto") {
cometSpark.sql(queryString).noop()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
