This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new cff76974 perf: Add benchmarks for Spark Scan + Comet Exec (#863)
cff76974 is described below
commit cff76974e064e5652984674b118f696281b8783d
Author: Andy Grove <[email protected]>
AuthorDate: Fri Aug 23 10:28:39 2024 -0600
perf: Add benchmarks for Spark Scan + Comet Exec (#863)
* Add benchmarks for Spark Scan + Comet Exec
* address feedback
* address feedback
* revert removing env var from usage examples
* fix
---
.../spark/sql/benchmark/CometTPCDSMicroBenchmark.scala | 14 ++++++++++++++
.../spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala | 11 +++++++++++
2 files changed, 25 insertions(+)
diff --git
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
index 016c2a37..40a84a12 100644
---
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
+++
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
@@ -124,6 +124,20 @@ object CometTPCDSMicroBenchmark extends
CometTPCQueryBenchmarkBase {
cometSpark.sql(queryString).noop()
}
}
+ benchmark.addCase(s"$name$nameSuffix: Comet (Exec)") { _ =>
+ withSQLConf(
+ CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
+ CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
+ CometConf.COMET_SHUFFLE_MODE.key -> "auto",
+ CometConf.COMET_REGEXP_ALLOW_INCOMPATIBLE.key -> "true",
+ // enabling COMET_EXPLAIN_NATIVE_ENABLED may add overhead but is
useful for debugging
+ CometConf.COMET_EXPLAIN_NATIVE_ENABLED.key -> "false",
+ CometConf.COMET_EXEC_ENABLED.key -> "true") {
+ cometSpark.sql(queryString).noop()
+ }
+ }
benchmark.run()
}
}
diff --git
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
index 2361346b..7e9bdbc9 100644
---
a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
+++
b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
@@ -79,6 +79,17 @@ trait CometTPCQueryBenchmarkBase extends SqlBasedBenchmark
with CometTPCQueryBas
cometSpark.sql(queryString).noop()
}
}
+ benchmark.addCase(s"$name$nameSuffix: Comet (Exec)") { _ =>
+ withSQLConf(
+ CometConf.COMET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_ENABLED.key -> "true",
+ CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
+ CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true",
+ CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
+ CometConf.COMET_SHUFFLE_MODE.key -> "auto") {
+ cometSpark.sql(queryString).noop()
+ }
+ }
benchmark.run()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
