This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 0dca5450 Remove TPC-DS benchmark results (#728)
0dca5450 is described below
commit 0dca54508d158495d4282286f8eba732c9338683
Author: Andy Grove <[email protected]>
AuthorDate: Sat Jul 27 06:29:57 2024 -0600
Remove TPC-DS benchmark results (#728)
---
.../2024-07-19/tpcds_allqueries.png | Bin 27257 -> 0 bytes
.../2024-07-19/tpcds_queries_compare.png | Bin 39132 -> 0 bytes
.../2024-07-19/tpcds_queries_speedup.png | Bin 85002 -> 0 bytes
.../benchmark-results/2024-07-19/comet-tpcds.json | 342 ---------------------
.../benchmark-results/2024-07-19/spark-tpcds.json | 327 --------------------
docs/source/contributor-guide/benchmarking.md | 22 --
6 files changed, 691 deletions(-)
diff --git
a/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_allqueries.png
b/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_allqueries.png
deleted file mode 100644
index 3a7017a0..00000000
Binary files
a/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_allqueries.png
and /dev/null differ
diff --git
a/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_queries_compare.png
b/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_queries_compare.png
deleted file mode 100644
index f49ef68b..00000000
Binary files
a/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_queries_compare.png
and /dev/null differ
diff --git
a/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_queries_speedup.png
b/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_queries_speedup.png
deleted file mode 100644
index 4a604273..00000000
Binary files
a/docs/source/_static/images/benchmark-results/2024-07-19/tpcds_queries_speedup.png
and /dev/null differ
diff --git
a/docs/source/contributor-guide/benchmark-results/2024-07-19/comet-tpcds.json
b/docs/source/contributor-guide/benchmark-results/2024-07-19/comet-tpcds.json
deleted file mode 100644
index a4bab0a9..00000000
---
a/docs/source/contributor-guide/benchmark-results/2024-07-19/comet-tpcds.json
+++ /dev/null
@@ -1,342 +0,0 @@
-{
- "engine": "datafusion-comet",
- "benchmark": "tpcds",
- "data_path": "/mnt/bigdata/tpcds/sf100/",
- "query_path": "../../tpcds/queries-spark",
- "spark_conf": {
- "spark.eventLog.enabled": "true",
- "spark.jars":
"file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar",
- "spark.comet.cast.allowIncompatible": "true",
- "spark.app.initial.jar.urls":
"spark://woody.lan:41193/jars/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar",
- "spark.app.id": "app-20240718163301-0003",
- "spark.executor.extraClassPath":
"/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar",
- "spark.comet.exec.shuffle.enabled": "true",
- "spark.app.name": "DataFusion Comet Benchmark derived from TPC-H /
TPC-DS",
- "spark.serializer.objectStreamReset": "100",
- "spark.submit.deployMode": "client",
- "spark.sql.autoBroadcastJoinThreshold": "-1",
- "spark.comet.exec.all.enabled": "true",
- "spark.executor.cores": "8",
- "spark.driver.port": "41193",
- "spark.driver.host": "woody.lan",
- "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false
-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
- "spark.comet.shuffle.enforceMode.enabled": "true",
- "spark.sql.warehouse.dir":
"file:/home/andy/git/apache/datafusion-benchmarks/runners/datafusion-comet/spark-warehouse",
- "spark.shuffle.manager":
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
- "spark.comet.exec.enabled": "true",
- "spark.app.submitTime": "1721341980775",
- "spark.repl.local.jars":
"file:///home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar",
- "spark.executor.id": "driver",
- "spark.master": "spark://woody:7077",
- "spark.app.startTime": "1721341981073",
- "spark.comet.exec.shuffle.mode": "auto",
- "spark.sql.extensions": "org.apache.comet.CometSparkSessionExtensions",
- "spark.driver.memory": "8G",
- "spark.driver.extraClassPath":
"/home/andy/git/apache/datafusion-comet/spark/target/comet-spark-spark3.4_2.12-0.1.0-SNAPSHOT.jar",
- "spark.sql.adaptive.coalescePartitions.enabled": "true",
- "spark.executor.memory": "32G",
- "spark.rdd.compress": "True",
- "spark.executor.extraJavaOptions":
"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
- "spark.executor.instances": "1",
- "spark.cores.max": "8",
- "spark.comet.enabled": "true",
- "spark.submit.pyFiles": ""
- },
- "1": [
- 5.936143159866333
- ],
- "2": [
- 8.26771068572998
- ],
- "3": [
- 3.143160104751587
- ],
- "4": [
- 99.57721495628357
- ],
- "5": [
- 20.31959581375122
- ],
- "6": [
- 2.659390687942505
- ],
- "7": [
- 6.321574449539185
- ],
- "8": [
- 3.3775765895843506
- ],
- "9": [
- 10.297355890274048
- ],
- "10": [
- 5.794115304946899
- ],
- "11": [
- 41.01704978942871
- ],
- "12": [
- 6.774289131164551
- ],
- "13": [
- 8.915542364120483
- ],
- "14": [
- 86.61133527755737
- ],
- "15": [
- 13.07849407196045
- ],
- "16": [
- 19.245990991592407
- ],
- "17": [
- 5.388644218444824
- ],
- "18": [
- 13.891231536865234
- ],
- "19": [
- 3.869602680206299
- ],
- "20": [
- 11.179648160934448
- ],
- "21": [
- 26.43477487564087
- ],
- "22": [
- 12.071606397628784
- ],
- "23": [
- 217.38866710662842
- ],
- "24": [
- 22.737252235412598
- ],
- "25": [
- 6.0180275440216064
- ],
- "26": [
- 10.12047791481018
- ],
- "27": [
- 5.269314527511597
- ],
- "28": [
- 13.336988687515259
- ],
- "29": [
- 9.294469594955444
- ],
- "30": [
- 2.487182378768921
- ],
- "31": [
- 12.829366683959961
- ],
- "32": [
- 8.359683275222778
- ],
- "33": [
- 9.587389707565308
- ],
- "34": [
- 4.090600252151489
- ],
- "35": [
- 7.087569713592529
- ],
- "36": [
- 5.502429962158203
- ],
- "37": [
- 5.975481033325195
- ],
- "38": [
- 8.321046113967896
- ],
- "39": [
- 86.69561338424683
- ],
- "40": [
- 18.024808168411255
- ],
- "41": [
- 0.14025306701660156
- ],
- "42": [
- 2.7543857097625732
- ],
- "43": [
- 5.10432767868042
- ],
- "44": [
- 3.023483991622925
- ],
- "45": [
- 8.575555086135864
- ],
- "46": [
- 6.691994905471802
- ],
- "47": [
- 14.728558540344238
- ],
- "48": [
- 6.201830148696899
- ],
- "49": [
- 11.18767786026001
- ],
- "50": [
- 10.928320407867432
- ],
- "51": [
- 16.93299388885498
- ],
- "52": [
- 2.7706453800201416
- ],
- "53": [
- 3.6064443588256836
- ],
- "54": [
- 8.893964052200317
- ],
- "55": [
- 2.645658254623413
- ],
- "56": [
- 9.719430446624756
- ],
- "57": [
- 17.113372802734375
- ],
- "58": [
- 30.33391761779785
- ],
- "59": [
- 14.553935050964355
- ],
- "60": [
- 10.117067813873291
- ],
- "61": [
- 9.30464506149292
- ],
- "62": [
- 11.765451192855835
- ],
- "63": [
- 3.6449074745178223
- ],
- "64": [
- 23.456274271011353
- ],
- "65": [
- 18.001898050308228
- ],
- "66": [
- 37.000144958496094
- ],
- "67": [
- 46.503966331481934
- ],
- "68": [
- 7.1533238887786865
- ],
- "69": [
- 5.305586814880371
- ],
- "70": [
- 12.200140714645386
- ],
- "71": [
- 6.394218683242798
- ],
- "72": [
- 51.827319860458374
- ],
- "73": [
- 4.081210136413574
- ],
- "74": [
- 27.13532018661499
- ],
- "75": [
- 26.004110097885132
- ],
- "76": [
- 4.606779336929321
- ],
- "77": [
- 13.787659645080566
- ],
- "78": [
- 41.01244926452637
- ],
- "79": [
- 6.395780801773071
- ],
- "80": [
- 36.78436040878296
- ],
- "81": [
- 2.570908784866333
- ],
- "82": [
- 8.755934238433838
- ],
- "83": [
- 2.8941328525543213
- ],
- "84": [
- 1.1297607421875
- ],
- "85": [
- 2.922498941421509
- ],
- "86": [
- 3.0235934257507324
- ],
- "87": [
- 8.466246366500854
- ],
- "88": [
- 21.329773664474487
- ],
- "89": [
- 3.9919168949127197
- ],
- "90": [
- 2.072303295135498
- ],
- "91": [
- 2.06138014793396
- ],
- "92": [
- 5.129335880279541
- ],
- "93": [
- 15.211457967758179
- ],
- "94": [
- 10.024892091751099
- ],
- "95": [
- 34.040268898010254
- ],
- "96": [
- 2.6338326930999756
- ],
- "97": [
- 12.165382146835327
- ],
- "98": [
- 3.7391271591186523
- ],
- "99": [
- 20.151211977005005
- ]
-}
\ No newline at end of file
diff --git
a/docs/source/contributor-guide/benchmark-results/2024-07-19/spark-tpcds.json
b/docs/source/contributor-guide/benchmark-results/2024-07-19/spark-tpcds.json
deleted file mode 100644
index f7ab268b..00000000
---
a/docs/source/contributor-guide/benchmark-results/2024-07-19/spark-tpcds.json
+++ /dev/null
@@ -1,327 +0,0 @@
-{
- "engine": "datafusion-comet",
- "benchmark": "tpcds",
- "data_path": "/mnt/bigdata/tpcds/sf100/",
- "query_path": "../../tpcds/queries-spark",
- "spark_conf": {
- "spark.eventLog.enabled": "true",
- "spark.driver.extraJavaOptions": "-Djava.net.preferIPv6Addresses=false
-XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add- [...]
- "spark.sql.warehouse.dir":
"file:/home/andy/git/apache/datafusion-benchmarks/runners/datafusion-comet/spark-warehouse",
- "spark.driver.port": "39959",
- "spark.app.startTime": "1721343961227",
- "spark.executor.id": "driver",
- "spark.master": "spark://woody:7077",
- "spark.app.name": "DataFusion Comet Benchmark derived from TPC-H /
TPC-DS",
- "spark.driver.memory": "8G",
- "spark.app.submitTime": "1721343960912",
- "spark.app.id": "app-20240718170601-0004",
- "spark.executor.memory": "32G",
- "spark.rdd.compress": "True",
- "spark.executor.extraJavaOptions":
"-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions
--add-opens=java.base/java.lang=ALL-UNNAMED
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
--add-opens=java.base/java.io=ALL-UNNAMED
--add-opens=java.base/java.net=ALL-UNNAMED
--add-opens=java.base/java.nio=ALL-UNNAMED
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --ad [...]
- "spark.serializer.objectStreamReset": "100",
- "spark.executor.instances": "1",
- "spark.cores.max": "8",
- "spark.submit.pyFiles": "",
- "spark.submit.deployMode": "client",
- "spark.sql.autoBroadcastJoinThreshold": "-1",
- "spark.executor.cores": "8",
- "spark.driver.host": "woody.lan"
- },
- "1": [
- 7.800227403640747
- ],
- "2": [
- 10.467795133590698
- ],
- "3": [
- 2.8007187843322754
- ],
- "4": [
- 91.76162981987
- ],
- "5": [
- 23.099835872650146
- ],
- "6": [
- 3.365507125854492
- ],
- "7": [
- 5.432553291320801
- ],
- "8": [
- 3.8576300144195557
- ],
- "9": [
- 6.3153862953186035
- ],
- "10": [
- 9.874432802200317
- ],
- "11": [
- 33.87704801559448
- ],
- "12": [
- 8.301852226257324
- ],
- "13": [
- 5.269356966018677
- ],
- "14": [
- 150.8186900615692
- ],
- "15": [
- 15.294623136520386
- ],
- "16": [
- 19.310945749282837
- ],
- "17": [
- 6.411248207092285
- ],
- "18": [
- 14.292846918106079
- ],
- "19": [
- 3.2587430477142334
- ],
- "20": [
- 13.411643266677856
- ],
- "21": [
- 35.572487115859985
- ],
- "22": [
- 22.02106499671936
- ],
- "23": [
- 249.43153548240662
- ],
- "24": [
- 21.229645490646362
- ],
- "25": [
- 5.656827211380005
- ],
- "26": [
- 11.043184041976929
- ],
- "27": [
- 3.837435722351074
- ],
- "28": [
- 10.440573692321777
- ],
- "29": [
- 13.529588222503662
- ],
- "30": [
- 2.973015308380127
- ],
- "31": [
- 12.835312604904175
- ],
- "32": [
- 12.979064464569092
- ],
- "33": [
- 10.316180229187012
- ],
- "34": [
- 3.0397789478302
- ],
- "35": [
- 11.558360815048218
- ],
- "36": [
- 4.719287872314453
- ],
- "37": [
- 9.003387689590454
- ],
- "38": [
- 17.37705898284912
- ],
- "39": [
- 113.10317611694336
- ],
- "40": [
- 25.56372880935669
- ],
- "41": [
- 0.2145369052886963
- ],
- "42": [
- 2.0899407863616943
- ],
- "43": [
- 5.081417798995972
- ],
- "44": [
- 1.0103275775909424
- ],
- "45": [
- 11.752943992614746
- ],
- "46": [
- 4.572746276855469
- ],
- "47": [
- 17.91774606704712
- ],
- "48": [
- 9.562321901321411
- ],
- "49": [
- 10.740519046783447
- ],
- "50": [
- 15.341339349746704
- ],
- "51": [
- 23.100194454193115
- ],
- "52": [
- 2.0887205600738525
- ],
- "53": [
- 3.6534769535064697
- ],
- "54": [
- 14.758203268051147
- ],
- "55": [
- 1.812988042831421
- ],
- "56": [
- 10.357929229736328
- ],
- "57": [
- 21.42304801940918
- ],
- "58": [
- 46.502179861068726
- ],
- "59": [
- 15.086335182189941
- ],
- "60": [
- 10.964382886886597
- ],
- "61": [
- 6.710279703140259
- ],
- "62": [
- 15.012513160705566
- ],
- "63": [
- 3.904587984085083
- ],
- "64": [
- 37.42494583129883
- ],
- "65": [
- 22.118192434310913
- ],
- "66": [
- 33.776007890701294
- ],
- "67": [
- 83.05218243598938
- ],
- "68": [
- 4.215189456939697
- ],
- "69": [
- 9.373996257781982
- ],
- "70": [
- 13.401167631149292
- ],
- "71": [
- 5.064984560012817
- ],
- "72": [
- 55.99225473403931
- ],
- "73": [
- 2.3760015964508057
- ],
- "74": [
- 30.810500144958496
- ],
- "75": [
- 31.74701237678528
- ],
- "76": [
- 4.408271789550781
- ],
- "77": [
- 13.593222379684448
- ],
- "78": [
- 55.4714629650116
- ],
- "79": [
- 4.276215314865112
- ],
- "80": [
- 51.73041367530823
- ],
- "81": [
- 3.35856556892395
- ],
- "82": [
- 15.728789806365967
- ],
- "83": [
- 5.831252336502075
- ],
- "84": [
- 2.2465932369232178
- ],
- "85": [
- 4.967155694961548
- ],
- "86": [
- 4.595327377319336
- ],
- "87": [
- 15.662192344665527
- ],
- "88": [
- 28.15371608734131
- ],
- "89": [
- 5.115787744522095
- ],
- "90": [
- 4.091581583023071
- ],
- "91": [
- 3.3242697715759277
- ],
- "92": [
- 7.603022813796997
- ],
- "93": [
- 34.54590129852295
- ],
- "94": [
- 10.030377388000488
- ],
- "95": [
- 31.767929792404175
- ],
- "96": [
- 1.7932329177856445
- ],
- "97": [
- 19.545964002609253
- ],
- "98": [
- 4.293541431427002
- ],
- "99": [
- 26.280267000198364
- ]
-}
\ No newline at end of file
diff --git a/docs/source/contributor-guide/benchmarking.md
b/docs/source/contributor-guide/benchmarking.md
index c8fa53a2..b4d2f5f4 100644
--- a/docs/source/contributor-guide/benchmarking.md
+++ b/docs/source/contributor-guide/benchmarking.md
@@ -104,25 +104,3 @@ The raw results of these benchmarks in JSON format is
available here:
- [Spark](./benchmark-results/2024-07-19/spark-tpch.json)
- [Comet](./benchmark-results/2024-07-19/comet-tpch.json)
- [DataFusion](./benchmark-results/2024-07-19/datafusion-tpch.json)
-
-### TPC-DS
-
-Comet currently provides an 23% speedup for TPC-DS @ SF=100GB. Note that we
used an optimized version of
-query 72 with a better join order for these benchmarks since the focus of
Spark (and Comet) is not on join
-reordering algorithms but raw execution speed.
-
-
-
-Here is a breakdown showing relative performance of Spark and Comet for each
query. DataFusion is
-not included here because it currently only supports around 90% of the TPC-DS
queries.
-
-
-
-The following chart shows how much Comet currently accelerates each query from
the benchmark.
-
-
-
-The raw results of these benchmarks in JSON format is available here:
-
-- [Spark](./benchmark-results/2024-07-19/spark-tpcds.json)
-- [Comet](./benchmark-results/2024-07-19/comet-tpcds.json)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
