This is an automated email from the ASF dual-hosted git repository. taiyang-li pushed a commit to branch fake_add_bolt_backend in repository https://gitbox.apache.org/repos/asf/gluten.git
commit aa18e3ba2255be4b34c805820c5e449f87bbe517 Author: liyang.127 <[email protected]> AuthorDate: Sat Jun 27 20:09:47 2026 +0800 [GLUTEN][CORE] Move benchmark data to shared directory Share the Velox benchmark input data under a backend-neutral path so future backends can reuse the same files without duplicating references. --- .../org/apache/gluten/execution/MiscOperatorSuite.scala | 2 +- .../benchmarks/data/bm_lineitem/orc/lineitem.orc | Bin .../benchmarks/data/bm_lineitem/orc/long_decimal_nonull.orc | Bin ...000-482d0e2c-7667-42bb-8750-64ecfe331d28-c000.snappy.orc | Bin .../data/bm_lineitem/orc/short_decimal_nonull.orc | Bin ...3a972d10-5e0b-4bde-b250-032afd81a132-c000.snappy.parquet | Bin .../benchmarks/data/generic_q1/q1_first_stage_0.json | 0 .../benchmarks/data/generic_q5/q5_first_stage_0.json | 0 .../benchmarks/data/generic_q5/q5_first_stage_0_split.json | 0 .../benchmarks/data/generic_q5/q5_middle_stage_0.json | 0 .../data/generic_q5/q5_middle_stage_0_iter_0.parquet | Bin .../data/generic_q5/q5_middle_stage_0_iter_1.parquet | Bin cpp/{velox => }/benchmarks/data/plan/q17_joins.json | 0 cpp/{velox => }/benchmarks/data/plan/select.json | 0 .../benchmarks/data/plan/select_long_decimal.json | 0 .../benchmarks/data/plan/select_short_decimal.json | 0 ...a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet | Bin ...6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet | Bin ...8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet | Bin ...41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet | Bin ...e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet | Bin ...2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet | Bin ...90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet | Bin ...0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet | Bin docs/developers/MicroBenchmarks.md | 12 ++++++------ 25 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala index fcf1fbbbf7..63650844c6 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala @@ -735,7 +735,7 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa test("orc scan") { val df = spark.read .format("orc") - .load("../cpp/velox/benchmarks/data/bm_lineitem/orc/lineitem.orc") + .load("../../cpp/benchmarks/data/bm_lineitem/orc/lineitem.orc") df.createOrReplaceTempView("lineitem_orc") runQueryAndCompare("select l_orderkey from lineitem_orc") { df => diff --git a/cpp/velox/benchmarks/data/bm_lineitem/orc/lineitem.orc b/cpp/benchmarks/data/bm_lineitem/orc/lineitem.orc similarity index 100% rename from cpp/velox/benchmarks/data/bm_lineitem/orc/lineitem.orc rename to cpp/benchmarks/data/bm_lineitem/orc/lineitem.orc diff --git a/cpp/velox/benchmarks/data/bm_lineitem/orc/long_decimal_nonull.orc b/cpp/benchmarks/data/bm_lineitem/orc/long_decimal_nonull.orc similarity index 100% rename from cpp/velox/benchmarks/data/bm_lineitem/orc/long_decimal_nonull.orc rename to cpp/benchmarks/data/bm_lineitem/orc/long_decimal_nonull.orc diff --git a/cpp/velox/benchmarks/data/bm_lineitem/orc/part-00000-482d0e2c-7667-42bb-8750-64ecfe331d28-c000.snappy.orc b/cpp/benchmarks/data/bm_lineitem/orc/part-00000-482d0e2c-7667-42bb-8750-64ecfe331d28-c000.snappy.orc similarity index 100% rename from cpp/velox/benchmarks/data/bm_lineitem/orc/part-00000-482d0e2c-7667-42bb-8750-64ecfe331d28-c000.snappy.orc rename to cpp/benchmarks/data/bm_lineitem/orc/part-00000-482d0e2c-7667-42bb-8750-64ecfe331d28-c000.snappy.orc diff --git a/cpp/velox/benchmarks/data/bm_lineitem/orc/short_decimal_nonull.orc b/cpp/benchmarks/data/bm_lineitem/orc/short_decimal_nonull.orc similarity index 100% rename from cpp/velox/benchmarks/data/bm_lineitem/orc/short_decimal_nonull.orc rename to cpp/benchmarks/data/bm_lineitem/orc/short_decimal_nonull.orc diff --git a/cpp/velox/benchmarks/data/bm_lineitem/parquet/part-00287-3a972d10-5e0b-4bde-b250-032afd81a132-c000.snappy.parquet b/cpp/benchmarks/data/bm_lineitem/parquet/part-00287-3a972d10-5e0b-4bde-b250-032afd81a132-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/bm_lineitem/parquet/part-00287-3a972d10-5e0b-4bde-b250-032afd81a132-c000.snappy.parquet rename to cpp/benchmarks/data/bm_lineitem/parquet/part-00287-3a972d10-5e0b-4bde-b250-032afd81a132-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/generic_q1/q1_first_stage_0.json b/cpp/benchmarks/data/generic_q1/q1_first_stage_0.json similarity index 100% rename from cpp/velox/benchmarks/data/generic_q1/q1_first_stage_0.json rename to cpp/benchmarks/data/generic_q1/q1_first_stage_0.json diff --git a/cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0.json b/cpp/benchmarks/data/generic_q5/q5_first_stage_0.json similarity index 100% rename from cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0.json rename to cpp/benchmarks/data/generic_q5/q5_first_stage_0.json diff --git a/cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0_split.json b/cpp/benchmarks/data/generic_q5/q5_first_stage_0_split.json similarity index 100% rename from cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0_split.json rename to cpp/benchmarks/data/generic_q5/q5_first_stage_0_split.json diff --git a/cpp/velox/benchmarks/data/generic_q5/q5_middle_stage_0.json b/cpp/benchmarks/data/generic_q5/q5_middle_stage_0.json similarity index 100% rename from cpp/velox/benchmarks/data/generic_q5/q5_middle_stage_0.json rename to cpp/benchmarks/data/generic_q5/q5_middle_stage_0.json diff --git a/cpp/velox/benchmarks/data/generic_q5/q5_middle_stage_0_iter_0.parquet b/cpp/benchmarks/data/generic_q5/q5_middle_stage_0_iter_0.parquet similarity index 100% rename from cpp/velox/benchmarks/data/generic_q5/q5_middle_stage_0_iter_0.parquet rename to cpp/benchmarks/data/generic_q5/q5_middle_stage_0_iter_0.parquet diff --git a/cpp/velox/benchmarks/data/generic_q5/q5_middle_stage_0_iter_1.parquet b/cpp/benchmarks/data/generic_q5/q5_middle_stage_0_iter_1.parquet similarity index 100% rename from cpp/velox/benchmarks/data/generic_q5/q5_middle_stage_0_iter_1.parquet rename to cpp/benchmarks/data/generic_q5/q5_middle_stage_0_iter_1.parquet diff --git a/cpp/velox/benchmarks/data/plan/q17_joins.json b/cpp/benchmarks/data/plan/q17_joins.json similarity index 100% rename from cpp/velox/benchmarks/data/plan/q17_joins.json rename to cpp/benchmarks/data/plan/q17_joins.json diff --git a/cpp/velox/benchmarks/data/plan/select.json b/cpp/benchmarks/data/plan/select.json similarity index 100% rename from cpp/velox/benchmarks/data/plan/select.json rename to cpp/benchmarks/data/plan/select.json diff --git a/cpp/velox/benchmarks/data/plan/select_long_decimal.json b/cpp/benchmarks/data/plan/select_long_decimal.json similarity index 100% rename from cpp/velox/benchmarks/data/plan/select_long_decimal.json rename to cpp/benchmarks/data/plan/select_long_decimal.json diff --git a/cpp/velox/benchmarks/data/plan/select_short_decimal.json b/cpp/benchmarks/data/plan/select_short_decimal.json similarity index 100% rename from cpp/velox/benchmarks/data/plan/select_short_decimal.json rename to cpp/benchmarks/data/plan/select_short_decimal.json diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/customer/part-00000-a1a42661-7a85-42da-b831-f489a5545d61-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/nation/part-00000-8a852c7b-da96-4425-9ce8-d188f97f3afe-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/orders/part-00000-41cc94ee-a98d-4103-83b1-0b5bcebe03eb-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/part/part-00000-e664e655-0ada-4e0b-a8aa-213d1e4c3797-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/partsupp/part-00000-2c7e524c-3f91-4228-b003-64ad2c8059a1-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/region/part-00000-90d7c5d9-46eb-4bf3-8f00-6eee7436761d-c000.snappy.parquet diff --git a/cpp/velox/benchmarks/data/tpch_sf10m/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet b/cpp/benchmarks/data/tpch_sf10m/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet similarity index 100% rename from cpp/velox/benchmarks/data/tpch_sf10m/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet rename to cpp/benchmarks/data/tpch_sf10m/supplier/part-00000-0a763951-8a7b-4d7f-a34a-c97e9b4859f0-c000.snappy.parquet diff --git a/docs/developers/MicroBenchmarks.md b/docs/developers/MicroBenchmarks.md index ce0996a6cc..97edb9b648 100644 --- a/docs/developers/MicroBenchmarks.md +++ b/docs/developers/MicroBenchmarks.md @@ -426,24 +426,24 @@ done ### Run Examples -We also provide some example inputs in [cpp/velox/benchmarks/data](../../cpp/velox/benchmarks/data). -E.g. Files under [generic_q5](../../cpp/velox/benchmarks/data/generic_q5) simulates a first-stage in +We also provide some example inputs in [cpp/benchmarks/data](../../cpp/benchmarks/data). +E.g. Files under [generic_q5](../../cpp/benchmarks/data/generic_q5) simulates a first-stage in TPCH Q5, which has a heavy table scan. You can follow below steps to run this example. 1. -Open [generic_q5/q5_first_stage_0_split.json](../../cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0_split.json) +Open [generic_q5/q5_first_stage_0_split.json](../../cpp/benchmarks/data/generic_q5/q5_first_stage_0_split.json) with file editor. Search for `"uriFile": "LINEITEM"` and replace `LINEITEM` with the URI to one partition file in lineitem. In the next line, replace the number in `"length": "..."` with the actual file length. Suppose you are using the provided small TPCH table -in [cpp/velox/benchmarks/data/tpch_sf10m](../../cpp/velox/benchmarks/data/tpch_sf10m), the replaced +in [cpp/benchmarks/data/tpch_sf10m](../../cpp/benchmarks/data/tpch_sf10m), the replaced JSON should be like: ``` { "items": [ { - "uriFile": "file:///path/to/gluten/cpp/velox/benchmarks/data/tpch_sf10m/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet", + "uriFile": "file:///path/to/gluten/cpp/benchmarks/data/tpch_sf10m/lineitem/part-00000-6c374e0a-7d76-401b-8458-a8e31f8ab704-c000.snappy.parquet", "length": "1863237", "parquet": {} } @@ -462,7 +462,7 @@ processes=24 # Same value of spark.executor.instances threads=8 # Same value of spark.executor.cores for ((i=0; i<${processes}; i++)); do - ./generic_benchmark --plan /path/to/gluten/cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0.json --split /path/to/gluten/cpp/velox/benchmarks/data/generic_q5/q5_first_stage_0_split.json --noprint-result --with-shuffle --threads $threads --cpu $((i*threads)) & + ./generic_benchmark --plan /path/to/gluten/cpp/benchmarks/data/generic_q5/q5_first_stage_0.json --split /path/to/gluten/cpp/benchmarks/data/generic_q5/q5_first_stage_0_split.json --noprint-result --with-shuffle --threads $threads --cpu $((i*threads)) & done >stdout.log 2>stderr.log ``` --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
