This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a change to branch chore/track-datafusion-git-main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
from 50b1b9ae9 chore(deps): track DataFusion git main and port to its APIs
add a8b3c79c8 feat: broadcast small build side of SortMergeJoinExec in the
static planner (#1904)
add 84512cb5c Fix/python cargo lock drift (#1899)
add 6472c7f21 feat(scheduler): allow submitting an already-built physical
plan (#1924)
add b880ca419 feat: cancel running stages and tasks on job failure or
cancellation (#1903)
add 3b8701876 refactor(scheduler): keep submit_job non-breaking, add
submit_physical_plan (#1941)
add ea9adfbea feat(benchmarks): support reading TPC-H data from
S3-compatible object storage (#1942)
add 6ccbcd54c docs: add versioned upgrade guide (54.0.0) (#1939)
add e05fe5ea3 feat: bound shuffle fetch with a reduce-side in-flight
governor (#1951)
add ce9eefabc chore(deps): bump ctor from 1.0.7 to 1.0.8 (#1957)
add 182222743 chore(deps): bump bytesize from 2.4.0 to 2.4.2 (#1956)
add e0855643d chore(deps): bump astral-sh/setup-uv from 8.2.0 to 8.3.0
(#1955)
add a96c381fb chore(deps): bump taiki-e/install-action from 2.82.6 to
2.82.7 (#1918)
add 24e95f9ff fix AQE planning with S3 (#1962)
add 4f1d7da52 Add TPC-H distributed plan-stability test suite (#1966)
add 5298a9f1f feat: display input stage id on UnresolvedShuffleExec
(#1967) (#1969)
add ce768eb35 chore(deps): bump aws-config from 1.8.18 to 1.9.0 (#1976)
add 006af4b87 chore(deps): bump aws-credential-types from 1.2.14 to 1.3.0
(#1975)
add b93bcc0df chore(deps): bump astral-sh/setup-uv from 8.3.0 to 8.3.2
(#1970)
add 8a4ef0bd3 chore(deps): bump actions/labeler from 6.1.0 to 6.2.0 (#1972)
add fd30f3775 chore(deps): bump taiki-e/install-action from 2.82.7 to
2.82.11 (#1973)
add 647cefe4d docs: document TPC-H plan-stability golden files and
regeneration (#1979)
add ec79000d4 fix: satisfy clippy 1.97 useless_borrows_in_formatting
(#1978)
add 2959cb7f9 perf: avoid redundant TaskStatus clones in scheduler
update_task_status (#1983)
add 434012f2e chore(deps): bump regex from 1.12.4 to 1.13.0 (#1985)
add 4f5e7c71e chore(deps): bump taiki-e/install-action from 2.82.11 to
2.83.0 (#1984)
add aad835288 chore(deps): bump pyarrow from 23.0.0 to 23.0.1 in /python
(#1961)
add bf9d9b449 chore(deps): bump uuid from 1.23.1 to 1.23.4 in /python
(#1936)
add 2555985e4 chore(deps): bump insta from 1.47.2 to 1.48.0 in /python
(#1935)
add 2ed13c7b0 chore(deps): bump pyo3-log from 0.13.3 to 0.13.4 in /python
(#1934)
add 4bd3d0e0b chore(deps): bump rand from 0.10.1 to 0.10.2 (#1933)
add 4365c19f7 chore(deps): bump log from 0.4.29 to 0.4.33 in /python
(#1932)
add 168406090 chore(deps): bump pyo3-build-config from 0.28.3 to 0.29.0 in
/python (#1931)
add 4b2a8d4a2 chore(deps): bump rand from 0.10.1 to 0.10.2 in /python
(#1928)
add 36b661ad7 chore(deps): bump prost from 0.14.3 to 0.14.4 in /python
(#1929)
add 551089d61 chore(deps): bump github/codeql-action/init from 4.36.2 to
4.37.0 (#1974)
add 8dad3d50d docs: update companies using ballista (#1992)
add 9d75f1148 chore: add script and CI check to sync vendored DataFusion
protos (#1991)
add 1cd1affc4 feat: improve state creation (#1963)
add ac41a2224 docs: link AQE limitations to tracking issues and add AQE
contributor guide section (#1990)
add efaeda5ba feat(tui + rest): surface failed tasks (#1949)
add b491d3cf7 Add shuffle-read fetch metrics and surface per-operator
metrics in plan display (#1968)
add e1e9514ba feat: delete intermediate shuffle files on job success
(#1982)
add daebb3c92 feat(executor): log task execution duration in finished-task
message (#1999)
add 82db5595d perf: prune irrelevant partitions task (#1911)
add f67969cf6 feat: Share read-side runtime state across a session's tasks
on the executor (#1995)
add d54b48f36 chore: bump Ballista version to 54.0.0 (#2004)
add acede2739 docs: add user personas as a review contract for additive
changes (#2002)
add ac9bc060b docs: include TestPyPI wheels link in release vote email
(#2010)
add 51be47517 docs: remove outdated TPC-H benchmark results from READMEs
(#2015)
add 6901da089 chore(deps): bump uuid from 1.23.4 to 1.23.5 in /python
(#2021)
add b85ab0801 chore(deps): bump prost-types from 0.14.3 to 0.14.4 in
/python (#2020)
add cd5a88cd9 chore(deps): bump uuid from 1.23.4 to 1.23.5 (#2019)
add 7a91d406e chore(deps): bump lru from 0.18.0 to 0.18.1 (#2018)
add 3985f1074 chore(deps): bump taiki-e/install-action from 2.83.0 to
2.83.2 (#2017)
add 5dacdf02c ci: install Rust via rustup in wheel build to fix ASF
allowlist startup_failure (#2008)
add cf43961f8 chore(deps): bump rustls from 0.23.41 to 0.23.42 (#2035)
add de6d61519 Update project description to remove Apache Arrow (#2044)
add eff11b165 feat: make compression codec configurable (#2039)
add f45996cfb fix(scheduler): persist terminal status before cache
eviction (#2037)
add 30222651c chore(deps): bump uuid from 1.23.5 to 1.24.0 (#2054)
add dc91baedb chore(deps): bump regex from 1.13.0 to 1.13.1 (#2053)
add 524f0842d chore(deps): bump uuid from 1.23.5 to 1.24.0 in /python
(#2052)
add 713599866 chore(deps): bump ctor from 1.0.8 to 1.0.9 (#2051)
add b758d35e7 chore(deps): bump clap from 4.6.1 to 4.6.2 (#2050)
add 4974e678c chore(deps): bump cryptography from 46.0.3 to 48.0.1 in
/python (#2056)
add 48b08b884 chore(deps): bump pyjwt from 2.12.0 to 2.13.0 in /python
(#2055)
add 874d14908 chore(deps): bump actions/setup-node from 6.4.0 to 7.0.0
(#2034)
add bcd9b56ce chore(ci): Add a CI workflow that checks that the used CI
actions and their versions are allowed (#2058)
add 1c25d30d7 refactor(tui): Drop the header banner/logo (#2057)
add e843c8e86 Add TPC-DS correctness testing (#2048)
add 173e28f6a fix: spill sort shuffle writer when the memory pool rejects
a reservation grow (#2061)
add 66af2392a fix: enable the executor client pool by default (#2069)
add f3fc9d0ca Rename `task_slots` -> `vcores` (executor spec + CLI) (#2045)
add 449e7b5ef fix: map a task's partition through UnionExec when
restricting file scans (#2070)
add 0f6ec8c6c fix: keep EmptyExec partition count intact across stage
serialization (#2062)
add 26b293918 fix: remove the static planner SortMergeJoinExec broadcast
conversion (#2072)
add bce81902f docs: add benchmarking page to the contributors guide (#2075)
add c4260b828 ci: skip expensive workflows for docs-only changes (#2073)
add 992c42b06 chore(deps): bump tokio from 1.52.3 to 1.52.4 (#2080)
add 6d517b4a4 chore(deps): bump tokio from 1.52.3 to 1.52.4 in /python
(#2079)
add 160bb3326 chore(deps): bump taiki-e/install-action from 2.83.2 to
2.83.3 (#2076)
add fcf2d92c4 chore(deps): bump github/codeql-action/init from 4.37.0 to
4.37.1 (#2078)
add 543a86144 chore(deps): bump tokio from 1.52.4 to 1.53.0 (#2083)
add fbd8525bc chore(deps): bump tokio from 1.52.4 to 1.53.0 in /python
(#2082)
add fd5aa65d6 feat: make broadcast_join_threshold_bytes/rows authoritative
under AQE and the static planner (#2086)
add 847ee7c45 feat: add BALLISTA_PROTOCOL_VERSION + k8s health probes
(#2088)
add 2ed3464df Multi-partition tasks: partition_slice plumbing +
SortShuffleWriter refactor (#2038)
add 968086f6c docs(config): rewrite max_partitions_per_task description to
be timeless (#2093)
add 752d4de13 fix: propagate sort shuffle memory_limit_per_task_bytes to
executors (#2090)
add 5408c7613 feat: allow an external semaphore for executor task
concurrency (#1892)
add 2d938371a feat: log sort shuffle spills at WARN, keep write completion
at DEBUG (#2091)
add 9ca2d354b chore(deps): bump async-trait from 0.1.89 to 0.1.91 (#2100)
add 0d7253d71 chore(deps): bump futures from 0.3.32 to 0.3.33 (#2099)
add 92198ed12 chore(deps): bump serde from 1.0.228 to 1.0.229 (#2097)
add f3542e00f chore(ci): bump taiki-e/install-action from 2.83.3 to 2.83.4
(#2096)
add 49d1fec80 chore(ci): bump actions/setup-python from 6 to 7 (#2098)
add 60c5b26c9 chore(scheduler,executor): downgrade #2038 per-task traces
to debug (#2104)
add d518d569a feat(benchmarks): persist TPC-H results resiliently and add
a compare subcommand (#2102)
add c77e48127 Fixed: notify_shutdown sending signal before drop (#2107)
add 99ba9e860 docs: refresh SF1000 benchmark results on 2x16-core AQE-on
reference cluster (#2109)
add 0e0a695e1 chore(deps): bump clap from 4.6.2 to 4.6.3 (#2118)
add f8efcdf9a chore(deps): bump libc from 0.2.186 to 0.2.187 (#2116)
add 826834a09 chore(deps): bump tokio from 1.53.0 to 1.53.1 in /python
(#2115)
add 97dab80e1 chore(deps): bump tokio from 1.53.0 to 1.53.1 (#2114)
add 81a184ecf chore(ci): bump actions/checkout from 7.0.0 to 7.0.1 (#2113)
add e6f657569 chore(ci): bump actions/labeler from 6.2.0 to 7.0.0 (#2112)
add 0dac3493d chore(deps): bump serde_json from 1.0.150 to 1.0.151 (#2111)
add d3a8724e9 chore(ci): bump taiki-e/install-action from 2.83.4 to 2.84.0
(#2110)
add bb2fdfee2 ci(tpch): add AQE-on smoke pass with multi-partition tasks
(#2120)
add b16270801 see through Shared wrapper when classifying task IO errors
(#2119)
add 62228ecc2 feat(core): RuntimeStatsExec — passthrough tap for row
counts + quantile sketches (#2094)
add b80947575 feat(core): add BufferExec — generic flow-control op
with a memory-pool Dam mode (#2095)
add bd17e435c feat: Improve TPC-H performance with AQE on (#2084)
add 6540d0851 Remove the hash-partitioning shuffle writer (#2106)
add 8be1e8986 docs: correct scan partitioning description in tuning guide
(#2125)
add 65dc2dfb4 feat(benchmarks): add --skip to omit queries from a tpch
suite run (#2129)
add 9b587954a chore: upgrade DataFusion to 54.1.0 (#2126)
add 2c9367033 fix: Various fixes for AQE join selection (#2124)
add af7db0be4 feat(core): report the actual sort shuffle spill trigger in
logs (#2143)
add b55d1e99e Generate the config reference documentation from the config
registry (#2146)
add ce115795f fix: preserve null_equality so distributed INTERSECT/EXCEPT
match NULLs (#2064)
add 2bb0e8eb9 feat(core): add UnorderedRangeRepartitionExec — value-range
router over unordered inputs (#2123)
add 7e2e4d95c chore(deps): bump tokio-stream from 0.1.18 to 0.1.19 in
/python (#2157)
add ba6ae5c71 chore(deps): bump tokio-stream from 0.1.18 to 0.1.19 (#2156)
add 696ca29b3 fix(executor): keep heartbeating while all task slots are
busy (#2159)
add d738ca9bd docs: refresh benchmarking.md with current SF1000 results
(Ballista main @ 696ca29b, Spark 3.4 baseline) (#2161)
add 90f8096ed feat(executor): default to a bounded, auto-sized memory pool
(#2160)
add 9efb1f46e docs: refresh benchmarking.md - dedicated shuffle PVC, r6i
node, updated numbers (#2171)
add 2904c4dc8 Merge apache/main into chore/track-datafusion-git-main; bump
datafusion rev
No new revisions were added by this update.
Summary of changes:
.github/dependabot.yml | 35 +-
.../workflows/allowlist-check.yml | 27 +-
.github/workflows/build.yml | 71 +-
.github/workflows/codeql.yml | 6 +-
.github/workflows/dependencies.yml | 27 +-
.github/workflows/dev.yml | 8 +-
.github/workflows/dev_pr.yml | 2 +-
.github/workflows/docker.yml | 10 +-
.github/workflows/docs.yaml | 6 +-
.github/workflows/rust.yml | 82 +-
.github/workflows/{tpch.yml => tpcds.yml} | 86 +-
.github/workflows/tpch.yml | 80 +-
.github/workflows/web-tui.yml | 20 +-
.gitignore | 1 +
CONTRIBUTING.md | 16 +
Cargo.lock | 1439 ++++++++------------
Cargo.toml | 13 +-
README.md | 43 +-
ballista-cli/Cargo.toml | 7 +-
ballista-cli/README.md | 4 +-
ballista-cli/Trunk.toml | 9 -
ballista-cli/ballista-web-tui.html | 14 -
ballista-cli/src/main.rs | 31 +-
ballista-cli/src/tui/app.rs | 6 +-
ballista-cli/src/tui/domain/executors.rs | 36 +-
ballista-cli/src/tui/domain/jobs/stages.rs | 14 +-
ballista-cli/src/tui/infrastructure/theme.rs | 57 +-
ballista-cli/src/tui/ui/header/ballista-logo.svg | 29 -
ballista-cli/src/tui/ui/header/mod.rs | 52 +-
ballista-cli/src/tui/ui/header/scheduler_state.rs | 2 +-
.../src/tui/ui/main/executors/executors_table.rs | 12 +-
.../src/tui/ui/main/jobs/stage_tasks_popup.rs | 24 +-
ballista/client/Cargo.toml | 13 +-
ballista/client/README.md | 24 -
ballista/client/src/extension.rs | 8 +-
ballista/client/tests/context_checks.rs | 6 +-
ballista/client/tests/physical_plan_submission.rs | 132 ++
ballista/client/tests/sort_shuffle.rs | 123 +-
ballista/core/Cargo.toml | 3 +-
ballista/core/proto/ballista.proto | 146 +-
ballista/core/proto/datafusion.proto | 95 +-
ballista/core/proto/datafusion_common.proto | 50 +-
ballista/core/src/bin/update_config_docs.rs | 835 ++++++++++++
ballista/core/src/client.rs | 12 +
ballista/core/src/config.rs | 292 +++-
ballista/core/src/error.rs | 75 +-
ballista/core/src/execution_plans/buffer.rs | 523 +++++++
ballista/core/src/execution_plans/chaos_exec.rs | 13 +-
.../execution_plans/distributed_explain_analyze.rs | 3 +
.../core/src/execution_plans/distributed_query.rs | 88 ++
ballista/core/src/execution_plans/mod.rs | 10 +-
.../execution_plans/range_repartition_common.rs | 522 +++++++
ballista/core/src/execution_plans/runtime_stats.rs | 742 ++++++++++
.../core/src/execution_plans/shuffle_reader.rs | 703 +++++++++-
.../core/src/execution_plans/shuffle_writer.rs | 1154 ++++++++++------
.../src/execution_plans/sort_shuffle/buffer.rs | 5 +-
.../src/execution_plans/sort_shuffle/config.rs | 16 +-
.../core/src/execution_plans/sort_shuffle/spill.rs | 22 +-
.../src/execution_plans/sort_shuffle/writer.rs | 791 +++++++++--
.../execution_plans/unordered_range_repartition.rs | 931 +++++++++++++
.../core/src/execution_plans/unresolved_shuffle.rs | 22 +-
ballista/core/src/extension.rs | 82 +-
ballista/core/src/lib.rs | 12 +
ballista/core/src/object_store.rs | 2 +-
ballista/core/src/serde/generated/ballista.rs | 198 ++-
ballista/core/src/serde/mod.rs | 661 ++++++++-
ballista/core/src/serde/scheduler/from_proto.rs | 99 +-
ballista/core/src/serde/scheduler/mod.rs | 106 +-
ballista/core/src/serde/scheduler/to_proto.rs | 160 ++-
ballista/core/src/utils.rs | 41 +-
ballista/executor/Cargo.toml | 8 +-
ballista/executor/src/bin/main.rs | 27 +-
ballista/executor/src/client_pool.rs | 2 +
ballista/executor/src/collect.rs | 14 +-
ballista/executor/src/config.rs | 117 +-
ballista/executor/src/execution_engine.rs | 352 +++--
ballista/executor/src/execution_loop.rs | 143 +-
ballista/executor/src/executor.rs | 169 ++-
ballista/executor/src/executor_process.rs | 670 +++++++--
ballista/executor/src/executor_server.rs | 115 +-
ballista/executor/src/flight_service.rs | 2 +-
ballista/executor/src/health.rs | 143 ++
ballista/executor/src/lib.rs | 25 +-
ballista/executor/src/metrics/mod.rs | 6 +-
ballista/executor/src/runtime_cache.rs | 231 ++++
ballista/executor/src/standalone.rs | 25 +-
ballista/scheduler/Cargo.toml | 9 +-
ballista/scheduler/src/api/handlers.rs | 200 ++-
ballista/scheduler/src/api/health.rs | 163 +++
ballista/scheduler/src/api/mod.rs | 2 +
ballista/scheduler/src/cluster/memory.rs | 40 +-
ballista/scheduler/src/cluster/mod.rs | 452 +++---
ballista/scheduler/src/config.rs | 15 +-
ballista/scheduler/src/metrics/mod.rs | 22 +
.../src/physical_optimizer/join_selection.rs | 7 +-
ballista/scheduler/src/planner.rs | 411 +++++-
ballista/scheduler/src/scheduler_process.rs | 11 +-
ballista/scheduler/src/scheduler_server/event.rs | 27 +-
ballista/scheduler/src/scheduler_server/grpc.rs | 285 +++-
ballista/scheduler/src/scheduler_server/mod.rs | 152 ++-
.../src/scheduler_server/query_stage_scheduler.rs | 80 +-
.../src/state/aqe/execution_plan/dynamic_join.rs | 894 +++++++++++-
ballista/scheduler/src/state/aqe/mod.rs | 186 ++-
.../src/state/aqe/optimizer_rule/join_selection.rs | 5 +-
.../state/aqe/optimizer_rule/propagate_empty.rs | 7 +-
ballista/scheduler/src/state/aqe/planner.rs | 35 +-
.../scheduler/src/state/aqe/test/alter_stages.rs | 10 +-
.../src/state/aqe/test/broadcast_thresholds.rs | 299 ++++
.../scheduler/src/state/aqe/test/coalesce_rule.rs | 2 +-
.../scheduler/src/state/aqe/test/job_failure.rs | 137 ++
.../scheduler/src/state/aqe/test/join_selection.rs | 38 +-
ballista/scheduler/src/state/aqe/test/mod.rs | 26 +-
.../scheduler/src/state/aqe/test/plan_to_stages.rs | 33 +-
.../scheduler/src/state/aqe/test/stats_table.rs | 232 ++++
ballista/scheduler/src/state/execution_graph.rs | 422 ++++--
ballista/scheduler/src/state/execution_stage.rs | 894 ++++++++----
ballista/scheduler/src/state/executor_manager.rs | 75 +-
ballista/scheduler/src/state/mod.rs | 30 +-
ballista/scheduler/src/state/task_builder.rs | 598 ++++++++
ballista/scheduler/src/state/task_manager.rs | 885 +++++++++---
ballista/scheduler/src/test_utils.rs | 67 +-
.../scheduler/tests/tpch_plan_stability/README.md | 59 +
.../tests/tpch_plan_stability/approved/q1.txt | 18 +
.../tests/tpch_plan_stability/approved/q10.txt | 57 +
.../tests/tpch_plan_stability/approved/q11.txt | 81 ++
.../tests/tpch_plan_stability/approved/q12.txt | 30 +
.../tests/tpch_plan_stability/approved/q13.txt | 32 +
.../tests/tpch_plan_stability/approved/q14.txt | 25 +
.../tests/tpch_plan_stability/approved/q15.txt | 53 +
.../tests/tpch_plan_stability/approved/q16.txt | 44 +
.../tests/tpch_plan_stability/approved/q17.txt | 36 +
.../tests/tpch_plan_stability/approved/q18.txt | 46 +
.../tests/tpch_plan_stability/approved/q19.txt | 26 +
.../tests/tpch_plan_stability/approved/q2.txt | 123 ++
.../tests/tpch_plan_stability/approved/q20.txt | 66 +
.../tests/tpch_plan_stability/approved/q21.txt | 79 ++
.../tests/tpch_plan_stability/approved/q22.txt | 35 +
.../tests/tpch_plan_stability/approved/q3.txt | 40 +
.../tests/tpch_plan_stability/approved/q4.txt | 31 +
.../tests/tpch_plan_stability/approved/q5.txt | 83 ++
.../tests/tpch_plan_stability/approved/q6.txt | 6 +
.../tests/tpch_plan_stability/approved/q7.txt | 84 ++
.../tests/tpch_plan_stability/approved/q8.txt | 110 ++
.../tests/tpch_plan_stability/approved/q9.txt | 82 ++
.../tests/tpch_plan_stability/fixtures.rs | 229 ++++
.../scheduler/tests/tpch_plan_stability/main.rs | 115 ++
.../tests/tpch_plan_stability/stats_table.rs | 152 +++
benchmarks/.gitignore | 3 +-
benchmarks/Cargo.toml | 8 +-
benchmarks/README.md | 102 ++
benchmarks/benches/sort_shuffle.rs | 3 +-
benchmarks/queries-tpcds/q1.sql | 26 +
benchmarks/queries-tpcds/q10.sql | 60 +
benchmarks/queries-tpcds/q11.sql | 82 ++
benchmarks/queries-tpcds/q12.sql | 35 +
benchmarks/queries-tpcds/q13.sql | 53 +
benchmarks/queries-tpcds/q14.sql | 211 +++
benchmarks/queries-tpcds/q15.sql | 21 +
benchmarks/queries-tpcds/q16.sql | 32 +
benchmarks/queries-tpcds/q17.sql | 46 +
benchmarks/queries-tpcds/q18.sql | 35 +
benchmarks/queries-tpcds/q19.sql | 26 +
benchmarks/queries-tpcds/q2.sql | 61 +
benchmarks/queries-tpcds/q20.sql | 31 +
benchmarks/queries-tpcds/q21.sql | 31 +
benchmarks/queries-tpcds/q22.sql | 21 +
benchmarks/queries-tpcds/q23.sql | 108 ++
benchmarks/queries-tpcds/q24.sql | 108 ++
benchmarks/queries-tpcds/q25.sql | 49 +
benchmarks/queries-tpcds/q26.sql | 22 +
benchmarks/queries-tpcds/q27.sql | 24 +
benchmarks/queries-tpcds/q28.sql | 54 +
benchmarks/queries-tpcds/q29.sql | 48 +
benchmarks/queries-tpcds/q3.sql | 22 +
benchmarks/queries-tpcds/q30.sql | 31 +
benchmarks/queries-tpcds/q31.sql | 53 +
benchmarks/queries-tpcds/q32.sql | 29 +
benchmarks/queries-tpcds/q33.sql | 76 ++
benchmarks/queries-tpcds/q34.sql | 32 +
benchmarks/queries-tpcds/q35.sql | 59 +
benchmarks/queries-tpcds/q36.sql | 31 +
benchmarks/queries-tpcds/q37.sql | 18 +
benchmarks/queries-tpcds/q38.sql | 24 +
benchmarks/queries-tpcds/q39.sql | 55 +
benchmarks/queries-tpcds/q4.sql | 117 ++
benchmarks/queries-tpcds/q40.sql | 29 +
benchmarks/queries-tpcds/q41.sql | 53 +
benchmarks/queries-tpcds/q42.sql | 23 +
benchmarks/queries-tpcds/q43.sql | 20 +
benchmarks/queries-tpcds/q44.sql | 36 +
benchmarks/queries-tpcds/q45.sql | 21 +
benchmarks/queries-tpcds/q46.sql | 36 +
benchmarks/queries-tpcds/q47.sql | 52 +
benchmarks/queries-tpcds/q48.sql | 68 +
benchmarks/queries-tpcds/q49.sql | 130 ++
benchmarks/queries-tpcds/q5.sql | 129 ++
benchmarks/queries-tpcds/q50.sql | 60 +
benchmarks/queries-tpcds/q51.sql | 46 +
benchmarks/queries-tpcds/q52.sql | 23 +
benchmarks/queries-tpcds/q53.sql | 29 +
benchmarks/queries-tpcds/q54.sql | 57 +
benchmarks/queries-tpcds/q55.sql | 15 +
benchmarks/queries-tpcds/q56.sql | 70 +
benchmarks/queries-tpcds/q57.sql | 49 +
benchmarks/queries-tpcds/q58.sql | 66 +
benchmarks/queries-tpcds/q59.sql | 45 +
benchmarks/queries-tpcds/q6.sql | 27 +
benchmarks/queries-tpcds/q60.sql | 79 ++
benchmarks/queries-tpcds/q61.sql | 45 +
benchmarks/queries-tpcds/q62.sql | 36 +
benchmarks/queries-tpcds/q63.sql | 30 +
benchmarks/queries-tpcds/q64.sql | 122 ++
benchmarks/queries-tpcds/q65.sql | 30 +
benchmarks/queries-tpcds/q66.sql | 221 +++
benchmarks/queries-tpcds/q67.sql | 45 +
benchmarks/queries-tpcds/q68.sql | 43 +
benchmarks/queries-tpcds/q69.sql | 48 +
benchmarks/queries-tpcds/q7.sql | 22 +
benchmarks/queries-tpcds/q70.sql | 39 +
benchmarks/queries-tpcds/q71.sql | 41 +
benchmarks/queries-tpcds/q72.sql | 30 +
benchmarks/queries-tpcds/q73.sql | 29 +
benchmarks/queries-tpcds/q74.sql | 62 +
benchmarks/queries-tpcds/q75.sql | 71 +
benchmarks/queries-tpcds/q76.sql | 25 +
benchmarks/queries-tpcds/q77.sql | 109 ++
benchmarks/queries-tpcds/q78.sql | 59 +
benchmarks/queries-tpcds/q79.sql | 24 +
benchmarks/queries-tpcds/q8.sql | 109 ++
benchmarks/queries-tpcds/q80.sql | 97 ++
benchmarks/queries-tpcds/q81.sql | 32 +
benchmarks/queries-tpcds/q82.sql | 18 +
benchmarks/queries-tpcds/q83.sql | 68 +
benchmarks/queries-tpcds/q84.sql | 22 +
benchmarks/queries-tpcds/q85.sql | 85 ++
benchmarks/queries-tpcds/q86.sql | 27 +
benchmarks/queries-tpcds/q87.sql | 24 +
benchmarks/queries-tpcds/q88.sql | 95 ++
benchmarks/queries-tpcds/q89.sql | 29 +
benchmarks/queries-tpcds/q9.sql | 52 +
benchmarks/queries-tpcds/q90.sql | 23 +
benchmarks/queries-tpcds/q91.sql | 32 +
benchmarks/queries-tpcds/q92.sql | 31 +
benchmarks/queries-tpcds/q93.sql | 19 +
benchmarks/queries-tpcds/q94.sql | 30 +
benchmarks/queries-tpcds/q95.sql | 33 +
benchmarks/queries-tpcds/q96.sql | 17 +
benchmarks/queries-tpcds/q97.sql | 26 +
benchmarks/queries-tpcds/q98.sql | 34 +
benchmarks/queries-tpcds/q99.sql | 36 +
benchmarks/src/bin/shuffle_bench.rs | 100 +-
benchmarks/src/bin/tpcds.rs | 330 +++++
benchmarks/src/bin/tpch.rs | 1085 ++++++++-------
benchmarks/src/lib.rs | 368 +++++
benchmarks/{tpch-gen.sh => tpcds-gen.sh} | 30 +-
.../{rust_fmt.sh => rust_config_docs_check.sh} | 4 +-
ci/scripts/{rust_fmt.sh => rust_proto_check.sh} | 5 +-
dev/release/create-tarball.sh | 2 +
dev/release/rat_exclude_files.txt | 1 +
.../update-tpch-plan-stability.sh | 10 +-
.../rust_fmt.sh => dev/update_config_docs.sh | 6 +-
dev/update_datafusion_proto.py | 143 ++
dev/update_datafusion_versions.py | 8 +
...alone-entrypoint.sh => vendor-tpcds-queries.sh} | 23 +-
docker-compose.yml | 2 +-
docs/developer/architecture.md | 42 +
docs/source/_static/images/tpch_allqueries.png | Bin 27455 -> 0 bytes
.../source/_static/images/tpch_queries_compare.png | Bin 32843 -> 0 bytes
.../_static/images/tpch_queries_speedup_abs.png | Bin 37391 -> 0 bytes
.../_static/images/tpch_queries_speedup_rel.png | Bin 47169 -> 0 bytes
docs/source/contributors-guide/architecture.md | 62 +
docs/source/contributors-guide/benchmarking.md | 244 ++++
docs/source/contributors-guide/development.md | 34 +
docs/source/contributors-guide/user-personas.md | 148 ++
docs/source/index.rst | 8 +
docs/source/upgrading/54.0.0.md | 121 ++
docs/source/upgrading/55.0.0.md | 73 +
.../{user-guide/python => upgrading}/index.rst | 11 +-
docs/source/user-guide/cli.md | 2 +-
docs/source/user-guide/configs.md | 94 +-
docs/source/user-guide/deployment/docker.md | 2 +-
docs/source/user-guide/deployment/kubernetes.md | 98 ++
docs/source/user-guide/python/quickstart.md | 6 +-
docs/source/user-guide/tuning-guide.md | 216 +--
examples/Cargo.toml | 10 +-
examples/examples/mtls-cluster.rs | 9 +-
examples/examples/standalone-substrait.rs | 8 +-
examples/tests/object_store.rs | 143 ++
python/Cargo.lock | 803 +++++------
python/Cargo.toml | 14 +-
python/pyproject.toml | 4 +-
python/python/tests/test_context.py | 4 +-
python/src/cluster.rs | 12 +-
python/uv.lock | 224 ++-
294 files changed, 25835 insertions(+), 5285 deletions(-)
copy dev/docker/executor-entrypoint.sh =>
.github/workflows/allowlist-check.yml (60%)
mode change 100755 => 100644
copy .github/workflows/{tpch.yml => tpcds.yml} (69%)
delete mode 100644 ballista-cli/src/tui/ui/header/ballista-logo.svg
create mode 100644 ballista/client/tests/physical_plan_submission.rs
create mode 100644 ballista/core/src/bin/update_config_docs.rs
create mode 100644 ballista/core/src/execution_plans/buffer.rs
create mode 100644
ballista/core/src/execution_plans/range_repartition_common.rs
create mode 100644 ballista/core/src/execution_plans/runtime_stats.rs
create mode 100644
ballista/core/src/execution_plans/unordered_range_repartition.rs
create mode 100644 ballista/executor/src/health.rs
create mode 100644 ballista/executor/src/runtime_cache.rs
create mode 100644 ballista/scheduler/src/api/health.rs
create mode 100644
ballista/scheduler/src/state/aqe/test/broadcast_thresholds.rs
create mode 100644 ballista/scheduler/src/state/aqe/test/job_failure.rs
create mode 100644 ballista/scheduler/src/state/aqe/test/stats_table.rs
create mode 100644 ballista/scheduler/src/state/task_builder.rs
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/README.md
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q1.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q10.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q11.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q12.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q13.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q14.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q15.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q16.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q17.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q18.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q19.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q2.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q20.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q21.txt
create mode 100644
ballista/scheduler/tests/tpch_plan_stability/approved/q22.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q3.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q4.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q5.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q6.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q7.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q8.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/approved/q9.txt
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/fixtures.rs
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/main.rs
create mode 100644 ballista/scheduler/tests/tpch_plan_stability/stats_table.rs
create mode 100644 benchmarks/queries-tpcds/q1.sql
create mode 100644 benchmarks/queries-tpcds/q10.sql
create mode 100644 benchmarks/queries-tpcds/q11.sql
create mode 100644 benchmarks/queries-tpcds/q12.sql
create mode 100644 benchmarks/queries-tpcds/q13.sql
create mode 100644 benchmarks/queries-tpcds/q14.sql
create mode 100644 benchmarks/queries-tpcds/q15.sql
create mode 100644 benchmarks/queries-tpcds/q16.sql
create mode 100644 benchmarks/queries-tpcds/q17.sql
create mode 100644 benchmarks/queries-tpcds/q18.sql
create mode 100644 benchmarks/queries-tpcds/q19.sql
create mode 100644 benchmarks/queries-tpcds/q2.sql
create mode 100644 benchmarks/queries-tpcds/q20.sql
create mode 100644 benchmarks/queries-tpcds/q21.sql
create mode 100644 benchmarks/queries-tpcds/q22.sql
create mode 100644 benchmarks/queries-tpcds/q23.sql
create mode 100644 benchmarks/queries-tpcds/q24.sql
create mode 100644 benchmarks/queries-tpcds/q25.sql
create mode 100644 benchmarks/queries-tpcds/q26.sql
create mode 100644 benchmarks/queries-tpcds/q27.sql
create mode 100644 benchmarks/queries-tpcds/q28.sql
create mode 100644 benchmarks/queries-tpcds/q29.sql
create mode 100644 benchmarks/queries-tpcds/q3.sql
create mode 100644 benchmarks/queries-tpcds/q30.sql
create mode 100644 benchmarks/queries-tpcds/q31.sql
create mode 100644 benchmarks/queries-tpcds/q32.sql
create mode 100644 benchmarks/queries-tpcds/q33.sql
create mode 100644 benchmarks/queries-tpcds/q34.sql
create mode 100644 benchmarks/queries-tpcds/q35.sql
create mode 100644 benchmarks/queries-tpcds/q36.sql
create mode 100644 benchmarks/queries-tpcds/q37.sql
create mode 100644 benchmarks/queries-tpcds/q38.sql
create mode 100644 benchmarks/queries-tpcds/q39.sql
create mode 100644 benchmarks/queries-tpcds/q4.sql
create mode 100644 benchmarks/queries-tpcds/q40.sql
create mode 100644 benchmarks/queries-tpcds/q41.sql
create mode 100644 benchmarks/queries-tpcds/q42.sql
create mode 100644 benchmarks/queries-tpcds/q43.sql
create mode 100644 benchmarks/queries-tpcds/q44.sql
create mode 100644 benchmarks/queries-tpcds/q45.sql
create mode 100644 benchmarks/queries-tpcds/q46.sql
create mode 100644 benchmarks/queries-tpcds/q47.sql
create mode 100644 benchmarks/queries-tpcds/q48.sql
create mode 100644 benchmarks/queries-tpcds/q49.sql
create mode 100644 benchmarks/queries-tpcds/q5.sql
create mode 100644 benchmarks/queries-tpcds/q50.sql
create mode 100644 benchmarks/queries-tpcds/q51.sql
create mode 100644 benchmarks/queries-tpcds/q52.sql
create mode 100644 benchmarks/queries-tpcds/q53.sql
create mode 100644 benchmarks/queries-tpcds/q54.sql
create mode 100644 benchmarks/queries-tpcds/q55.sql
create mode 100644 benchmarks/queries-tpcds/q56.sql
create mode 100644 benchmarks/queries-tpcds/q57.sql
create mode 100644 benchmarks/queries-tpcds/q58.sql
create mode 100644 benchmarks/queries-tpcds/q59.sql
create mode 100644 benchmarks/queries-tpcds/q6.sql
create mode 100644 benchmarks/queries-tpcds/q60.sql
create mode 100644 benchmarks/queries-tpcds/q61.sql
create mode 100644 benchmarks/queries-tpcds/q62.sql
create mode 100644 benchmarks/queries-tpcds/q63.sql
create mode 100644 benchmarks/queries-tpcds/q64.sql
create mode 100644 benchmarks/queries-tpcds/q65.sql
create mode 100644 benchmarks/queries-tpcds/q66.sql
create mode 100644 benchmarks/queries-tpcds/q67.sql
create mode 100644 benchmarks/queries-tpcds/q68.sql
create mode 100644 benchmarks/queries-tpcds/q69.sql
create mode 100644 benchmarks/queries-tpcds/q7.sql
create mode 100644 benchmarks/queries-tpcds/q70.sql
create mode 100644 benchmarks/queries-tpcds/q71.sql
create mode 100644 benchmarks/queries-tpcds/q72.sql
create mode 100644 benchmarks/queries-tpcds/q73.sql
create mode 100644 benchmarks/queries-tpcds/q74.sql
create mode 100644 benchmarks/queries-tpcds/q75.sql
create mode 100644 benchmarks/queries-tpcds/q76.sql
create mode 100644 benchmarks/queries-tpcds/q77.sql
create mode 100644 benchmarks/queries-tpcds/q78.sql
create mode 100644 benchmarks/queries-tpcds/q79.sql
create mode 100644 benchmarks/queries-tpcds/q8.sql
create mode 100644 benchmarks/queries-tpcds/q80.sql
create mode 100644 benchmarks/queries-tpcds/q81.sql
create mode 100644 benchmarks/queries-tpcds/q82.sql
create mode 100644 benchmarks/queries-tpcds/q83.sql
create mode 100644 benchmarks/queries-tpcds/q84.sql
create mode 100644 benchmarks/queries-tpcds/q85.sql
create mode 100644 benchmarks/queries-tpcds/q86.sql
create mode 100644 benchmarks/queries-tpcds/q87.sql
create mode 100644 benchmarks/queries-tpcds/q88.sql
create mode 100644 benchmarks/queries-tpcds/q89.sql
create mode 100644 benchmarks/queries-tpcds/q9.sql
create mode 100644 benchmarks/queries-tpcds/q90.sql
create mode 100644 benchmarks/queries-tpcds/q91.sql
create mode 100644 benchmarks/queries-tpcds/q92.sql
create mode 100644 benchmarks/queries-tpcds/q93.sql
create mode 100644 benchmarks/queries-tpcds/q94.sql
create mode 100644 benchmarks/queries-tpcds/q95.sql
create mode 100644 benchmarks/queries-tpcds/q96.sql
create mode 100644 benchmarks/queries-tpcds/q97.sql
create mode 100644 benchmarks/queries-tpcds/q98.sql
create mode 100644 benchmarks/queries-tpcds/q99.sql
create mode 100644 benchmarks/src/bin/tpcds.rs
create mode 100644 benchmarks/src/lib.rs
copy benchmarks/{tpch-gen.sh => tpcds-gen.sh} (53%)
copy ci/scripts/{rust_fmt.sh => rust_config_docs_check.sh} (81%)
copy ci/scripts/{rust_fmt.sh => rust_proto_check.sh} (74%)
copy ci/scripts/rust_fmt.sh => dev/update-tpch-plan-stability.sh (67%)
copy ci/scripts/rust_fmt.sh => dev/update_config_docs.sh (77%)
create mode 100755 dev/update_datafusion_proto.py
copy dev/{docker/standalone-entrypoint.sh => vendor-tpcds-queries.sh} (53%)
delete mode 100644 docs/source/_static/images/tpch_allqueries.png
delete mode 100644 docs/source/_static/images/tpch_queries_compare.png
delete mode 100644 docs/source/_static/images/tpch_queries_speedup_abs.png
delete mode 100644 docs/source/_static/images/tpch_queries_speedup_rel.png
create mode 100644 docs/source/contributors-guide/benchmarking.md
create mode 100644 docs/source/contributors-guide/user-personas.md
create mode 100644 docs/source/upgrading/54.0.0.md
create mode 100644 docs/source/upgrading/55.0.0.md
copy docs/source/{user-guide/python => upgrading}/index.rst (85%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
