This is an automated email from the ASF dual-hosted git repository.
thinkharderdev pushed a change to branch cluster-state-refactor-2
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
from 67331435 do not hold ref across await point
add 93cf14f7 Cluster state refactor part 1 (#560)
add 75b4f845 replace master with main (#621)
add a5f306e5 implement new release process (#623)
add 70032e1f add docs on who can release (#632)
add 0b6b28f8 Upgrade to DataFusion 16 (again) (#636)
add b0d8cc7e Update datafusion dependency to the latest version (#612)
add 1b0be751 Upgrade to DataFusion 17 (#639)
add 1a6bc262 check in benchmark image (#647)
add 918c3448 Remove `python` dir & python-related workflows (#654)
add 0b8d99c3 Handle job resubmission (#586)
add 8f8154f4 Add executor self-registration mechanism in the heartbeat
service (#649)
add c66433ce Merge remote-tracking branch 'origin/main' into
cluster-state-refactor-2
No new revisions were added by this update.
Summary of changes:
.github/dependabot.yml | 2 +-
.github/workflows/comment_bot.yml | 6 +-
.github/workflows/dev.yml | 4 +-
.github/workflows/python_build.yml | 128 --
.github/workflows/python_test.yaml | 73 -
Cargo.toml | 4 +-
README.md | 2 +-
ballista-cli/Cargo.toml | 4 +-
ballista-cli/src/command.rs | 6 +-
ballista-cli/src/exec.rs | 12 +-
ballista-cli/src/main.rs | 10 +-
ballista/client/Cargo.toml | 6 +-
ballista/client/src/context.rs | 60 +-
ballista/core/Cargo.toml | 8 +-
ballista/core/build.rs | 4 +-
ballista/core/proto/ballista.proto | 409 +----
ballista/core/proto/datafusion.proto | 849 +++++++---
ballista/core/src/client.rs | 15 +-
ballista/core/src/config.rs | 13 +-
ballista/core/src/error.rs | 43 +-
ballista/core/src/event_loop.rs | 7 +-
.../core/src/execution_plans/distributed_query.rs | 30 +-
.../core/src/execution_plans/shuffle_reader.rs | 20 +-
.../core/src/execution_plans/shuffle_writer.rs | 17 +-
.../core/src/execution_plans/unresolved_shuffle.rs | 4 -
ballista/core/src/lib.rs | 2 +-
ballista/core/src/plugin/plugin_manager.rs | 2 +-
ballista/core/src/plugin/udf.rs | 6 +-
ballista/core/src/serde/generated/ballista.rs | 772 ++-------
ballista/core/src/serde/mod.rs | 746 +++------
.../core/src/serde/physical_plan/from_proto.rs | 418 -----
ballista/core/src/serde/physical_plan/mod.rs | 1707 --------------------
ballista/core/src/serde/physical_plan/to_proto.rs | 468 ------
ballista/core/src/serde/scheduler/from_proto.rs | 24 +-
ballista/core/src/serde/scheduler/mod.rs | 4 +-
ballista/core/src/serde/scheduler/to_proto.rs | 19 +-
ballista/core/src/utils.rs | 15 +-
ballista/executor/Cargo.toml | 8 +-
ballista/executor/build.rs | 2 +-
ballista/executor/src/collect.rs | 2 +-
ballista/executor/src/execution_loop.rs | 8 +-
ballista/executor/src/executor_process.rs | 13 +-
ballista/executor/src/executor_server.rs | 37 +-
ballista/executor/src/flight_service.rs | 41 +-
ballista/executor/src/standalone.rs | 3 +-
ballista/scheduler/Cargo.toml | 6 +-
ballista/scheduler/build.rs | 4 +-
ballista/scheduler/scheduler_config_spec.toml | 13 +-
ballista/scheduler/src/api/handlers.rs | 2 +-
ballista/scheduler/src/api/mod.rs | 2 +-
ballista/scheduler/src/bin/main.rs | 2 +
ballista/scheduler/src/cluster/kv.rs | 25 +-
ballista/scheduler/src/cluster/memory.rs | 19 +-
ballista/scheduler/src/cluster/mod.rs | 3 +-
ballista/scheduler/src/cluster/storage/etcd.rs | 14 +-
ballista/scheduler/src/cluster/storage/mod.rs | 13 +-
ballista/scheduler/src/cluster/storage/sled.rs | 44 +-
ballista/scheduler/src/cluster/test/mod.rs | 6 +-
ballista/scheduler/src/config.rs | 9 +
ballista/scheduler/src/display.rs | 2 +-
ballista/scheduler/src/flight_sql.rs | 106 +-
ballista/scheduler/src/metrics/prometheus.rs | 16 +-
ballista/scheduler/src/planner.rs | 43 +-
ballista/scheduler/src/scheduler_process.rs | 6 +-
ballista/scheduler/src/scheduler_server/event.rs | 1 +
.../src/scheduler_server/external_scaler.rs | 2 +-
ballista/scheduler/src/scheduler_server/grpc.rs | 188 ++-
ballista/scheduler/src/scheduler_server/mod.rs | 77 +-
.../src/scheduler_server/query_stage_scheduler.rs | 118 +-
ballista/scheduler/src/standalone.rs | 2 +-
ballista/scheduler/src/state/execution_graph.rs | 39 +-
.../src/state/execution_graph/execution_stage.rs | 10 +-
.../scheduler/src/state/execution_graph_dot.rs | 74 +-
ballista/scheduler/src/state/executor_manager.rs | 26 +-
ballista/scheduler/src/state/mod.rs | 21 +-
ballista/scheduler/src/state/session_manager.rs | 21 -
ballista/scheduler/src/state/task_manager.rs | 8 +-
ballista/scheduler/src/test_utils.rs | 131 +-
benchmarks/Cargo.toml | 4 +-
benchmarks/src/bin/nyctaxi.rs | 15 +-
benchmarks/src/bin/tpch.rs | 168 +-
dev/release/README.md | 174 +-
dev/release/create-tarball.sh | 2 +-
dev/release/update_change_log-ballista.sh | 2 +-
dev/release/update_change_log.sh | 2 +-
docs/source/conf.py | 4 +-
docs/source/index.rst | 2 +-
.../source/user-guide/deployment/docker-compose.md | 2 +-
...bench-h-workstation-10-distributed-perquery.png | Bin 0 -> 33223 bytes
examples/Cargo.toml | 2 +-
python/.cargo/config | 22 -
python/.dockerignore | 19 -
python/.gitignore | 20 -
python/CHANGELOG.md | 129 --
python/Cargo.toml | 54 -
python/LICENSE.txt | 202 ---
python/README.md | 185 ---
python/ballista/__init__.py | 113 --
python/ballista/functions.py | 23 -
python/ballista/tests/__init__.py | 16 -
python/ballista/tests/generic.py | 87 -
python/ballista/tests/test_aggregation.py | 48 -
python/ballista/tests/test_catalog.py | 72 -
python/ballista/tests/test_context.py | 63 -
python/ballista/tests/test_dataframe.py | 195 ---
python/ballista/tests/test_functions.py | 219 ---
python/ballista/tests/test_imports.py | 65 -
python/ballista/tests/test_sql.py | 250 ---
python/ballista/tests/test_udaf.py | 136 --
python/pyproject.toml | 55 -
python/requirements-310.txt | 210 ---
python/requirements-37.txt | 318 ----
python/requirements.in | 27 -
python/requirements.txt | 282 ----
python/src/ballista_context.rs | 137 --
python/src/catalog.rs | 127 --
python/src/context.rs | 267 ---
python/src/dataframe.rs | 185 ---
python/src/dataset.rs | 134 --
python/src/dataset_exec.rs | 282 ----
python/src/datatype.rs | 39 -
python/src/errors.rs | 111 --
python/src/expression.rs | 138 --
python/src/functions.rs | 346 ----
python/src/lib.rs | 73 -
python/src/pyarrow_filter_expression.rs | 219 ---
python/src/udaf.rs | 150 --
python/src/udf.rs | 95 --
python/src/utils.rs | 47 -
129 files changed, 2057 insertions(+), 10566 deletions(-)
delete mode 100644 .github/workflows/python_build.yml
delete mode 100644 .github/workflows/python_test.yaml
delete mode 100644 ballista/core/src/serde/physical_plan/from_proto.rs
delete mode 100644 ballista/core/src/serde/physical_plan/mod.rs
delete mode 100644 ballista/core/src/serde/physical_plan/to_proto.rs
create mode 100644 docs/sqlbench-h-workstation-10-distributed-perquery.png
delete mode 100644 python/.cargo/config
delete mode 100644 python/.dockerignore
delete mode 100644 python/.gitignore
delete mode 100644 python/CHANGELOG.md
delete mode 100644 python/Cargo.toml
delete mode 100644 python/LICENSE.txt
delete mode 100644 python/README.md
delete mode 100644 python/ballista/__init__.py
delete mode 100644 python/ballista/functions.py
delete mode 100644 python/ballista/tests/__init__.py
delete mode 100644 python/ballista/tests/generic.py
delete mode 100644 python/ballista/tests/test_aggregation.py
delete mode 100644 python/ballista/tests/test_catalog.py
delete mode 100644 python/ballista/tests/test_context.py
delete mode 100644 python/ballista/tests/test_dataframe.py
delete mode 100644 python/ballista/tests/test_functions.py
delete mode 100644 python/ballista/tests/test_imports.py
delete mode 100644 python/ballista/tests/test_sql.py
delete mode 100644 python/ballista/tests/test_udaf.py
delete mode 100644 python/pyproject.toml
delete mode 100644 python/requirements-310.txt
delete mode 100644 python/requirements-37.txt
delete mode 100644 python/requirements.in
delete mode 100644 python/requirements.txt
delete mode 100644 python/src/ballista_context.rs
delete mode 100644 python/src/catalog.rs
delete mode 100644 python/src/context.rs
delete mode 100644 python/src/dataframe.rs
delete mode 100644 python/src/dataset.rs
delete mode 100644 python/src/dataset_exec.rs
delete mode 100644 python/src/datatype.rs
delete mode 100644 python/src/errors.rs
delete mode 100644 python/src/expression.rs
delete mode 100644 python/src/functions.rs
delete mode 100644 python/src/lib.rs
delete mode 100644 python/src/pyarrow_filter_expression.rs
delete mode 100644 python/src/udaf.rs
delete mode 100644 python/src/udf.rs
delete mode 100644 python/src/utils.rs
