This is an automated email from the ASF dual-hosted git repository.
thinkharderdev pushed a change to branch issue-585
in repository https://gitbox.apache.org/repos/asf/arrow-ballista.git
from 94a1716a Handle job resubmission
add 03e351f8 Fix benchmark image link (#596)
add 7f285339 Add support for Azure (#599)
add 45034da7 Remove outdated script and use evergreen version of rust
(#597)
add f59179a0 feat: update script such that ballista-cli image is built as
well (#601)
add 18e2cc88 Fix Cargo.toml format issue (#616)
add cbad55d7 Refactor executor main (#614)
add b1713967 Refactor scheduler main (#615)
add dfc168c6 Python: add method to get explain output as a string (#593)
add a621a3f8 Update contributor guide (#617)
add 93cf14f7 Cluster state refactor part 1 (#560)
add 75b4f845 replace master with main (#621)
add a5f306e5 implement new release process (#623)
add 70032e1f add docs on who can release (#632)
add 0b6b28f8 Upgrade to DataFusion 16 (again) (#636)
add b0d8cc7e Update datafusion dependency to the latest version (#612)
add 1b0be751 Upgrade to DataFusion 17 (#639)
add 4c8dd28c Merge branch 'main' into issue-585
add 61a8a2af Make resubmission configurable and add test
No new revisions were added by this update.
Summary of changes:
.dockerignore | 1 +
.github/dependabot.yml | 2 +-
.github/workflows/comment_bot.yml | 6 +-
.github/workflows/rust.yml | 7 +-
CONTRIBUTING.md | 212 +--
Cargo.toml | 4 +-
README.md | 2 +-
ballista-cli/Cargo.toml | 4 +-
ballista-cli/src/command.rs | 6 +-
ballista-cli/src/exec.rs | 12 +-
ballista-cli/src/main.rs | 10 +-
ballista/client/Cargo.toml | 7 +-
ballista/client/src/context.rs | 60 +-
ballista/core/Cargo.toml | 9 +-
ballista/core/build.rs | 4 +-
ballista/core/proto/ballista.proto | 408 +----
ballista/core/proto/datafusion.proto | 849 +++++++---
ballista/core/src/client.rs | 15 +-
ballista/core/src/config.rs | 13 +-
ballista/core/src/error.rs | 43 +-
ballista/core/src/event_loop.rs | 7 +-
.../core/src/execution_plans/distributed_query.rs | 30 +-
.../core/src/execution_plans/shuffle_reader.rs | 20 +-
.../core/src/execution_plans/shuffle_writer.rs | 17 +-
.../core/src/execution_plans/unresolved_shuffle.rs | 4 -
ballista/core/src/lib.rs | 2 +-
ballista/core/src/plugin/plugin_manager.rs | 2 +-
ballista/core/src/plugin/udf.rs | 6 +-
ballista/core/src/serde/generated/ballista.rs | 768 ++-------
ballista/core/src/serde/mod.rs | 742 +++------
.../core/src/serde/physical_plan/from_proto.rs | 418 -----
ballista/core/src/serde/physical_plan/mod.rs | 1707 --------------------
ballista/core/src/serde/physical_plan/to_proto.rs | 468 ------
ballista/core/src/serde/scheduler/from_proto.rs | 24 +-
ballista/core/src/serde/scheduler/mod.rs | 4 +-
ballista/core/src/serde/scheduler/to_proto.rs | 19 +-
ballista/core/src/utils.rs | 29 +-
ballista/executor/Cargo.toml | 13 +-
ballista/executor/build.rs | 2 +-
ballista/executor/src/bin/main.rs | 83 +
ballista/executor/src/collect.rs | 2 +-
ballista/executor/src/execution_loop.rs | 8 +-
.../executor/src/{main.rs => executor_process.rs} | 139 +-
ballista/executor/src/executor_server.rs | 36 +-
ballista/executor/src/flight_service.rs | 41 +-
ballista/executor/src/lib.rs | 1 +
ballista/executor/src/standalone.rs | 3 +-
ballista/scheduler/Cargo.toml | 10 +-
ballista/scheduler/build.rs | 4 +-
ballista/scheduler/scheduler_config_spec.toml | 13 +
ballista/scheduler/src/api/handlers.rs | 2 +-
ballista/scheduler/src/api/mod.rs | 2 +-
ballista/scheduler/src/{ => bin}/main.rs | 170 +-
ballista/scheduler/src/config.rs | 14 +
ballista/scheduler/src/display.rs | 2 +-
ballista/scheduler/src/flight_sql.rs | 106 +-
ballista/scheduler/src/lib.rs | 1 +
ballista/scheduler/src/metrics/prometheus.rs | 16 +-
ballista/scheduler/src/planner.rs | 43 +-
ballista/scheduler/src/scheduler_process.rs | 123 ++
.../src/scheduler_server/external_scaler.rs | 2 +-
ballista/scheduler/src/scheduler_server/grpc.rs | 86 +-
ballista/scheduler/src/scheduler_server/mod.rs | 100 +-
.../src/scheduler_server/query_stage_scheduler.rs | 62 +-
ballista/scheduler/src/standalone.rs | 8 +-
ballista/scheduler/src/state/backend/cluster.rs | 693 ++++++++
ballista/scheduler/src/state/backend/etcd.rs | 14 +-
ballista/scheduler/src/state/backend/memory.rs | 14 +-
ballista/scheduler/src/state/backend/mod.rs | 14 +-
ballista/scheduler/src/state/backend/sled.rs | 38 +-
ballista/scheduler/src/state/execution_graph.rs | 115 +-
.../src/state/execution_graph/execution_stage.rs | 10 +-
.../scheduler/src/state/execution_graph_dot.rs | 74 +-
ballista/scheduler/src/state/executor_manager.rs | 487 ++----
ballista/scheduler/src/state/mod.rs | 37 +-
ballista/scheduler/src/state/session_manager.rs | 21 -
ballista/scheduler/src/state/task_manager.rs | 8 +-
ballista/scheduler/src/test_utils.rs | 65 +-
benchmarks/Cargo.toml | 4 +-
benchmarks/src/bin/nyctaxi.rs | 15 +-
benchmarks/src/bin/tpch.rs | 168 +-
dev/build-ballista-docker.sh | 7 +-
...lista-rust.sh => build-ballista-executables.sh} | 4 +
dev/build-ui.sh | 23 -
dev/docker/ballista-builder.Dockerfile | 2 +-
...executor.Dockerfile => ballista-cli.Dockerfile} | 11 +-
.../{executor-entrypoint.sh => cli-entrypoint.sh} | 2 +-
dev/release/README.md | 174 +-
dev/release/create-tarball.sh | 2 +-
dev/release/update_change_log-ballista.sh | 2 +-
dev/release/update_change_log.sh | 2 +-
docs/developer/README.md | 11 -
docs/developer/configuration.md | 34 -
docs/developer/dev-env.md | 51 -
docs/developer/integration-testing.md | 29 -
docs/source/conf.py | 4 +-
docs/source/index.rst | 2 +-
.../source/user-guide/deployment/docker-compose.md | 2 +-
examples/Cargo.toml | 2 +-
python/Cargo.toml | 4 +-
python/ballista/tests/test_dataframe.py | 17 +
python/src/catalog.rs | 15 +-
python/src/context.rs | 24 +-
python/src/dataframe.rs | 51 +-
python/src/dataset_exec.rs | 4 -
python/src/functions.rs | 150 +-
python/src/udaf.rs | 12 +-
107 files changed, 3313 insertions(+), 6138 deletions(-)
delete mode 100644 ballista/core/src/serde/physical_plan/from_proto.rs
delete mode 100644 ballista/core/src/serde/physical_plan/mod.rs
delete mode 100644 ballista/core/src/serde/physical_plan/to_proto.rs
create mode 100644 ballista/executor/src/bin/main.rs
rename ballista/executor/src/{main.rs => executor_process.rs} (85%)
rename ballista/scheduler/src/{ => bin}/main.rs (55%)
create mode 100644 ballista/scheduler/src/scheduler_process.rs
create mode 100644 ballista/scheduler/src/state/backend/cluster.rs
rename dev/{build-ballista-rust.sh => build-ballista-executables.sh} (79%)
delete mode 100755 dev/build-ui.sh
copy dev/docker/{ballista-executor.Dockerfile => ballista-cli.Dockerfile} (76%)
copy dev/docker/{executor-entrypoint.sh => cli-entrypoint.sh} (96%)
delete mode 100644 docs/developer/configuration.md
delete mode 100644 docs/developer/dev-env.md
delete mode 100644 docs/developer/integration-testing.md
