This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 97c91927 chore(deps): update to datafusion 47.0.0 (#1250)
97c91927 is described below
commit 97c919274d9de496b630e66c12ad29c3fccd110b
Author: Marko Milenković <milenkov...@users.noreply.github.com>
AuthorDate: Sun Jun 1 10:37:34 2025 +0100
chore(deps): update to datafusion 47.0.0 (#1250)
* chore(deps): update to datafusion 47
* remove `.cargo/config.toml` as not needed anymore
* update df proto definitions
* update deps
---
.cargo/config.toml | 15 -
Cargo.lock | 454 ++++++++++++++-------
Cargo.toml | 14 +-
ballista-cli/Cargo.toml | 4 +-
ballista/client/Cargo.toml | 14 +-
ballista/client/tests/context_checks.rs | 3 -
ballista/core/Cargo.toml | 2 +-
ballista/core/proto/datafusion.proto | 8 +-
ballista/core/proto/datafusion_common.proto | 4 +
.../core/src/execution_plans/distributed_query.rs | 3 +
.../core/src/execution_plans/shuffle_reader.rs | 3 +
.../core/src/execution_plans/shuffle_writer.rs | 5 +-
.../core/src/execution_plans/unresolved_shuffle.rs | 9 +-
ballista/executor/Cargo.toml | 4 +-
ballista/executor/src/collect.rs | 1 +
ballista/executor/src/executor.rs | 4 +-
ballista/executor/src/flight_service.rs | 8 +-
ballista/scheduler/Cargo.toml | 6 +-
ballista/scheduler/src/cluster/mod.rs | 8 +-
.../scheduler/src/state/execution_graph_dot.rs | 2 +-
benchmarks/Cargo.toml | 6 +-
examples/Cargo.toml | 10 +-
22 files changed, 378 insertions(+), 209 deletions(-)
diff --git a/.cargo/config.toml b/.cargo/config.toml
deleted file mode 100644
index 729582f9..00000000
--- a/.cargo/config.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[env]
-# there were issues with stack memory, so memory was increased
-# more details at:
-#
-# - https://github.com/apache/datafusion-ballista/pull/1201
-# - https://github.com/apache/datafusion/issues/15087
-# - https://github.com/apache/datafusion/pull/15600
-#
-# no specific reason why this stack size has been selected,
-# apart from being maximum stack size
-#
-# TODO we should remove file when updating to datafusion 47
-# if https://github.com/apache/datafusion/pull/15600 is merged
-# until then
-RUST_MIN_STACK="20971520"
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index b0a77d4d..c0788937 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -181,9 +181,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796"
+checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -202,9 +202,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e"
+checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -216,9 +216,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985"
+checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472"
dependencies = [
"ahash",
"arrow-buffer",
@@ -233,9 +233,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c"
+checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824"
dependencies = [
"bytes",
"half",
@@ -244,9 +244,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9"
+checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -265,9 +265,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1644877d8bc9a0ef022d9153dc29375c2bda244c39aec05a91d0e87ccf77995f"
+checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -281,9 +281,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429"
+checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -293,9 +293,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a194f47959a4e111463cb6d02c8576fe084b3d7a3c092314baf3b9629b62595b"
+checksum = "e2e0fad280f41a918d53ba48288a246ff04202d463b3b380fbc0edecdcb52cfd"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -320,9 +320,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c"
+checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -335,9 +335,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ee5b4ca98a7fb2efb9ab3309a5d1c88b5116997ff93f3147efdc1062a6158e9"
+checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -357,9 +357,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21"
+checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -370,9 +370,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68"
+checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -383,15 +383,15 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9"
+checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49"
[[package]]
name = "arrow-select"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619"
+checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8"
dependencies = [
"ahash",
"arrow-array",
@@ -403,9 +403,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03"
+checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -916,7 +916,7 @@ dependencies = [
[[package]]
name = "ballista"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"async-trait",
"ballista-core",
@@ -936,7 +936,7 @@ dependencies = [
[[package]]
name = "ballista-benchmarks"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"ballista",
"ballista-core",
@@ -955,7 +955,7 @@ dependencies = [
[[package]]
name = "ballista-cli"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"ballista",
"clap 4.5.36",
@@ -970,7 +970,7 @@ dependencies = [
[[package]]
name = "ballista-core"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"arrow-flight",
"async-trait",
@@ -1004,7 +1004,7 @@ dependencies = [
[[package]]
name = "ballista-examples"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"ballista",
"ballista-core",
@@ -1024,7 +1024,7 @@ dependencies = [
[[package]]
name = "ballista-executor"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"arrow",
"arrow-flight",
@@ -1052,7 +1052,7 @@ dependencies = [
[[package]]
name = "ballista-scheduler"
-version = "46.0.0"
+version = "47.0.0"
dependencies = [
"arrow-flight",
"async-trait",
@@ -1710,11 +1710,10 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3"
+checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080"
dependencies = [
- "apache-avro",
"arrow",
"arrow-ipc",
"arrow-schema",
@@ -1727,6 +1726,10 @@ dependencies = [
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
+ "datafusion-datasource-avro",
+ "datafusion-datasource-csv",
+ "datafusion-datasource-json",
+ "datafusion-datasource-parquet",
"datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
@@ -1741,12 +1744,12 @@ dependencies = [
"datafusion-physical-expr-common",
"datafusion-physical-optimizer",
"datafusion-physical-plan",
+ "datafusion-session",
"datafusion-sql",
"flate2",
"futures",
"itertools 0.14.0",
"log",
- "num-traits",
"object_store",
"parking_lot",
"parquet",
@@ -1763,29 +1766,35 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2"
+checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66"
dependencies = [
"arrow",
"async-trait",
"dashmap",
"datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
+ "datafusion-physical-expr",
"datafusion-physical-plan",
+ "datafusion-session",
"datafusion-sql",
"futures",
"itertools 0.14.0",
"log",
+ "object_store",
"parking_lot",
+ "tokio",
]
[[package]]
name = "datafusion-catalog-listing"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43"
+checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00"
dependencies = [
"arrow",
"async-trait",
@@ -1797,6 +1806,7 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-session",
"futures",
"log",
"object_store",
@@ -1805,9 +1815,9 @@ dependencies = [
[[package]]
name = "datafusion-cli"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "342388ddb5faec28cb96edfb1154d7abdc5746109fe5896d6895b0c9115aeb81"
+checksum = "ea1ac762ce2bea131b9ed60904283b93c3697399db273710405417f36b5e86d0"
dependencies = [
"arrow",
"async-trait",
@@ -1830,9 +1840,9 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a"
+checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c"
dependencies = [
"ahash",
"apache-avro",
@@ -1855,19 +1865,20 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e0fcf41523b22e14cc349b01526e8b9f59206653037f2949a4adbfde5f8cb668"
+checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b"
dependencies = [
+ "futures",
"log",
"tokio",
]
[[package]]
name = "datafusion-datasource"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92"
+checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1"
dependencies = [
"arrow",
"async-compression",
@@ -1875,7 +1886,6 @@ dependencies = [
"bytes",
"bzip2 0.5.2",
"chrono",
- "datafusion-catalog",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
@@ -1883,13 +1893,16 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
+ "datafusion-session",
"flate2",
"futures",
"glob",
"itertools 0.14.0",
"log",
"object_store",
+ "parquet",
"rand 0.8.5",
+ "tempfile",
"tokio",
"tokio-util",
"url",
@@ -1897,17 +1910,123 @@ dependencies = [
"zstd",
]
+[[package]]
+name = "datafusion-datasource-avro"
+version = "47.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d4ea5111aab9d3f2a8bff570343cccb03ce4c203875ef5a566b7d6f1eb72559e"
+dependencies = [
+ "apache-avro",
+ "arrow",
+ "async-trait",
+ "bytes",
+ "chrono",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "futures",
+ "num-traits",
+ "object_store",
+ "tokio",
+]
+
+[[package]]
+name = "datafusion-datasource-csv"
+version = "47.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "bytes",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "futures",
+ "object_store",
+ "regex",
+ "tokio",
+]
+
+[[package]]
+name = "datafusion-datasource-json"
+version = "47.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "bytes",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "futures",
+ "object_store",
+ "serde_json",
+ "tokio",
+]
+
+[[package]]
+name = "datafusion-datasource-parquet"
+version = "47.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "bytes",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-datasource",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions-aggregate",
+ "datafusion-physical-expr",
+ "datafusion-physical-expr-common",
+ "datafusion-physical-optimizer",
+ "datafusion-physical-plan",
+ "datafusion-session",
+ "futures",
+ "itertools 0.14.0",
+ "log",
+ "object_store",
+ "parking_lot",
+ "parquet",
+ "rand 0.8.5",
+ "tokio",
+]
+
[[package]]
name = "datafusion-doc"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256"
+checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292"
[[package]]
name = "datafusion-execution"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1"
+checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84"
dependencies = [
"arrow",
"dashmap",
@@ -1924,9 +2043,9 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f"
+checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964"
dependencies = [
"arrow",
"chrono",
@@ -1945,9 +2064,9 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e"
+checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839"
dependencies = [
"arrow",
"datafusion-common",
@@ -1958,9 +2077,9 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40"
+checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e"
dependencies = [
"arrow",
"arrow-buffer",
@@ -1987,9 +2106,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05"
+checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba"
dependencies = [
"ahash",
"arrow",
@@ -2008,9 +2127,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb"
+checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4"
dependencies = [
"ahash",
"arrow",
@@ -2021,9 +2140,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a"
+checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537"
dependencies = [
"arrow",
"arrow-ord",
@@ -2042,9 +2161,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-table"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5"
+checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a"
dependencies = [
"arrow",
"async-trait",
@@ -2058,9 +2177,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6"
+checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193"
dependencies = [
"datafusion-common",
"datafusion-doc",
@@ -2075,9 +2194,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82"
+checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2085,9 +2204,9 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7"
+checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1"
dependencies = [
"datafusion-expr",
"quote",
@@ -2096,9 +2215,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b"
+checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515"
dependencies = [
"arrow",
"chrono",
@@ -2115,9 +2234,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85"
+checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65"
dependencies = [
"ahash",
"arrow",
@@ -2137,9 +2256,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb"
+checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863"
dependencies = [
"ahash",
"arrow",
@@ -2151,9 +2270,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763"
+checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2"
dependencies = [
"arrow",
"datafusion-common",
@@ -2170,9 +2289,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f"
+checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55"
dependencies = [
"ahash",
"arrow",
@@ -2200,9 +2319,9 @@ dependencies = [
[[package]]
name = "datafusion-proto"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f"
+checksum = "a4a1afb2bdb05de7ff65be6883ebfd4ec027bd9f1f21c46aa3afd01927160a83"
dependencies = [
"arrow",
"chrono",
@@ -2216,20 +2335,44 @@ dependencies = [
[[package]]
name = "datafusion-proto-common"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f"
+checksum = "35b7a5876ebd6b564fb9a1fd2c3a2a9686b787071a256b47e4708f0916f9e46f"
dependencies = [
"arrow",
"datafusion-common",
"prost",
]
+[[package]]
+name = "datafusion-session"
+version = "47.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f"
+dependencies = [
+ "arrow",
+ "async-trait",
+ "dashmap",
+ "datafusion-common",
+ "datafusion-common-runtime",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-physical-expr",
+ "datafusion-physical-plan",
+ "datafusion-sql",
+ "futures",
+ "itertools 0.14.0",
+ "log",
+ "object_store",
+ "parking_lot",
+ "tokio",
+]
+
[[package]]
name = "datafusion-sql"
-version = "46.0.1"
+version = "47.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64"
+checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607"
dependencies = [
"arrow",
"bigdecimal",
@@ -2447,11 +2590,11 @@ checksum =
"1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "flatbuffers"
-version = "24.12.23"
+version = "25.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096"
+checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.9.0",
"rustc_version",
]
@@ -2462,6 +2605,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
dependencies = [
"crc32fast",
+ "libz-rs-sys",
"miniz_oxide",
]
@@ -3159,15 +3303,6 @@ dependencies = [
"either",
]
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
[[package]]
name = "itertools"
version = "0.14.0"
@@ -3370,6 +3505,15 @@ dependencies = [
"redox_syscall 0.5.11",
]
+[[package]]
+name = "libz-rs-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a"
+dependencies = [
+ "zlib-rs",
+]
+
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
@@ -3410,7 +3554,7 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5"
dependencies = [
- "twox-hash",
+ "twox-hash 1.6.3",
]
[[package]]
@@ -3644,18 +3788,21 @@ dependencies = [
[[package]]
name = "object_store"
-version = "0.11.2"
+version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf"
+checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"chrono",
+ "form_urlencoded",
"futures",
+ "http 1.3.1",
+ "http-body-util",
"humantime",
"hyper",
- "itertools 0.13.0",
+ "itertools 0.14.0",
"md-5",
"parking_lot",
"percent-encoding",
@@ -3666,7 +3813,8 @@ dependencies = [
"rustls-pemfile",
"serde",
"serde_json",
- "snafu",
+ "serde_urlencoded",
+ "thiserror 2.0.12",
"tokio",
"tracing",
"url",
@@ -3737,9 +3885,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "54.3.1"
+version = "55.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bfb15796ac6f56b429fd99e33ba133783ad75b27c36b4b5ce06f1f82cc97754e"
+checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c"
dependencies = [
"ahash",
"arrow-array",
@@ -3767,7 +3915,7 @@ dependencies = [
"snap",
"thrift",
"tokio",
- "twox-hash",
+ "twox-hash 2.1.0",
"zstd",
]
@@ -4038,22 +4186,21 @@ dependencies = [
[[package]]
name = "procfs"
-version = "0.16.0"
+version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "731e0d9356b0c25f16f33b5be79b1c57b562f141ebfcdb0ad8ac2c13a24293b4"
+checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
dependencies = [
"bitflags 2.9.0",
"hex",
- "lazy_static",
"procfs-core",
"rustix 0.38.44",
]
[[package]]
name = "procfs-core"
-version = "0.16.0"
+version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
+checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
dependencies = [
"bitflags 2.9.0",
"hex",
@@ -4061,9 +4208,9 @@ dependencies = [
[[package]]
name = "prometheus"
-version = "0.13.4"
+version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
+checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if",
"fnv",
@@ -4073,7 +4220,7 @@ dependencies = [
"parking_lot",
"procfs",
"protobuf",
- "thiserror 1.0.69",
+ "thiserror 2.0.12",
]
[[package]]
@@ -4130,9 +4277,23 @@ dependencies = [
[[package]]
name = "protobuf"
-version = "2.28.0"
+version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
+checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
+dependencies = [
+ "once_cell",
+ "protobuf-support",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "protobuf-support"
+version = "3.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
+dependencies = [
+ "thiserror 1.0.69",
+]
[[package]]
name = "psm"
@@ -4477,9 +4638,9 @@ checksum =
"e33e4fb37ba46888052c763e4ec2acfedd8f00f62897b630cadb6298b833675e"
[[package]]
name = "rstest"
-version = "0.24.0"
+version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "03e905296805ab93e13c1ec3a03f4b6c4f35e9498a3d5fa96dc626d22c03cd89"
+checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d"
dependencies = [
"futures-timer",
"futures-util",
@@ -4489,9 +4650,9 @@ dependencies = [
[[package]]
name = "rstest_macros"
-version = "0.24.0"
+version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ef0053bbffce09062bee4bcc499b0fbe7a57b879f1efe088d6d8d4c7adcdef9b"
+checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746"
dependencies = [
"cfg-if",
"glob",
@@ -4883,27 +5044,6 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
-[[package]]
-name = "snafu"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019"
-dependencies = [
- "snafu-derive",
-]
-
-[[package]]
-name = "snafu-derive"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
[[package]]
name = "snap"
version = "1.1.1"
@@ -4940,9 +5080,9 @@ dependencies = [
[[package]]
name = "sqlparser"
-version = "0.54.0"
+version = "0.55.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899"
+checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11"
dependencies = [
"log",
"recursive",
@@ -5594,6 +5734,12 @@ dependencies = [
"static_assertions",
]
+[[package]]
+name = "twox-hash"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908"
+
[[package]]
name = "typed-builder"
version = "0.19.1"
@@ -6368,6 +6514,12 @@ dependencies = [
"syn 2.0.100",
]
+[[package]]
+name = "zlib-rs"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8"
+
[[package]]
name = "zstd"
version = "0.13.3"
diff --git a/Cargo.toml b/Cargo.toml
index 23566b80..45cb6f89 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -21,16 +21,16 @@ members = ["ballista-cli", "ballista/client",
"ballista/core", "ballista/executo
resolver = "2"
[workspace.dependencies]
-arrow = { version = "54", features = ["ipc_compression"] }
-arrow-flight = { version = "54", features = ["flight-sql-experimental"] }
+arrow = { version = "55", features = ["ipc_compression"] }
+arrow-flight = { version = "55", features = ["flight-sql-experimental"] }
clap = { version = "4.5", features = ["derive", "cargo"] }
configure_me = { version = "0.4.0" }
configure_me_codegen = { version = "0.4.4" }
-datafusion = "46.0.1"
-datafusion-cli = "46.0.1"
-datafusion-proto = "46.0.1"
-datafusion-proto-common = "46.0.1"
-object_store = "0.11"
+datafusion = "47.0.0"
+datafusion-cli = "47.0.0"
+datafusion-proto = "47.0.0"
+datafusion-proto-common = "47.0.0"
+object_store = "0.12"
prost = "0.13"
prost-types = "0.13"
tonic = { version = "0.12" }
diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml
index f937692b..8e631d6b 100644
--- a/ballista-cli/Cargo.toml
+++ b/ballista-cli/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-cli"
description = "Command Line Client for Ballista distributed query engine."
-version = "46.0.0"
+version = "47.0.0"
authors = ["Apache DataFusion <d...@datafusion.apache.org>"]
edition = "2021"
keywords = ["ballista", "cli"]
@@ -28,7 +28,7 @@ repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
[dependencies]
-ballista = { path = "../ballista/client", version = "46.0.0", features =
["standalone"] }
+ballista = { path = "../ballista/client", version = "47.0.0", features =
["standalone"] }
clap = { workspace = true, features = ["derive", "cargo"] }
datafusion = { workspace = true }
datafusion-cli = { workspace = true }
diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml
index ff023cb3..947f1ee3 100644
--- a/ballista/client/Cargo.toml
+++ b/ballista/client/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "46.0.0"
+version = "47.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -28,9 +28,9 @@ edition = "2021"
[dependencies]
async-trait = { workspace = true }
-ballista-core = { path = "../core", version = "46.0.0" }
-ballista-executor = { path = "../executor", version = "46.0.0", optional =
true }
-ballista-scheduler = { path = "../scheduler", version = "46.0.0", optional =
true }
+ballista-core = { path = "../core", version = "47.0.0" }
+ballista-executor = { path = "../executor", version = "47.0.0", optional =
true }
+ballista-scheduler = { path = "../scheduler", version = "47.0.0", optional =
true }
datafusion = { workspace = true }
log = { workspace = true }
@@ -38,12 +38,12 @@ tokio = { workspace = true }
url = { workspace = true }
[dev-dependencies]
-ballista-executor = { path = "../executor", version = "46.0.0" }
-ballista-scheduler = { path = "../scheduler", version = "46.0.0" }
+ballista-executor = { path = "../executor", version = "47.0.0" }
+ballista-scheduler = { path = "../scheduler", version = "47.0.0" }
ctor = { workspace = true }
datafusion-proto = { workspace = true }
env_logger = { workspace = true }
-rstest = { version = "0.24" }
+rstest = { version = "0.25" }
tempfile = { workspace = true }
tonic = { workspace = true }
diff --git a/ballista/client/tests/context_checks.rs
b/ballista/client/tests/context_checks.rs
index 026b83c2..edb1e73b 100644
--- a/ballista/client/tests/context_checks.rs
+++ b/ballista/client/tests/context_checks.rs
@@ -518,8 +518,6 @@ mod supported {
Ok(())
}
- // tests if `ctx.enable_url_table()` works correctly
- // it did not work before datafusion 46.0.0
#[rstest]
#[case::standalone(standalone_context())]
#[case::remote(remote_context())]
@@ -555,7 +553,6 @@ mod supported {
Ok(())
}
- // export RUST_MIN_STACK=20971520
#[rstest]
#[case::standalone(standalone_context())]
#[case::remote(remote_context())]
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index b05edf3e..d0c6bec6 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-core"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "46.0.0"
+version = "47.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
diff --git a/ballista/core/proto/datafusion.proto
b/ballista/core/proto/datafusion.proto
index dc1aae10..2f62be28 100644
--- a/ballista/core/proto/datafusion.proto
+++ b/ballista/core/proto/datafusion.proto
@@ -273,7 +273,6 @@
INSERT_APPEND = 3;
INSERT_OVERWRITE = 4;
INSERT_REPLACE = 5;
-
}
Type dml_type = 1;
LogicalPlanNode input = 2;
@@ -484,6 +483,7 @@
LogicalExprNode expr = 1;
string alias = 2;
repeated TableReference relation = 3;
+ map<string, string> metadata = 4;
}
message BinaryExprNode {
@@ -725,6 +725,7 @@
CsvSinkExecNode csv_sink = 28;
ParquetSinkExecNode parquet_sink = 29;
UnnestExecNode unnest = 30;
+ JsonScanExecNode json_scan = 31;
}
}
@@ -996,6 +997,7 @@
reserved 10;
datafusion_common.Constraints constraints = 11;
+ optional uint64 batch_size = 12;
}
message ParquetScanExecNode {
@@ -1023,6 +1025,10 @@
bool newlines_in_values = 7;
}
+ message JsonScanExecNode {
+ FileScanExecConf base_conf = 1;
+ }
+
message AvroScanExecNode {
FileScanExecConf base_conf = 1;
}
diff --git a/ballista/core/proto/datafusion_common.proto
b/ballista/core/proto/datafusion_common.proto
index 3d0594f8..b81c91fa 100644
--- a/ballista/core/proto/datafusion_common.proto
+++ b/ballista/core/proto/datafusion_common.proto
@@ -545,6 +545,10 @@
uint64 max_row_group_size = 15;
string created_by = 16;
+
+ oneof coerce_int96_opt {
+ string coerce_int96 = 32;
+ }
}
enum JoinSide {
diff --git a/ballista/core/src/execution_plans/distributed_query.rs
b/ballista/core/src/execution_plans/distributed_query.rs
index 600042d2..90cf1190 100644
--- a/ballista/core/src/execution_plans/distributed_query.rs
+++ b/ballista/core/src/execution_plans/distributed_query.rs
@@ -140,6 +140,9 @@ impl<T: 'static + AsLogicalPlan> DisplayAs for
DistributedQueryExec<T> {
self.scheduler_url
)
}
+ DisplayFormatType::TreeRender => {
+ writeln!(f, "scheduler_url={}", self.scheduler_url)
+ }
}
}
}
diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs
b/ballista/core/src/execution_plans/shuffle_reader.rs
index 0bfe6ec9..b2ef2a82 100644
--- a/ballista/core/src/execution_plans/shuffle_reader.rs
+++ b/ballista/core/src/execution_plans/shuffle_reader.rs
@@ -103,6 +103,9 @@ impl DisplayAs for ShuffleReaderExec {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
write!(f, "ShuffleReaderExec: partitions={}",
self.partition.len())
}
+ DisplayFormatType::TreeRender => {
+ write!(f, "partitions={}", self.partition.len())
+ }
}
}
}
diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs
b/ballista/core/src/execution_plans/shuffle_writer.rs
index 4847f982..afb1409d 100644
--- a/ballista/core/src/execution_plans/shuffle_writer.rs
+++ b/ballista/core/src/execution_plans/shuffle_writer.rs
@@ -359,10 +359,13 @@ impl DisplayAs for ShuffleWriterExec {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
write!(
f,
- "ShuffleWriterExec: {:?}",
+ "ShuffleWriterExec: partitions:{:?}",
self.shuffle_output_partitioning
)
}
+ DisplayFormatType::TreeRender => {
+ write!(f, "partitions={:?}", self.shuffle_output_partitioning)
+ }
}
}
}
diff --git a/ballista/core/src/execution_plans/unresolved_shuffle.rs
b/ballista/core/src/execution_plans/unresolved_shuffle.rs
index 75a168e0..e7d46ba7 100644
--- a/ballista/core/src/execution_plans/unresolved_shuffle.rs
+++ b/ballista/core/src/execution_plans/unresolved_shuffle.rs
@@ -72,7 +72,14 @@ impl DisplayAs for UnresolvedShuffleExec {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
write!(
f,
- "UnresolvedShuffleExec: {:?}",
+ "UnresolvedShuffleExec: partitions={:?}",
+ self.properties().output_partitioning()
+ )
+ }
+ DisplayFormatType::TreeRender => {
+ write!(
+ f,
+ "partitions={:?}",
self.properties().output_partitioning()
)
}
diff --git a/ballista/executor/Cargo.toml b/ballista/executor/Cargo.toml
index dafdb0ee..b38f4d63 100644
--- a/ballista/executor/Cargo.toml
+++ b/ballista/executor/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-executor"
description = "Ballista Distributed Compute - Executor"
license = "Apache-2.0"
-version = "46.0.0"
+version = "47.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -42,7 +42,7 @@ default = ["build-binary", "mimalloc"]
arrow = { workspace = true }
arrow-flight = { workspace = true }
async-trait = { workspace = true }
-ballista-core = { path = "../core", version = "46.0.0" }
+ballista-core = { path = "../core", version = "47.0.0" }
configure_me = { workspace = true, optional = true }
dashmap = { workspace = true }
datafusion = { workspace = true }
diff --git a/ballista/executor/src/collect.rs b/ballista/executor/src/collect.rs
index 3d7d89f9..94dcc06f 100644
--- a/ballista/executor/src/collect.rs
+++ b/ballista/executor/src/collect.rs
@@ -63,6 +63,7 @@ impl DisplayAs for CollectExec {
DisplayFormatType::Default | DisplayFormatType::Verbose => {
write!(f, "CollectExec")
}
+ DisplayFormatType::TreeRender => write!(f, "CollectExec"),
}
}
}
diff --git a/ballista/executor/src/executor.rs
b/ballista/executor/src/executor.rs
index a3901c13..5e0b2045 100644
--- a/ballista/executor/src/executor.rs
+++ b/ballista/executor/src/executor.rs
@@ -286,7 +286,9 @@ mod test {
f: &mut std::fmt::Formatter,
) -> std::fmt::Result {
match t {
- DisplayFormatType::Default | DisplayFormatType::Verbose => {
+ DisplayFormatType::Default
+ | DisplayFormatType::Verbose
+ | DisplayFormatType::TreeRender => {
write!(f, "NeverendingOperator")
}
}
diff --git a/ballista/executor/src/flight_service.rs
b/ballista/executor/src/flight_service.rs
index fc04954c..279ab78c 100644
--- a/ballista/executor/src/flight_service.rs
+++ b/ballista/executor/src/flight_service.rs
@@ -220,9 +220,9 @@ where
T: Read + Seek,
{
if tx.is_closed() {
- return Err(FlightError::Tonic(Status::internal(
+ return Err(FlightError::Tonic(Box::new(Status::internal(
"Can't send a batch, channel is closed",
- )));
+ ))));
}
for batch in reader {
@@ -231,9 +231,9 @@ where
if let SendError(Err(err)) = err {
err
} else {
- FlightError::Tonic(Status::internal(format!(
+ FlightError::Tonic(Box::new(Status::internal(format!(
"Can't send a batch, something went wrong: {err:?}"
- )))
+ ))))
}
})?
}
diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml
index 0b56a9f6..fa9216e6 100644
--- a/ballista/scheduler/Cargo.toml
+++ b/ballista/scheduler/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-scheduler"
description = "Ballista Distributed Compute - Scheduler"
license = "Apache-2.0"
-version = "46.0.0"
+version = "47.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -49,7 +49,7 @@ rest-api = []
arrow-flight = { workspace = true }
async-trait = { workspace = true }
axum = "0.7.7"
-ballista-core = { path = "../core", version = "46.0.0" }
+ballista-core = { path = "../core", version = "47.0.0" }
clap = { workspace = true, optional = true }
configure_me = { workspace = true, optional = true }
dashmap = { workspace = true }
@@ -62,7 +62,7 @@ log = { workspace = true }
object_store = { workspace = true }
once_cell = { version = "1.16.0", optional = true }
parking_lot = { workspace = true }
-prometheus = { version = "0.13", features = ["process"], optional = true }
+prometheus = { version = "0.14", features = ["process"], optional = true }
prost = { workspace = true }
prost-types = { workspace = true }
rand = { workspace = true }
diff --git a/ballista/scheduler/src/cluster/mod.rs
b/ballista/scheduler/src/cluster/mod.rs
index 2b1a2f2b..25263a51 100644
--- a/ballista/scheduler/src/cluster/mod.rs
+++ b/ballista/scheduler/src/cluster/mod.rs
@@ -667,7 +667,13 @@ pub(crate) fn get_scan_files(
.downcast_ref::<DataSourceExec>()
.and_then(|c|
c.data_source().as_any().downcast_ref::<FileScanConfig>())
{
- collector.push(config.file_groups.clone());
+ collector.push(
+ config
+ .file_groups
+ .iter()
+ .map(|f| f.clone().into_inner())
+ .collect(),
+ );
Ok(TreeNodeRecursion::Jump)
} else {
Ok(TreeNodeRecursion::Continue)
diff --git a/ballista/scheduler/src/state/execution_graph_dot.rs
b/ballista/scheduler/src/state/execution_graph_dot.rs
index d4e5fff3..cb0531af 100644
--- a/ballista/scheduler/src/state/execution_graph_dot.rs
+++ b/ballista/scheduler/src/state/execution_graph_dot.rs
@@ -376,7 +376,7 @@ fn get_file_scan(scan: &FileScanConfig) -> String {
let partitioned_files: Vec<PartitionedFile> = scan
.file_groups
.iter()
- .flat_map(|part_file| part_file.clone())
+ .flat_map(|part_file| part_file.clone().into_inner())
.collect();
let paths: Vec<Path> = partitioned_files
.iter()
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index 779c993c..bd995210 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-benchmarks"
description = "Ballista Benchmarks"
-version = "46.0.0"
+version = "47.0.0"
edition = "2021"
authors = ["Apache DataFusion <d...@datafusion.apache.org>"]
homepage = "https://datafusion.apache.org/ballista/";
@@ -32,7 +32,7 @@ default = ["mimalloc"]
snmalloc = ["snmalloc-rs"]
[dependencies]
-ballista = { path = "../ballista/client", version = "46.0.0" }
+ballista = { path = "../ballista/client", version = "47.0.0" }
datafusion = { workspace = true }
datafusion-proto = { workspace = true }
env_logger = { workspace = true }
@@ -51,4 +51,4 @@ tokio = { version = "^1.44", features = [
] }
[dev-dependencies]
-ballista-core = { path = "../ballista/core", version = "46.0.0" }
+ballista-core = { path = "../ballista/core", version = "47.0.0" }
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 0f5823a4..93b66e1e 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-examples"
description = "Ballista usage examples"
-version = "46.0.0"
+version = "47.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
authors = ["Apache DataFusion <d...@datafusion.apache.org>"]
@@ -33,10 +33,10 @@ path = "examples/standalone-sql.rs"
required-features = ["ballista/standalone"]
[dependencies]
-ballista = { path = "../ballista/client", version = "46.0.0" }
-ballista-core = { path = "../ballista/core", version = "46.0.0", feature =
["binary-build"] }
-ballista-executor = { path = "../ballista/executor", version = "46.0.0",
default-features = false }
-ballista-scheduler = { path = "../ballista/scheduler", version = "46.0.0",
default-features = false }
+ballista = { path = "../ballista/client", version = "47.0.0" }
+ballista-core = { path = "../ballista/core", version = "47.0.0", feature =
["binary-build"] }
+ballista-executor = { path = "../ballista/executor", version = "47.0.0",
default-features = false }
+ballista-scheduler = { path = "../ballista/scheduler", version = "47.0.0",
default-features = false }
datafusion = { workspace = true }
env_logger = { workspace = true }
futures = { workspace = true }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@datafusion.apache.org
For additional commands, e-mail: commits-h...@datafusion.apache.org
