This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 8172786e Add S3 object store support to executor and scheduler (#1230)
8172786e is described below
commit 8172786eafc320b2a152980a66ae1ac8f2d583f5
Author: Marko Milenković <[email protected]>
AuthorDate: Fri Apr 18 16:11:54 2025 +0100
Add S3 object store support to executor and scheduler (#1230)
* add object store support for executor and scheduler
* fix integration test
* fix cargo after rebase
* add alias method to create session and improve tests
* fix clippy
---
Cargo.lock | 402 ++++++++++++++----------
ballista/core/Cargo.toml | 6 +-
ballista/core/src/extension.rs | 2 +-
ballista/core/src/lib.rs | 2 +
{examples => ballista/core}/src/object_store.rs | 42 ++-
ballista/executor/src/bin/main.rs | 7 +-
ballista/executor/src/flight_service.rs | 5 +-
ballista/scheduler/src/bin/main.rs | 11 +-
ballista/scheduler/src/config.rs | 16 +
examples/Cargo.toml | 3 +-
examples/examples/custom-client.rs | 7 +-
examples/examples/custom-executor.rs | 11 +-
examples/examples/custom-scheduler.rs | 12 +-
examples/src/lib.rs | 3 -
examples/tests/object_store.rs | 105 ++++---
15 files changed, 363 insertions(+), 271 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 07f2c28e..327ddb83 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -181,9 +181,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1"
+checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -202,9 +202,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a"
+checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -216,9 +216,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5"
+checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985"
dependencies = [
"ahash",
"arrow-buffer",
@@ -233,9 +233,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4e899dade2c3b7f5642eb8366cfd898958bcca099cde6dfea543c7e8d3ad88d4"
+checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c"
dependencies = [
"bytes",
"half",
@@ -244,9 +244,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a"
+checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -265,9 +265,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65"
+checksum = "1644877d8bc9a0ef022d9153dc29375c2bda244c39aec05a91d0e87ccf77995f"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -281,9 +281,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0a329fb064477c9ec5f0870d2f5130966f91055c7c5bce2b3a084f116bc28c3b"
+checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -293,9 +293,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c7408f2bf3b978eddda272c7699f439760ebc4ac70feca25fefa82c5b8ce808d"
+checksum = "a194f47959a4e111463cb6d02c8576fe084b3d7a3c092314baf3b9629b62595b"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -320,9 +320,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86"
+checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -335,9 +335,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd"
+checksum = "0ee5b4ca98a7fb2efb9ab3309a5d1c88b5116997ff93f3147efdc1062a6158e9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -346,18 +346,20 @@ dependencies = [
"arrow-schema",
"chrono",
"half",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"lexical-core",
+ "memchr",
"num",
"serde",
"serde_json",
+ "simdutf8",
]
[[package]]
name = "arrow-ord"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160"
+checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -368,9 +370,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd"
+checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -381,15 +383,15 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85934a9d0261e0fa5d4e2a5295107d743b543a6e0484a835d4b8db2da15306f9"
+checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9"
[[package]]
name = "arrow-select"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c"
+checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619"
dependencies = [
"ahash",
"arrow-array",
@@ -401,9 +403,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458"
+checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -489,9 +491,9 @@ checksum =
"ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "aws-config"
-version = "1.6.0"
+version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6a84fe2c5e9965fba0fbc2001db252f1d57527d82a905cca85127df227bca748"
+checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -531,9 +533,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.12.6"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01"
+checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -541,9 +543,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.27.1"
+version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f"
+checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f"
dependencies = [
"bindgen",
"cc",
@@ -579,9 +581,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.62.0"
+version = "1.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1d5330ad4e8a1ff49e9f26b738611caa72b105c41d41733801d1a36e8f9de936"
+checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -593,6 +595,7 @@ dependencies = [
"aws-smithy-types",
"aws-types",
"bytes",
+ "fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
@@ -601,9 +604,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.63.0"
+version = "1.65.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7956b1a85d49082347a7d17daa2e32df191f3e23c03d47294b99f95413026a78"
+checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -615,6 +618,7 @@ dependencies = [
"aws-smithy-types",
"aws-types",
"bytes",
+ "fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
@@ -623,9 +627,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.63.0"
+version = "1.65.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "065c533fbe6f84962af33fcf02b0350b7c1f79285baab5924615d2be3b232855"
+checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -638,6 +642,7 @@ dependencies = [
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
+ "fastrand",
"http 0.2.12",
"once_cell",
"regex-lite",
@@ -701,9 +706,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http-client"
-version = "1.0.0"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0497ef5d53065b7cd6a35e9c1654bd1fefeae5c52900d91d1b188b0af0f29324"
+checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -731,6 +736,16 @@ dependencies = [
"aws-smithy-types",
]
+[[package]]
+name = "aws-smithy-observability"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0"
+dependencies = [
+ "aws-smithy-runtime-api",
+ "once_cell",
+]
+
[[package]]
name = "aws-smithy-query"
version = "0.60.7"
@@ -743,13 +758,14 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
-version = "1.8.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f6328865e36c6fd970094ead6b05efd047d3a80ec5fc3be5e743910da9f2ebf8"
+checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http-client",
+ "aws-smithy-observability",
"aws-smithy-runtime-api",
"aws-smithy-types",
"bytes",
@@ -942,7 +958,7 @@ name = "ballista-cli"
version = "46.0.0"
dependencies = [
"ballista",
- "clap 4.5.32",
+ "clap 4.5.36",
"datafusion",
"datafusion-cli",
"dirs",
@@ -958,8 +974,10 @@ version = "46.0.0"
dependencies = [
"arrow-flight",
"async-trait",
+ "aws-config",
+ "aws-credential-types",
"chrono",
- "clap 4.5.32",
+ "clap 4.5.36",
"configure_me",
"datafusion",
"datafusion-proto",
@@ -968,6 +986,8 @@ dependencies = [
"itertools 0.14.0",
"log",
"md-5",
+ "object_store",
+ "parking_lot",
"prost",
"prost-types",
"rand 0.9.0",
@@ -994,7 +1014,6 @@ dependencies = [
"env_logger",
"log",
"object_store",
- "parking_lot",
"testcontainers-modules",
"tokio",
"tonic",
@@ -1038,7 +1057,7 @@ dependencies = [
"axum",
"ballista-core",
"base64 0.22.1",
- "clap 4.5.32",
+ "clap 4.5.36",
"configure_me",
"configure_me_codegen",
"dashmap",
@@ -1090,9 +1109,9 @@ dependencies = [
[[package]]
name = "bigdecimal"
-version = "0.4.7"
+version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c"
+checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013"
dependencies = [
"autocfg",
"libm",
@@ -1148,9 +1167,9 @@ dependencies = [
[[package]]
name = "blake3"
-version = "1.7.0"
+version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7"
+checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3"
dependencies = [
"arrayref",
"arrayvec",
@@ -1308,9 +1327,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.17"
+version = "1.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a"
+checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362"
dependencies = [
"jobserver",
"libc",
@@ -1340,22 +1359,22 @@ checksum =
"613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
-version = "0.4.39"
+version = "0.4.40"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
+checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c"
dependencies = [
"android-tzdata",
"iana-time-zone",
"num-traits",
"serde",
- "windows-targets 0.52.6",
+ "windows-link",
]
[[package]]
name = "chrono-tz"
-version = "0.10.1"
+version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f"
+checksum = "efdce149c370f133a071ca8ef6ea340b7b88748ab0810097a9e2976eaa34b4f3"
dependencies = [
"chrono",
"chrono-tz-build",
@@ -1364,9 +1383,9 @@ dependencies = [
[[package]]
name = "chrono-tz-build"
-version = "0.4.0"
+version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7"
+checksum = "8f10f8c9340e31fc120ff885fcdb54a0b48e474bbd77cab557f0c30a3e569402"
dependencies = [
"parse-zoneinfo",
"phf_codegen",
@@ -1396,9 +1415,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.32"
+version = "4.5.36"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83"
+checksum = "2df961d8c8a0d08aa9945718ccf584145eee3f3aa06cddbeac12933781102e04"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1406,9 +1425,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.32"
+version = "4.5.36"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8"
+checksum = "132dbda40fb6753878316a489d5a1242a8ef2f0d9e47ba01c951ea8aa7d013a5"
dependencies = [
"anstream",
"anstyle",
@@ -1635,9 +1654,9 @@ checksum =
"4f211af61d8efdd104f96e57adf5e426ba1bc3ed7a4ead616e15e5881fd79c4d"
[[package]]
name = "darling"
-version = "0.20.10"
+version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
+checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
dependencies = [
"darling_core",
"darling_macro",
@@ -1645,9 +1664,9 @@ dependencies = [
[[package]]
name = "darling_core"
-version = "0.20.10"
+version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
+checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
dependencies = [
"fnv",
"ident_case",
@@ -1659,9 +1678,9 @@ dependencies = [
[[package]]
name = "darling_macro"
-version = "0.20.10"
+version = "0.20.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
+checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core",
"quote",
@@ -1793,7 +1812,7 @@ dependencies = [
"async-trait",
"aws-config",
"aws-credential-types",
- "clap 4.5.32",
+ "clap 4.5.36",
"datafusion",
"dirs",
"env_logger",
@@ -1821,7 +1840,7 @@ dependencies = [
"base64 0.22.1",
"half",
"hashbrown 0.14.5",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"libc",
"log",
"object_store",
@@ -1916,7 +1935,7 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-functions-window-common",
"datafusion-physical-expr-common",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"paste",
"recursive",
"serde_json",
@@ -1931,7 +1950,7 @@ checksum =
"18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e"
dependencies = [
"arrow",
"datafusion-common",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"itertools 0.14.0",
"paste",
]
@@ -2085,7 +2104,7 @@ dependencies = [
"datafusion-common",
"datafusion-expr",
"datafusion-physical-expr",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"itertools 0.14.0",
"log",
"recursive",
@@ -2108,7 +2127,7 @@ dependencies = [
"datafusion-physical-expr-common",
"half",
"hashbrown 0.14.5",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"itertools 0.14.0",
"log",
"paste",
@@ -2170,7 +2189,7 @@ dependencies = [
"futures",
"half",
"hashbrown 0.14.5",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"itertools 0.14.0",
"log",
"parking_lot",
@@ -2215,7 +2234,7 @@ dependencies = [
"bigdecimal",
"datafusion-common",
"datafusion-expr",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"log",
"recursive",
"regex",
@@ -2346,9 +2365,9 @@ dependencies = [
[[package]]
name = "env_logger"
-version = "0.11.7"
+version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697"
+checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f"
dependencies = [
"anstream",
"anstyle",
@@ -2365,9 +2384,9 @@ checksum =
"877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
-version = "0.3.10"
+version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
+checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
dependencies = [
"libc",
"windows-sys 0.59.0",
@@ -2403,7 +2422,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78"
dependencies = [
"cfg-if",
- "rustix 1.0.3",
+ "rustix 1.0.5",
"windows-sys 0.59.0",
]
@@ -2437,9 +2456,9 @@ dependencies = [
[[package]]
name = "flate2"
-version = "1.1.0"
+version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
+checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece"
dependencies = [
"crc32fast",
"miniz_oxide",
@@ -2644,7 +2663,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http 1.3.1",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"slab",
"tokio",
"tokio-util",
@@ -2653,9 +2672,9 @@ dependencies = [
[[package]]
name = "half"
-version = "2.5.0"
+version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1"
+checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
dependencies = [
"cfg-if",
"crunchy",
@@ -2866,9 +2885,9 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.10"
+version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
+checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
dependencies = [
"bytes",
"futures-channel",
@@ -2876,6 +2895,7 @@ dependencies = [
"http 1.3.1",
"http-body 1.0.1",
"hyper",
+ "libc",
"pin-project-lite",
"socket2",
"tokio",
@@ -2900,14 +2920,15 @@ dependencies = [
[[package]]
name = "iana-time-zone"
-version = "0.1.61"
+version = "0.1.63"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
+checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
+ "log",
"wasm-bindgen",
"windows-core",
]
@@ -2962,9 +2983,9 @@ dependencies = [
[[package]]
name = "icu_locid_transform_data"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
+checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d"
[[package]]
name = "icu_normalizer"
@@ -2986,9 +3007,9 @@ dependencies = [
[[package]]
name = "icu_normalizer_data"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
+checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7"
[[package]]
name = "icu_properties"
@@ -3007,9 +3028,9 @@ dependencies = [
[[package]]
name = "icu_properties_data"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
+checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2"
[[package]]
name = "icu_provider"
@@ -3079,9 +3100,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.8.0"
+version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
+checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e"
dependencies = [
"equivalent",
"hashbrown 0.15.2",
@@ -3163,9 +3184,9 @@ checksum =
"4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "jiff"
-version = "0.2.4"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e"
+checksum = "1f33145a5cbea837164362c7bd596106eb7c5198f97d1ba6f6ebb3223952e488"
dependencies = [
"jiff-static",
"log",
@@ -3176,9 +3197,9 @@ dependencies = [
[[package]]
name = "jiff-static"
-version = "0.2.4"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9"
+checksum = "43ce13c40ec6956157a3635d97a1ee2df323b263f09ea14165131289cb0f5c19"
dependencies = [
"proc-macro2",
"quote",
@@ -3187,10 +3208,11 @@ dependencies = [
[[package]]
name = "jobserver"
-version = "0.1.32"
+version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
+checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
dependencies = [
+ "getrandom 0.3.2",
"libc",
]
@@ -3328,9 +3350,9 @@ checksum =
"8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
[[package]]
name = "libmimalloc-sys"
-version = "0.1.40"
+version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "07d0e07885d6a754b9c7993f2625187ad694ee985d60f23355ff0e7077261502"
+checksum = "ec9d6fac27761dabcd4ee73571cdb06b7022dc99089acbe5435691edffaac0f4"
dependencies = [
"cc",
"libc",
@@ -3344,7 +3366,7 @@ checksum =
"c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.9.0",
"libc",
- "redox_syscall 0.5.10",
+ "redox_syscall 0.5.11",
]
[[package]]
@@ -3355,9 +3377,9 @@ checksum =
"d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "linux-raw-sys"
-version = "0.9.3"
+version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
+checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "litemap"
@@ -3377,9 +3399,9 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.26"
+version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
+checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "lz4_flex"
@@ -3443,9 +3465,9 @@ checksum =
"78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "mimalloc"
-version = "0.1.44"
+version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "99585191385958383e13f6b822e6b6d8d9cf928e7d286ceb092da92b43c87bc1"
+checksum = "995942f432bbb4822a7e9c3faa87a695185b0d09273ba85f097b54f4e458f2af"
dependencies = [
"libmimalloc-sys",
]
@@ -3464,9 +3486,9 @@ checksum =
"68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
-version = "0.8.5"
+version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
+checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a"
dependencies = [
"adler2",
]
@@ -3652,9 +3674,9 @@ dependencies = [
[[package]]
name = "once_cell"
-version = "1.21.1"
+version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc"
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "openssl-probe"
@@ -3707,16 +3729,16 @@ checksum =
"1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.10",
+ "redox_syscall 0.5.11",
"smallvec",
"windows-targets 0.52.6",
]
[[package]]
name = "parquet"
-version = "54.2.1"
+version = "54.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd"
+checksum = "bfb15796ac6f56b429fd99e33ba133783ad75b27c36b4b5ce06f1f82cc97754e"
dependencies = [
"ahash",
"arrow-array",
@@ -3746,7 +3768,6 @@ dependencies = [
"tokio",
"twox-hash",
"zstd",
- "zstd-sys",
]
[[package]]
@@ -3803,9 +3824,9 @@ checksum =
"e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pest"
-version = "2.7.15"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc"
+checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6"
dependencies = [
"memchr",
"thiserror 2.0.12",
@@ -3814,9 +3835,9 @@ dependencies = [
[[package]]
name = "pest_derive"
-version = "2.7.15"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e"
+checksum = "d725d9cfd79e87dccc9341a2ef39d1b6f6353d68c4b33c177febbe1a402c97c5"
dependencies = [
"pest",
"pest_generator",
@@ -3824,9 +3845,9 @@ dependencies = [
[[package]]
name = "pest_generator"
-version = "2.7.15"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b"
+checksum = "db7d01726be8ab66ab32f9df467ae8b1148906685bbe75c82d1e65d7f5b3f841"
dependencies = [
"pest",
"pest_meta",
@@ -3837,9 +3858,9 @@ dependencies = [
[[package]]
name = "pest_meta"
-version = "2.7.15"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea"
+checksum = "7f9f832470494906d1fca5329f8ab5791cc60beb230c74815dff541cbd2b5ca0"
dependencies = [
"once_cell",
"pest",
@@ -3853,7 +3874,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
dependencies = [
"fixedbitset",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
]
[[package]]
@@ -3964,9 +3985,9 @@ dependencies = [
[[package]]
name = "prettyplease"
-version = "0.2.31"
+version = "0.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb"
+checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6"
dependencies = [
"proc-macro2",
"syn 2.0.100",
@@ -4129,9 +4150,9 @@ checksum =
"5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
[[package]]
name = "quick-xml"
-version = "0.37.2"
+version = "0.37.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003"
+checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369"
dependencies = [
"memchr",
"serde",
@@ -4179,9 +4200,9 @@ dependencies = [
[[package]]
name = "quinn-udp"
-version = "0.5.10"
+version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
+checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5"
dependencies = [
"cfg_aliases",
"libc",
@@ -4307,9 +4328,9 @@ dependencies = [
[[package]]
name = "redox_syscall"
-version = "0.5.10"
+version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
+checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3"
dependencies = [
"bitflags 2.9.0",
]
@@ -4525,22 +4546,22 @@ dependencies = [
[[package]]
name = "rustix"
-version = "1.0.3"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96"
+checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
dependencies = [
"bitflags 2.9.0",
"errno",
"libc",
- "linux-raw-sys 0.9.3",
+ "linux-raw-sys 0.9.4",
"windows-sys 0.59.0",
]
[[package]]
name = "rustls"
-version = "0.23.25"
+version = "0.23.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
+checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -4583,9 +4604,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.103.0"
+version = "0.103.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f"
+checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03"
dependencies = [
"aws-lc-rs",
"ring",
@@ -4779,7 +4800,7 @@ dependencies = [
"chrono",
"hex",
"indexmap 1.9.3",
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"serde",
"serde_derive",
"serde_json",
@@ -4857,9 +4878,9 @@ dependencies = [
[[package]]
name = "smallvec"
-version = "1.14.0"
+version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
+checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9"
[[package]]
name = "snafu"
@@ -4908,9 +4929,9 @@ dependencies = [
[[package]]
name = "socket2"
-version = "0.5.8"
+version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
+checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef"
dependencies = [
"libc",
"windows-sys 0.52.0",
@@ -5092,7 +5113,7 @@ dependencies = [
"fastrand",
"getrandom 0.3.2",
"once_cell",
- "rustix 1.0.3",
+ "rustix 1.0.5",
"windows-sys 0.59.0",
]
@@ -5206,9 +5227,9 @@ dependencies = [
[[package]]
name = "time"
-version = "0.3.40"
+version = "0.3.41"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618"
+checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40"
dependencies = [
"deranged",
"itoa",
@@ -5227,9 +5248,9 @@ checksum =
"c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c"
[[package]]
name = "time-macros"
-version = "0.2.21"
+version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04"
+checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49"
dependencies = [
"num-conv",
"time-core",
@@ -5383,7 +5404,7 @@ version = "0.22.24"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
dependencies = [
- "indexmap 2.8.0",
+ "indexmap 2.9.0",
"serde",
"serde_spanned",
"toml_datetime",
@@ -5889,11 +5910,37 @@ checksum =
"712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
-version = "0.52.0"
+version = "0.61.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
+checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980"
dependencies = [
- "windows-targets 0.52.6",
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings 0.4.0",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.100",
]
[[package]]
@@ -5909,7 +5956,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
dependencies = [
"windows-result",
- "windows-strings",
+ "windows-strings 0.3.1",
"windows-targets 0.53.0",
]
@@ -5931,6 +5978,15 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "windows-strings"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97"
+dependencies = [
+ "windows-link",
+]
+
[[package]]
name = "windows-sys"
version = "0.48.0"
@@ -6145,9 +6201,9 @@ checksum =
"271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]]
name = "winnow"
-version = "0.7.4"
+version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36"
+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
dependencies = [
"memchr",
]
@@ -6180,7 +6236,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e"
dependencies = [
"libc",
- "rustix 1.0.3",
+ "rustix 1.0.5",
]
[[package]]
@@ -6322,18 +6378,18 @@ dependencies = [
[[package]]
name = "zstd-safe"
-version = "7.2.1"
+version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
-version = "2.0.13+zstd.1.5.6"
+version = "2.0.15+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
+checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237"
dependencies = [
"cc",
"pkg-config",
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 6a4e356f..ab9b39e4 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -34,7 +34,7 @@ exclude = ["*.proto"]
rustc-args = ["--cfg", "docsrs"]
[features]
-build-binary = ["configure_me", "clap"]
+build-binary = ["aws-config", "aws-credential-types", "configure_me", "clap",
"object_store"]
docsrs = []
# Used for testing ONLY: causes all values to hash to the same value (test for
collisions)
force_hash_collisions = ["datafusion/force_hash_collisions"]
@@ -42,6 +42,8 @@ force_hash_collisions = ["datafusion/force_hash_collisions"]
[dependencies]
arrow-flight = { workspace = true }
async-trait = { workspace = true }
+aws-config = { version = "1.6.0", optional = true }
+aws-credential-types = { version = "1.2.0", optional = true }
chrono = { version = "0.4", default-features = false }
clap = { workspace = true, optional = true }
configure_me = { workspace = true, optional = true }
@@ -52,6 +54,8 @@ futures = { workspace = true }
itertools = "0.14"
log = { workspace = true }
md-5 = { version = "^0.10.0" }
+object_store = { workspace = true, features = ["aws","http"], optional = true }
+parking_lot = { workspace = true }
prost = { workspace = true }
prost-types = { workspace = true }
rand = { workspace = true }
diff --git a/ballista/core/src/extension.rs b/ballista/core/src/extension.rs
index 4a25ee12..74d66713 100644
--- a/ballista/core/src/extension.rs
+++ b/ballista/core/src/extension.rs
@@ -355,7 +355,7 @@ impl SessionConfigHelperExt for SessionConfig {
"could not set configuration key: `{}`, value:
`{:?}`, reason: {}",
key,
value,
- e.to_string()
+ e
)
}
}
diff --git a/ballista/core/src/lib.rs b/ballista/core/src/lib.rs
index 5d55b9ec..15f9dd19 100644
--- a/ballista/core/src/lib.rs
+++ b/ballista/core/src/lib.rs
@@ -34,6 +34,8 @@ pub mod error;
pub mod event_loop;
pub mod execution_plans;
pub mod extension;
+#[cfg(feature = "build-binary")]
+pub mod object_store;
pub mod planner;
pub mod registry;
pub mod serde;
diff --git a/examples/src/object_store.rs b/ballista/core/src/object_store.rs
similarity index 88%
rename from examples/src/object_store.rs
rename to ballista/core/src/object_store.rs
index c596a3da..594056e8 100644
--- a/examples/src/object_store.rs
+++ b/ballista/core/src/object_store.rs
@@ -23,35 +23,35 @@
//! [ObjectStore] is provided by [ObjectStoreRegistry], and configured
//! using [ExtensionOptions], which can be configured using SQL `SET` command.
-use ballista::prelude::SessionConfigExt;
use datafusion::common::{config_err, exec_err};
use datafusion::config::{
ConfigEntry, ConfigExtension, ConfigField, ExtensionOptions, Visit,
};
use datafusion::error::Result;
use datafusion::execution::object_store::ObjectStoreRegistry;
-use datafusion::execution::runtime_env::RuntimeEnvBuilder;
-use datafusion::execution::SessionState;
+
+use datafusion::error::DataFusionError;
+use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
+use datafusion::execution::{SessionState, SessionStateBuilder};
use datafusion::prelude::SessionConfig;
-use datafusion::{
- error::DataFusionError,
- execution::{runtime_env::RuntimeEnv, SessionStateBuilder},
-};
use object_store::aws::AmazonS3Builder;
+use object_store::http::HttpBuilder;
use object_store::local::LocalFileSystem;
-use object_store::ObjectStore;
+use object_store::{ClientOptions, ObjectStore};
use parking_lot::RwLock;
use std::any::Any;
use std::fmt::Display;
use std::sync::Arc;
use url::Url;
+use crate::extension::SessionConfigExt;
+
/// Custom [SessionConfig] constructor method
///
/// This method registers config extension [S3Options]
/// which is used to configure [ObjectStore] with ACCESS and
/// SECRET key
-pub fn custom_session_config_with_s3_options() -> SessionConfig {
+pub fn session_config_with_s3_support() -> SessionConfig {
SessionConfig::new_with_ballista()
.with_information_schema(true)
.with_option_extension(S3Options::default())
@@ -62,7 +62,7 @@ pub fn custom_session_config_with_s3_options() ->
SessionConfig {
/// It will register [CustomObjectStoreRegistry] which will
/// use configuration extension [S3Options] to configure
/// and created [ObjectStore]s
-pub fn custom_runtime_env_with_s3_support(
+pub fn runtime_env_with_s3_support(
session_config: &SessionConfig,
) -> Result<Arc<RuntimeEnv>> {
let s3options = session_config
@@ -82,14 +82,14 @@ pub fn custom_runtime_env_with_s3_support(
Ok(Arc::new(runtime_env))
}
-/// Custom [SessionState] constructor method
+/// Custom [SessionState] with S3 support enabled
///
/// It will configure [SessionState] with provided [SessionConfig],
/// and [RuntimeEnv].
-pub fn custom_session_state_with_s3_support(
+pub fn session_state_with_s3_support(
session_config: SessionConfig,
) -> datafusion::common::Result<SessionState> {
- let runtime_env = custom_runtime_env_with_s3_support(&session_config)?;
+ let runtime_env = runtime_env_with_s3_support(&session_config)?;
Ok(SessionStateBuilder::new()
.with_runtime_env(runtime_env)
@@ -98,6 +98,16 @@ pub fn custom_session_state_with_s3_support(
.build())
}
+/// Custom [SessionState] with S3 support.
+/// It is alias to [session_state_with_s3_support] with
[session_config_with_s3_support] as a
+/// parameter
+///
+/// It will configure [SessionState] S3 enabled [SessionConfig],
+/// and [RuntimeEnv].
+pub fn state_with_s3_support() -> datafusion::common::Result<SessionState> {
+ session_state_with_s3_support(session_config_with_s3_support())
+}
+
/// Custom [ObjectStoreRegistry] which will create
/// and configure [ObjectStore] using provided [S3Options]
#[derive(Debug)]
@@ -129,6 +139,12 @@ impl ObjectStoreRegistry for CustomObjectStoreRegistry {
log::info!("get_store: {:?}", &self.s3options.config.read());
match scheme {
"" | "file" => Ok(self.local.clone()),
+ "http" | "https" => Ok(Arc::new(
+ HttpBuilder::new()
+
.with_client_options(ClientOptions::new().with_allow_http(true))
+ .with_url(url.origin().ascii_serialization())
+ .build()?,
+ )),
"s3" => {
let s3store =
Self::s3_object_store_builder(url,
&self.s3options.config.read())?
diff --git a/ballista/executor/src/bin/main.rs
b/ballista/executor/src/bin/main.rs
index 18abb996..50a6154f 100644
--- a/ballista/executor/src/bin/main.rs
+++ b/ballista/executor/src/bin/main.rs
@@ -18,6 +18,9 @@
//! Ballista Rust executor binary.
use ballista_core::config::LogRotationPolicy;
+use ballista_core::object_store::{
+ runtime_env_with_s3_support, session_config_with_s3_support,
+};
use ballista_core::print_version;
use ballista_executor::config::prelude::*;
use ballista_executor::executor_process::{
@@ -43,7 +46,9 @@ async fn main() -> ballista_core::error::Result<()> {
std::process::exit(0);
}
- let config: ExecutorProcessConfig = opt.try_into()?;
+ let mut config: ExecutorProcessConfig = opt.try_into()?;
+ config.override_config_producer =
Some(Arc::new(session_config_with_s3_support));
+ config.override_runtime_producer =
Some(Arc::new(runtime_env_with_s3_support));
let rust_log = env::var(EnvFilter::DEFAULT_ENV);
let log_filter =
diff --git a/ballista/executor/src/flight_service.rs
b/ballista/executor/src/flight_service.rs
index 26ec814e..fc04954c 100644
--- a/ballista/executor/src/flight_service.rs
+++ b/ballista/executor/src/flight_service.rs
@@ -102,10 +102,7 @@ impl FlightService for BallistaFlightService {
let schema = reader.schema();
task::spawn_blocking(move || {
if let Err(e) = read_partition(reader, tx) {
- log::warn!(
- "error streaming shuffle partition: {}",
- e.to_string()
- );
+ log::warn!("error streaming shuffle partition: {}", e);
}
});
diff --git a/ballista/scheduler/src/bin/main.rs
b/ballista/scheduler/src/bin/main.rs
index ea31810a..90032e6d 100644
--- a/ballista/scheduler/src/bin/main.rs
+++ b/ballista/scheduler/src/bin/main.rs
@@ -19,9 +19,12 @@
use ballista_core::config::LogRotationPolicy;
use ballista_core::error::BallistaError;
+use ballista_core::object_store::{
+ session_config_with_s3_support, session_state_with_s3_support,
+};
use ballista_core::print_version;
use ballista_scheduler::cluster::BallistaCluster;
-use ballista_scheduler::config::{Config, ResultExt};
+use ballista_scheduler::config::{Config, ResultExt, SchedulerConfig};
use ballista_scheduler::scheduler_process::start_server;
use std::sync::Arc;
use std::{env, io};
@@ -89,7 +92,11 @@ async fn inner() -> ballista_core::error::Result<()> {
BallistaError::Configuration(e.to_string())
})?;
- let config = opt.try_into()?;
+ let config: SchedulerConfig = opt.try_into()?;
+ let config = config
+
.with_override_config_producer(Arc::new(session_config_with_s3_support))
+
.with_override_session_builder(Arc::new(session_state_with_s3_support));
+
let cluster = BallistaCluster::new_from_config(&config).await?;
start_server(cluster, addr, Arc::new(config)).await?;
diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs
index b221ecb6..2248e9b8 100644
--- a/ballista/scheduler/src/config.rs
+++ b/ballista/scheduler/src/config.rs
@@ -199,6 +199,22 @@ impl SchedulerConfig {
self.grpc_server_max_encoding_message_size = value;
self
}
+
+ pub fn with_override_config_producer(
+ mut self,
+ override_config_producer: ConfigProducer,
+ ) -> Self {
+ self.override_config_producer = Some(override_config_producer);
+ self
+ }
+
+ pub fn with_override_session_builder(
+ mut self,
+ override_session_builder: SessionBuilder,
+ ) -> Self {
+ self.override_session_builder = Some(override_session_builder);
+ self
+ }
}
#[derive(Clone, Debug, Default)]
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index a02d59aa..64fbb25a 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -34,14 +34,13 @@ required-features = ["ballista/standalone"]
[dependencies]
ballista = { path = "../ballista/client", version = "46.0.0" }
-ballista-core = { path = "../ballista/core", version = "46.0.0" }
+ballista-core = { path = "../ballista/core", version = "46.0.0", feature =
["binary-build"] }
ballista-executor = { path = "../ballista/executor", version = "46.0.0",
default-features = false }
ballista-scheduler = { path = "../ballista/scheduler", version = "46.0.0",
default-features = false }
datafusion = { workspace = true }
env_logger = { workspace = true }
log = { workspace = true }
object_store = { workspace = true, features = ["aws"] }
-parking_lot = { workspace = true }
tokio = { workspace = true, features = [
"macros",
"rt",
diff --git a/examples/examples/custom-client.rs
b/examples/examples/custom-client.rs
index 9e7ec859..29467a24 100644
--- a/examples/examples/custom-client.rs
+++ b/examples/examples/custom-client.rs
@@ -16,9 +16,7 @@
// under the License.
use ballista::extension::SessionContextExt;
-use ballista_examples::object_store::{
- custom_session_config_with_s3_options,
custom_session_state_with_s3_support,
-};
+use ballista_core::object_store::state_with_s3_support;
use datafusion::error::Result;
use datafusion::{assert_batches_eq, prelude::SessionContext};
@@ -61,8 +59,7 @@ async fn main() -> Result<()> {
let test_data = ballista_examples::test_util::examples_test_data();
// new sessions state with required custom session configuration and
runtime environment
- let state =
-
custom_session_state_with_s3_support(custom_session_config_with_s3_options())?;
+ let state = state_with_s3_support()?;
let ctx: SessionContext =
SessionContext::remote_with_state("df://localhost:50050",
state).await?;
diff --git a/examples/examples/custom-executor.rs
b/examples/examples/custom-executor.rs
index 53418212..6e0e6c69 100644
--- a/examples/examples/custom-executor.rs
+++ b/examples/examples/custom-executor.rs
@@ -15,14 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use ballista_examples::object_store::{
- custom_runtime_env_with_s3_support, custom_session_config_with_s3_options,
+use ballista_core::object_store::{
+ runtime_env_with_s3_support, session_config_with_s3_support,
};
use ballista_executor::executor_process::{
start_executor_process, ExecutorProcessConfig,
};
-use datafusion::prelude::SessionConfig;
use std::sync::Arc;
///
/// # Custom Ballista Executor
@@ -39,12 +38,10 @@ async fn main() -> ballista_core::error::Result<()> {
let config: ExecutorProcessConfig = ExecutorProcessConfig {
// overriding default config producer with custom producer
// which has required S3 configuration options
- override_config_producer:
Some(Arc::new(custom_session_config_with_s3_options)),
+ override_config_producer:
Some(Arc::new(session_config_with_s3_support)),
// overriding default runtime producer with custom producer
// which knows how to create S3 connections
- override_runtime_producer: Some(Arc::new(|session_config:
&SessionConfig| {
- custom_runtime_env_with_s3_support(session_config)
- })),
+ override_runtime_producer: Some(Arc::new(runtime_env_with_s3_support)),
..Default::default()
};
diff --git a/examples/examples/custom-scheduler.rs
b/examples/examples/custom-scheduler.rs
index 9783ae28..f999825b 100644
--- a/examples/examples/custom-scheduler.rs
+++ b/examples/examples/custom-scheduler.rs
@@ -16,13 +16,13 @@
// under the License.
use ballista_core::error::BallistaError;
-use ballista_examples::object_store::{
- custom_session_config_with_s3_options,
custom_session_state_with_s3_support,
+use ballista_core::object_store::{
+ session_config_with_s3_support, session_state_with_s3_support,
};
+
use ballista_scheduler::cluster::BallistaCluster;
use ballista_scheduler::config::SchedulerConfig;
use ballista_scheduler::scheduler_process::start_server;
-use datafusion::prelude::SessionConfig;
use std::net::AddrParseError;
use std::sync::Arc;
@@ -41,12 +41,10 @@ async fn main() -> ballista_core::error::Result<()> {
let config: SchedulerConfig = SchedulerConfig {
// overriding default runtime producer with custom producer
// which knows how to create S3 connections
- override_config_producer:
Some(Arc::new(custom_session_config_with_s3_options)),
+ override_config_producer:
Some(Arc::new(session_config_with_s3_support)),
// overriding default session builder, which has custom session
configuration
// runtime environment and session state.
- override_session_builder: Some(Arc::new(|session_config:
SessionConfig| {
- custom_session_state_with_s3_support(session_config)
- })),
+ override_session_builder:
Some(Arc::new(session_state_with_s3_support)),
..Default::default()
};
diff --git a/examples/src/lib.rs b/examples/src/lib.rs
index f8d7cc59..6dc48f6b 100644
--- a/examples/src/lib.rs
+++ b/examples/src/lib.rs
@@ -15,7 +15,4 @@
// specific language governing permissions and limitations
// under the License.
-/// Provides required structures and methods to
-/// integrate with S3 object store
-pub mod object_store;
pub mod test_util;
diff --git a/examples/tests/object_store.rs b/examples/tests/object_store.rs
index aa1a7864..b45a08ee 100644
--- a/examples/tests/object_store.rs
+++ b/examples/tests/object_store.rs
@@ -32,14 +32,9 @@ mod standalone {
use ballista::extension::SessionContextExt;
use ballista_examples::test_util::examples_test_data;
+ use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::{assert_batches_eq, prelude::SessionContext};
- use datafusion::{
- error::DataFusionError,
- execution::{
- runtime_env::{RuntimeConfig, RuntimeEnv},
- SessionStateBuilder,
- },
- };
+ use datafusion::{error::DataFusionError, execution::SessionStateBuilder};
use std::sync::Arc;
use testcontainers_modules::testcontainers::runners::AsyncRunner;
@@ -59,8 +54,7 @@ mod standalone {
.map_err(|e| DataFusionError::External(e.into()))?;
let test_data = examples_test_data();
- let config = RuntimeConfig::new();
- let runtime_env = RuntimeEnv::try_new(config)?;
+ let runtime_env = RuntimeEnvBuilder::new().build()?;
runtime_env.register_object_store(
&format!("s3://{}", crate::common::BUCKET)
@@ -71,6 +65,7 @@ mod standalone {
);
let state = SessionStateBuilder::new()
.with_runtime_env(runtime_env.into())
+ .with_default_features()
.build();
let ctx: SessionContext =
SessionContext::standalone_with_state(state).await?;
@@ -108,6 +103,22 @@ mod standalone {
];
assert_batches_eq!(expected, &result);
+
+ let result = ctx
+ .sql("select max(id) as max, min(id) as min from written_table")
+ .await?
+ .collect()
+ .await?;
+ let expected = [
+ "+-----+-----+",
+ "| max | min |",
+ "+-----+-----+",
+ "| 7 | 0 |",
+ "+-----+-----+",
+ ];
+
+ assert_batches_eq!(expected, &result);
+
Ok(())
}
}
@@ -118,14 +129,9 @@ mod remote {
use ballista::extension::SessionContextExt;
use ballista_examples::test_util::examples_test_data;
+ use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::{assert_batches_eq, prelude::SessionContext};
- use datafusion::{
- error::DataFusionError,
- execution::{
- runtime_env::{RuntimeConfig, RuntimeEnv},
- SessionStateBuilder,
- },
- };
+ use datafusion::{error::DataFusionError, execution::SessionStateBuilder};
use std::sync::Arc;
use testcontainers_modules::testcontainers::runners::AsyncRunner;
@@ -146,8 +152,7 @@ mod remote {
let object_store = crate::common::create_s3_store(&host.to_string(),
port)
.map_err(|e| DataFusionError::External(e.into()))?;
- let config = RuntimeConfig::new();
- let runtime_env = RuntimeEnv::try_new(config)?;
+ let runtime_env = RuntimeEnvBuilder::new().build()?;
runtime_env.register_object_store(
&format!("s3://{}", crate::common::BUCKET)
@@ -216,19 +221,14 @@ mod custom_s3_config {
use crate::common::{ACCESS_KEY_ID, SECRET_KEY};
use ballista::extension::SessionContextExt;
use ballista::prelude::SessionConfigExt;
+ use ballista_core::object_store::{CustomObjectStoreRegistry, S3Options};
use ballista_core::RuntimeProducer;
- use ballista_examples::object_store::{CustomObjectStoreRegistry,
S3Options};
use ballista_examples::test_util::examples_test_data;
+ use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::execution::SessionState;
use datafusion::prelude::SessionConfig;
use datafusion::{assert_batches_eq, prelude::SessionContext};
- use datafusion::{
- error::DataFusionError,
- execution::{
- runtime_env::{RuntimeConfig, RuntimeEnv},
- SessionStateBuilder,
- },
- };
+ use datafusion::{error::DataFusionError, execution::SessionStateBuilder};
use std::sync::Arc;
use testcontainers_modules::testcontainers::runners::AsyncRunner;
@@ -264,32 +264,15 @@ mod custom_s3_config {
//
// configuration producer registers user defined config extension
// S3Option with relevant S3 configuration
- let config_producer = Arc::new(|| {
- SessionConfig::new_with_ballista()
- .with_information_schema(true)
- .with_option_extension(S3Options::default())
- });
+ let config_producer =
+
Arc::new(ballista_core::object_store::session_config_with_s3_support);
// Setting up runtime producer
//
// Runtime producer creates object store registry
// which can create object store connecter based on
// S3Option configuration.
let runtime_producer: RuntimeProducer =
- Arc::new(|session_config: &SessionConfig| {
- let s3options = session_config
- .options()
- .extensions
- .get::<S3Options>()
- .ok_or(DataFusionError::Configuration(
- "S3 Options not set".to_string(),
- ))?;
-
- let config =
RuntimeConfig::new().with_object_store_registry(Arc::new(
- CustomObjectStoreRegistry::new(s3options.clone()),
- ));
-
- Ok(Arc::new(RuntimeEnv::try_new(config)?))
- });
+ Arc::new(ballista_core::object_store::runtime_env_with_s3_support);
// Session builder creates SessionState
//
@@ -297,7 +280,9 @@ mod custom_s3_config {
// producing same runtime environment, and providing same
// object store registry.
- let session_builder = Arc::new(produce_state);
+ let session_builder =
+
Arc::new(ballista_core::object_store::session_state_with_s3_support);
+
let state = session_builder(config_producer())?;
// setting up ballista cluster with new runtime, configuration, and
session state producers
@@ -367,6 +352,21 @@ mod custom_s3_config {
"+----+------------+---------------------+",
];
+ assert_batches_eq!(expected, &result);
+
+ let result = ctx
+ .sql("select max(id) as max, min(id) as min from written_table")
+ .await?
+ .collect()
+ .await?;
+ let expected = [
+ "+-----+-----+",
+ "| max | min |",
+ "+-----+-----+",
+ "| 7 | 0 |",
+ "+-----+-----+",
+ ];
+
assert_batches_eq!(expected, &result);
Ok(())
}
@@ -491,15 +491,16 @@ mod custom_s3_config {
"S3 Options not set".to_string(),
))?;
- let config = RuntimeConfig::new().with_object_store_registry(Arc::new(
- CustomObjectStoreRegistry::new(s3options.clone()),
- ));
-
- let runtime_env = RuntimeEnv::try_new(config)?;
+ let runtime_env = RuntimeEnvBuilder::new()
+
.with_object_store_registry(Arc::new(CustomObjectStoreRegistry::new(
+ s3options.clone(),
+ )))
+ .build()?;
Ok(SessionStateBuilder::new()
.with_runtime_env(runtime_env.into())
.with_config(session_config)
+ .with_default_features()
.build())
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
