This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 4f21e503a chore(deps): bump opendal to 0.56, migrate to reqsign-core
3.0 (#2401)
4f21e503a is described below
commit 4f21e503afc5363c10fafac917f0febe136c464e
Author: Krisztián Szűcs <[email protected]>
AuthorDate: Fri May 8 14:50:39 2026 +0200
chore(deps): bump opendal to 0.56, migrate to reqsign-core 3.0 (#2401)
## Summary
Bumps `opendal` from `0.55` to `0.56`, which is now published on
crates.io.
The 0.56 release made two breaking API changes that required code
updates:
- **`opendal::Scheme` was removed.** `OpenDalResolvingStorage` used it
as the HashMap key for caching per-scheme storage instances. Replaced
with `&'static str` (e.g. `"s3"`, `"gcs"`), with S3-family aliases
(`s3a`, `s3n`) and GCS aliases (`gs`) still collapsing to a single
entry.
- **reqsign was split into focused crates.** The old `reqsign 0.16`
monolith (with `AwsCredentialLoad` / `reqwest::Client`) was replaced by
`reqsign-aws-v4` + `reqsign-core` 3.0. `CustomAwsCredentialLoader` now
wraps any `ProvideCredential<Credential = AwsCredential>` implementation
instead of `Arc<dyn AwsCredentialLoad>`, which removes the `Arc`
wrapping at call sites.
---
Cargo.lock | 563 +++++++++++++++++++--
Cargo.toml | 2 +-
crates/storage/opendal/Cargo.toml | 7 +-
crates/storage/opendal/src/lib.rs | 6 +-
crates/storage/opendal/src/resolving.rs | 51 +-
crates/storage/opendal/src/s3.rs | 46 +-
crates/storage/opendal/tests/file_io_s3_test.rs | 29 +-
.../opendal/tests/resolving_storage_test.rs | 25 +-
8 files changed, 591 insertions(+), 138 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index c55c10240..350474c24 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -599,6 +599,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
dependencies = [
"aws-lc-sys",
+ "untrusted 0.7.1",
"zeroize",
]
@@ -1086,7 +1087,7 @@ version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
dependencies = [
- "darling 0.20.11",
+ "darling 0.23.0",
"ident_case",
"prettyplease",
"proc-macro2",
@@ -1324,7 +1325,17 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
]
[[package]]
@@ -1549,6 +1560,22 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "ctor"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "424e0138278faeb2b401f174ad17e715c829512d74f3d1e81eb43365c2e0590e"
+dependencies = [
+ "ctor-proc-macro",
+ "dtor",
+]
+
+[[package]]
+name = "ctor-proc-macro"
+version = "0.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
+
[[package]]
name = "ctr"
version = "0.9.2"
@@ -2518,7 +2545,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2559,6 +2586,21 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
+[[package]]
+name = "dtor"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "404d02eeb088a82cfd873006cb713fe411306c7d182c344905e101fb1167d301"
+dependencies = [
+ "dtor-proc-macro",
+]
+
+[[package]]
+name = "dtor-proc-macro"
+version = "0.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5"
+
[[package]]
name = "dunce"
version = "1.0.5"
@@ -2671,7 +2713,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3282,7 +3324,6 @@ dependencies = [
"tokio",
"tokio-rustls",
"tower-service",
- "webpki-roots 1.0.6",
]
[[package]]
@@ -3373,7 +3414,7 @@ dependencies = [
"pretty_assertions",
"rand 0.9.4",
"regex",
- "reqwest",
+ "reqwest 0.12.28",
"roaring",
"serde",
"serde_arrow",
@@ -3452,7 +3493,7 @@ dependencies = [
"iceberg-catalog-sql",
"iceberg-storage-opendal",
"iceberg_test_utils",
- "reqwest",
+ "reqwest 0.12.28",
"rstest",
"sqlx",
"tempfile",
@@ -3470,7 +3511,7 @@ dependencies = [
"iceberg_test_utils",
"itertools 0.13.0",
"mockito",
- "reqwest",
+ "reqwest 0.12.28",
"serde",
"serde_derive",
"serde_json",
@@ -3614,8 +3655,9 @@ dependencies = [
"iceberg",
"iceberg_test_utils",
"opendal",
- "reqsign",
- "reqwest",
+ "reqsign-aws-v4",
+ "reqsign-core",
+ "reqwest 0.12.28",
"serde",
"tokio",
"typetag",
@@ -3869,11 +3911,13 @@ checksum =
"1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
dependencies = [
"jiff-static",
"jiff-tzdb-platform",
+ "js-sys",
"log",
"portable-atomic",
"portable-atomic-util",
"serde_core",
- "windows-sys 0.59.0",
+ "wasm-bindgen",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3902,6 +3946,55 @@ dependencies = [
"jiff-tzdb",
]
+[[package]]
+name = "jni"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
+dependencies = [
+ "cfg-if",
+ "combine",
+ "jni-macros",
+ "jni-sys",
+ "log",
+ "simd_cesu8",
+ "thiserror 2.0.18",
+ "walkdir",
+ "windows-link",
+]
+
+[[package]]
+name = "jni-macros"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn",
+]
+
[[package]]
name = "jobserver"
version = "0.1.34"
@@ -3924,16 +4017,18 @@ dependencies = [
[[package]]
name = "jsonwebtoken"
-version = "9.3.1"
+version = "10.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
+checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1"
dependencies = [
+ "aws-lc-rs",
"base64",
+ "getrandom 0.2.17",
"js-sys",
"pem",
- "ring",
"serde",
"serde_json",
+ "signature",
"simple_asn1",
]
@@ -4176,6 +4271,15 @@ dependencies = [
"digest",
]
+[[package]]
+name = "mea"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6747f54621d156e1b47eb6b25f39a941b9fc347f98f67d25d8881ff99e8ed832"
+dependencies = [
+ "slab",
+]
+
[[package]]
name = "memchr"
version = "2.8.0"
@@ -4422,7 +4526,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -4552,7 +4656,7 @@ dependencies = [
"percent-encoding",
"quick-xml 0.39.2",
"rand 0.10.1",
- "reqwest",
+ "reqwest 0.12.28",
"ring",
"rustls-pki-types",
"serde",
@@ -4587,31 +4691,194 @@ checksum =
"c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "opendal"
-version = "0.55.0"
+version = "0.56.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d075ab8a203a6ab4bc1bce0a4b9fe486a72bf8b939037f4b78d95386384bc80a"
+checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236"
+dependencies = [
+ "ctor",
+ "opendal-core",
+ "opendal-layer-concurrent-limit",
+ "opendal-layer-logging",
+ "opendal-layer-retry",
+ "opendal-layer-timeout",
+ "opendal-service-azdls",
+ "opendal-service-fs",
+ "opendal-service-gcs",
+ "opendal-service-oss",
+ "opendal-service-s3",
+]
+
+[[package]]
+name = "opendal-core"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1849dd2687e173e776d3af5fce1ba3ae47b9dd37a09d1c4deba850ef45fe00ca"
dependencies = [
"anyhow",
- "backon",
"base64",
"bytes",
- "crc32c",
"futures",
- "getrandom 0.2.17",
"http 1.4.0",
"http-body 1.0.1",
"jiff",
"log",
"md-5",
+ "mea",
"percent-encoding",
"quick-xml 0.38.4",
- "reqsign",
- "reqwest",
+ "reqsign-core",
+ "reqwest 0.13.3",
"serde",
"serde_json",
"tokio",
"url",
"uuid",
+ "web-time",
+]
+
+[[package]]
+name = "opendal-layer-concurrent-limit"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "048b1b29c503263bdd80a9afe46a68cd02ea9bd361185b1feab4b151078998e9"
+dependencies = [
+ "futures",
+ "http 1.4.0",
+ "mea",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-logging"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d2645adc988b12eda106e2679ae529facfbbaa868ceb706f6f8125c6af15c47b"
+dependencies = [
+ "log",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-retry"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "4eac134ffa4ddda6131a640a84a5315996424b9416c85052f8c64c1a33b70ad4"
+dependencies = [
+ "backon",
+ "log",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-layer-timeout"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "619586ab7480c2e3009f6d18eabab18957bc094778fd130bcc38924970a90f4c"
+dependencies = [
+ "opendal-core",
+ "tokio",
+]
+
+[[package]]
+name = "opendal-service-azdls"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8f9884c2d8cf8ba2bb077d79c877dac5863ba3bab9e2c9c1e41a2e0491404772"
+dependencies = [
+ "bytes",
+ "http 1.4.0",
+ "log",
+ "opendal-core",
+ "opendal-service-azure-common",
+ "quick-xml 0.38.4",
+ "reqsign-azure-storage",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "opendal-service-azure-common"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "ffb0e45d6c8dcf66ce2da20e241bcb80e6e540e109a4ff20f318f6c9b4c54e0c"
+dependencies = [
+ "http 1.4.0",
+ "opendal-core",
+]
+
+[[package]]
+name = "opendal-service-fs"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "cf0be0417abeeb0053376d816b90fceb9ca98f20dfb54ebf1f2a282729f83663"
+dependencies = [
+ "bytes",
+ "log",
+ "opendal-core",
+ "serde",
+ "tokio",
+ "xattr",
+]
+
+[[package]]
+name = "opendal-service-gcs"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "70a49477a10163431896d106136117f5670717f9c9e49cf6f710528800c6633a"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "http 1.4.0",
+ "log",
+ "opendal-core",
+ "percent-encoding",
+ "quick-xml 0.38.4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "reqsign-google",
+ "serde",
+ "serde_json",
+ "tokio",
+]
+
+[[package]]
+name = "opendal-service-oss"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "29c8a917829ad06d21b639558532cb0101fe49b040d946d673a73018683fac05"
+dependencies = [
+ "bytes",
+ "http 1.4.0",
+ "log",
+ "opendal-core",
+ "quick-xml 0.38.4",
+ "reqsign-aliyun-oss",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+]
+
+[[package]]
+name = "opendal-service-s3"
+version = "0.56.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "9dadddeb9bb50b0d30927dd914c298c4ddca47e4c1cfa7674d311f0cf9b051c8"
+dependencies = [
+ "base64",
+ "bytes",
+ "crc32c",
+ "http 1.4.0",
+ "log",
+ "md-5",
+ "opendal-core",
+ "quick-xml 0.38.4",
+ "reqsign-aws-v4",
+ "reqsign-core",
+ "reqsign-file-read-tokio",
+ "serde",
+ "url",
]
[[package]]
@@ -5078,7 +5345,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck",
- "itertools 0.13.0",
+ "itertools 0.14.0",
"log",
"multimap",
"petgraph",
@@ -5097,7 +5364,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
- "itertools 0.13.0",
+ "itertools 0.14.0",
"proc-macro2",
"quote",
"syn",
@@ -5148,16 +5415,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
-[[package]]
-name = "quick-xml"
-version = "0.37.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb"
-dependencies = [
- "memchr",
- "serde",
-]
-
[[package]]
name = "quick-xml"
version = "0.38.4"
@@ -5204,6 +5461,7 @@ version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
dependencies = [
+ "aws-lc-rs",
"bytes",
"getrandom 0.3.4",
"lru-slab",
@@ -5230,7 +5488,7 @@ dependencies = [
"once_cell",
"socket2 0.6.3",
"tracing",
- "windows-sys 0.59.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -5478,33 +5736,114 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6"
[[package]]
-name = "reqsign"
-version = "0.16.5"
+name = "reqsign-aliyun-oss"
+version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "43451dbf3590a7590684c25fb8d12ecdcc90ed3ac123433e500447c7d77ed701"
+checksum = "57ac2757f3140aa2e213b554148ae0b52733e624fc6723f0cc6bb3d440176c95"
+dependencies = [
+ "anyhow",
+ "form_urlencoded",
+ "http 1.4.0",
+ "log",
+ "percent-encoding",
+ "reqsign-core",
+ "rust-ini",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "reqsign-aws-v4"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "44eaca382e94505a49f1a4849658d153aebf79d9c1a58e5dd3b10361511e9f43"
+dependencies = [
+ "anyhow",
+ "bytes",
+ "form_urlencoded",
+ "http 1.4.0",
+ "log",
+ "percent-encoding",
+ "quick-xml 0.39.2",
+ "reqsign-core",
+ "rust-ini",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sha1",
+]
+
+[[package]]
+name = "reqsign-azure-storage"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7a321980405d596bd34aaf95c4722a3de4128a67fd19e74a81a83aa3fdf082e6"
dependencies = [
"anyhow",
- "async-trait",
"base64",
- "chrono",
+ "bytes",
"form_urlencoded",
- "getrandom 0.2.17",
+ "http 1.4.0",
+ "jsonwebtoken",
+ "log",
+ "pem",
+ "percent-encoding",
+ "reqsign-core",
+ "rsa",
+ "serde",
+ "serde_json",
+ "sha1",
+]
+
+[[package]]
+name = "reqsign-core"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b10302cf0a7d7e7352ba211fc92c3c5bebf1286153e49cc5aa87348078a8e102"
+dependencies = [
+ "anyhow",
+ "base64",
+ "bytes",
+ "form_urlencoded",
+ "futures",
"hex",
"hmac",
- "home",
+ "http 1.4.0",
+ "jiff",
+ "log",
+ "percent-encoding",
+ "sha1",
+ "sha2",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "reqsign-file-read-tokio"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e2d89295b3d17abea31851cc8de55d843d89c52132c864963c38d41920613dc5"
+dependencies = [
+ "anyhow",
+ "reqsign-core",
+ "tokio",
+]
+
+[[package]]
+name = "reqsign-google"
+version = "3.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "35cc609b49c69e76ecaceb775a03f792d1ed3e7755ab3548d4534fd801e3242e"
+dependencies = [
+ "form_urlencoded",
"http 1.4.0",
"jsonwebtoken",
"log",
- "once_cell",
"percent-encoding",
- "quick-xml 0.37.5",
- "rand 0.8.5",
- "reqwest",
+ "reqsign-aws-v4",
+ "reqsign-core",
"rsa",
- "rust-ini",
"serde",
"serde_json",
- "sha1",
"sha2",
"tokio",
]
@@ -5547,9 +5886,46 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
- "wasm-streams",
+ "wasm-streams 0.4.2",
+ "web-sys",
+]
+
+[[package]]
+name = "reqwest"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "http 1.4.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pki-types",
+ "rustls-platform-verifier",
+ "sync_wrapper",
+ "tokio",
+ "tokio-rustls",
+ "tokio-util",
+ "tower",
+ "tower-http",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-streams 0.5.0",
"web-sys",
- "webpki-roots 1.0.6",
]
[[package]]
@@ -5562,7 +5938,7 @@ dependencies = [
"cfg-if",
"getrandom 0.2.17",
"libc",
- "untrusted",
+ "untrusted 0.9.0",
"windows-sys 0.52.0",
]
@@ -5693,7 +6069,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -5733,6 +6109,33 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rustls-platform-verifier"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
+dependencies = [
+ "core-foundation",
+ "core-foundation-sys",
+ "jni",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-platform-verifier-android",
+ "rustls-webpki",
+ "security-framework",
+ "security-framework-sys",
+ "webpki-root-certs",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls-platform-verifier-android"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
+
[[package]]
name = "rustls-webpki"
version = "0.103.13"
@@ -5742,7 +6145,7 @@ dependencies = [
"aws-lc-rs",
"ring",
"rustls-pki-types",
- "untrusted",
+ "untrusted 0.9.0",
]
[[package]]
@@ -6149,6 +6552,16 @@ version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+[[package]]
+name = "simd_cesu8"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
[[package]]
name = "simdutf8"
version = "0.1.5"
@@ -6656,7 +7069,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -7215,6 +7628,12 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
[[package]]
name = "untrusted"
version = "0.9.0"
@@ -7486,6 +7905,19 @@ dependencies = [
"web-sys",
]
+[[package]]
+name = "wasm-streams"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
[[package]]
name = "wasmparser"
version = "0.244.0"
@@ -7518,6 +7950,15 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "webpki-roots"
version = "0.26.11"
@@ -7552,7 +7993,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -7957,6 +8398,16 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
+[[package]]
+name = "xattr"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
+dependencies = [
+ "libc",
+ "rustix",
+]
+
[[package]]
name = "xmlparser"
version = "0.13.6"
diff --git a/Cargo.toml b/Cargo.toml
index 7f612c44b..793bb49d8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -103,7 +103,7 @@ mockito = "1"
motore-macros = "0.4.3"
murmur3 = "0.5.2"
once_cell = "1.20"
-opendal = "0.55.0"
+opendal = "0.56"
ordered-float = "4"
parquet = "58"
pilota = "0.11.10"
diff --git a/crates/storage/opendal/Cargo.toml
b/crates/storage/opendal/Cargo.toml
index 84f7e1147..80eeaa3d0 100644
--- a/crates/storage/opendal/Cargo.toml
+++ b/crates/storage/opendal/Cargo.toml
@@ -35,7 +35,7 @@ opendal-fs = ["opendal/services-fs"]
opendal-gcs = ["opendal/services-gcs"]
opendal-memory = ["opendal/services-memory"]
opendal-oss = ["opendal/services-oss"]
-opendal-s3 = ["opendal/services-s3", "reqsign"]
+opendal-s3 = ["opendal/services-s3", "reqsign-aws-v4", "reqsign-core"]
[dependencies]
anyhow = { workspace = true }
@@ -44,8 +44,8 @@ iceberg = { workspace = true }
opendal = { workspace = true }
async-trait = { workspace = true }
bytes = { workspace = true }
-reqsign = { version = "0.16.3", optional = true, default-features = false }
-reqwest = { workspace = true }
+reqsign-aws-v4 = { version = "3.0.0", optional = true }
+reqsign-core = { version = "3.0.0", optional = true }
serde = { workspace = true }
typetag = { workspace = true }
url = { workspace = true }
@@ -54,6 +54,5 @@ futures = { workspace = true }
[dev-dependencies]
async-trait = { workspace = true }
iceberg_test_utils = { path = "../../test_utils", features = ["tests"] }
-reqsign = { version = "0.16.3", default-features = false }
reqwest = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/crates/storage/opendal/src/lib.rs
b/crates/storage/opendal/src/lib.rs
index a0336868e..65deaa5f4 100644
--- a/crates/storage/opendal/src/lib.rs
+++ b/crates/storage/opendal/src/lib.rs
@@ -482,7 +482,11 @@ impl Storage for OpenDalStorage {
} else {
format!("{relative_path}/")
};
- Ok(op.remove_all(&path).await.map_err(from_opendal_error)?)
+ Ok(op
+ .delete_with(&path)
+ .recursive(true)
+ .await
+ .map_err(from_opendal_error)?)
}
async fn delete_stream(&self, mut paths: BoxStream<'static, String>) ->
Result<()> {
diff --git a/crates/storage/opendal/src/resolving.rs
b/crates/storage/opendal/src/resolving.rs
index 64a16b18d..621495519 100644
--- a/crates/storage/opendal/src/resolving.rs
+++ b/crates/storage/opendal/src/resolving.rs
@@ -30,7 +30,6 @@ use iceberg::io::{
StorageFactory,
};
use iceberg::{Error, ErrorKind, Result};
-use opendal::Scheme;
use serde::{Deserialize, Serialize};
use url::Url;
@@ -52,26 +51,24 @@ pub const SCHEME_ABFS: &str = "abfs";
pub const SCHEME_WASBS: &str = "wasbs";
pub const SCHEME_WASB: &str = "wasb";
-/// Parse a URL scheme string into an [`opendal::Scheme`].
-fn parse_scheme(scheme: &str) -> Result<Scheme> {
+/// Parse a URL scheme string.
+fn parse_scheme(scheme: &str) -> Result<&'static str> {
match scheme {
- SCHEME_MEMORY => Ok(Scheme::Memory),
- SCHEME_FILE | "" => Ok(Scheme::Fs),
- SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok(Scheme::S3),
- SCHEME_GS | SCHEME_GCS => Ok(Scheme::Gcs),
- SCHEME_OSS => Ok(Scheme::Oss),
- SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB =>
Ok(Scheme::Azdls),
- s => s.parse::<Scheme>().map_err(|e| {
- Error::new(
- ErrorKind::FeatureUnsupported,
- format!("Unsupported storage scheme: {s}: {e}"),
- )
- }),
+ SCHEME_MEMORY => Ok("memory"),
+ SCHEME_FILE | "" => Ok("file"),
+ SCHEME_S3 | SCHEME_S3A | SCHEME_S3N => Ok("s3"),
+ SCHEME_GS | SCHEME_GCS => Ok("gcs"),
+ SCHEME_OSS => Ok("oss"),
+ SCHEME_ABFSS | SCHEME_ABFS | SCHEME_WASBS | SCHEME_WASB => Ok("azdls"),
+ s => Err(Error::new(
+ ErrorKind::FeatureUnsupported,
+ format!("Unsupported storage scheme: {s}"),
+ )),
}
}
-/// Extract the [`Scheme`] family from a path URL.
-fn extract_scheme(path: &str) -> Result<Scheme> {
+/// Extract the scheme from a path URL.
+fn extract_scheme(path: &str) -> Result<&'static str> {
let url = Url::parse(path).map_err(|e| {
Error::new(
ErrorKind::DataInvalid,
@@ -83,13 +80,13 @@ fn extract_scheme(path: &str) -> Result<Scheme> {
/// Build an [`OpenDalStorage`] variant for the given scheme and config
properties.
fn build_storage_for_scheme(
- scheme: Scheme,
+ scheme: &'static str,
props: &HashMap<String, String>,
#[cfg(feature = "opendal-s3")] customized_credential_load:
&Option<CustomAwsCredentialLoader>,
) -> Result<OpenDalStorage> {
match scheme {
#[cfg(feature = "opendal-s3")]
- Scheme::S3 => {
+ "s3" => {
let config = crate::s3::s3_config_parse(props.clone())?;
Ok(OpenDalStorage::S3 {
config: Arc::new(config),
@@ -97,30 +94,30 @@ fn build_storage_for_scheme(
})
}
#[cfg(feature = "opendal-gcs")]
- Scheme::Gcs => {
+ "gcs" => {
let config = crate::gcs::gcs_config_parse(props.clone())?;
Ok(OpenDalStorage::Gcs {
config: Arc::new(config),
})
}
#[cfg(feature = "opendal-oss")]
- Scheme::Oss => {
+ "oss" => {
let config = crate::oss::oss_config_parse(props.clone())?;
Ok(OpenDalStorage::Oss {
config: Arc::new(config),
})
}
#[cfg(feature = "opendal-azdls")]
- Scheme::Azdls => {
+ "azdls" => {
let config = crate::azdls::azdls_config_parse(props.clone())?;
Ok(OpenDalStorage::Azdls {
config: Arc::new(config),
})
}
#[cfg(feature = "opendal-fs")]
- Scheme::Fs => Ok(OpenDalStorage::LocalFs),
+ "file" => Ok(OpenDalStorage::LocalFs),
#[cfg(feature = "opendal-memory")]
- Scheme::Memory =>
Ok(OpenDalStorage::Memory(crate::memory::memory_config_build()?)),
+ "memory" =>
Ok(OpenDalStorage::Memory(crate::memory::memory_config_build()?)),
unsupported => Err(Error::new(
ErrorKind::FeatureUnsupported,
format!("Unsupported storage scheme: {unsupported}"),
@@ -194,14 +191,14 @@ impl StorageFactory for OpenDalResolvingStorageFactory {
///
/// Sub-storages are lazily created on first use for each scheme and cached
/// for subsequent operations. Scheme aliases like `s3`/`s3a`/`s3n` map to
-/// the same [`Scheme`] variant, so they share a storage instance.
+/// the same canonical scheme, so they share a storage instance.
#[derive(Debug, Serialize, Deserialize)]
pub struct OpenDalResolvingStorage {
/// Configuration properties shared across all backends.
props: HashMap<String, String>,
/// Cache of scheme to storage mappings.
#[serde(skip, default)]
- storages: RwLock<HashMap<Scheme, Arc<OpenDalStorage>>>,
+ storages: RwLock<HashMap<&'static str, Arc<OpenDalStorage>>>,
/// Custom AWS credential loader for S3 storage.
#[cfg(feature = "opendal-s3")]
#[serde(skip)]
@@ -286,7 +283,7 @@ impl Storage for OpenDalResolvingStorage {
async fn delete_stream(&self, mut paths: BoxStream<'static, String>) ->
Result<()> {
// Group paths by scheme so each resolved storage receives a batch,
// avoiding repeated operator creation per path.
- let mut grouped: HashMap<Scheme, Vec<String>> = HashMap::new();
+ let mut grouped: HashMap<&'static str, Vec<String>> = HashMap::new();
while let Some(path) = paths.next().await {
let scheme = extract_scheme(&path)?;
grouped.entry(scheme).or_default().push(path);
diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs
index 2e2141860..75f7684cc 100644
--- a/crates/storage/opendal/src/s3.rs
+++ b/crates/storage/opendal/src/s3.rs
@@ -18,7 +18,6 @@
use std::collections::HashMap;
use std::sync::Arc;
-use async_trait::async_trait;
use iceberg::io::{
CLIENT_REGION, S3_ACCESS_KEY_ID, S3_ALLOW_ANONYMOUS, S3_ASSUME_ROLE_ARN,
S3_ASSUME_ROLE_EXTERNAL_ID, S3_ASSUME_ROLE_SESSION_NAME,
S3_DISABLE_CONFIG_LOAD,
@@ -28,8 +27,11 @@ use iceberg::io::{
use iceberg::{Error, ErrorKind, Result};
use opendal::services::S3Config;
use opendal::{Configurator, Operator};
-pub use reqsign::{AwsCredential, AwsCredentialLoad};
-use reqwest::Client;
+/// AWS credentials: access key ID, secret access key, and optional session
token.
+pub use reqsign_aws_v4::Credential as AwsCredential;
+/// Trait for types that can asynchronously supply [`AwsCredential`] to a
[`CustomAwsCredentialLoader`].
+pub use reqsign_core::ProvideCredential;
+use reqsign_core::{ProvideCredentialChain, ProvideCredentialDyn};
use url::Url;
use crate::utils::{from_opendal_error, is_truthy};
@@ -143,20 +145,26 @@ pub(crate) fn s3_config_build(
// Set bucket name.
.bucket(bucket);
- if let Some(customized_credential_load) = customized_credential_load {
- builder = builder
-
.customized_credential_load(customized_credential_load.clone().into_opendal_loader());
+ if let Some(loader) = customized_credential_load {
+ let chain = ProvideCredentialChain::new().push(Arc::clone(&loader.0));
+ builder = builder.credential_provider_chain(chain);
}
Ok(Operator::new(builder).map_err(from_opendal_error)?.finish())
}
/// Custom AWS credential loader.
-/// This can be used to load credentials from a custom source, such as the AWS
SDK.
///
-/// This should be set as an extension on `FileIOBuilder`.
-#[derive(Clone)]
-pub struct CustomAwsCredentialLoader(Arc<dyn AwsCredentialLoad>);
+/// Wraps any [`ProvideCredential`] implementation for use with the S3 storage
backend.
+/// Use [`CustomAwsCredentialLoader::new`] to create one, then pass it to
+/// [`OpenDalStorageFactory::S3`](crate::OpenDalStorageFactory).
+pub struct CustomAwsCredentialLoader(Arc<dyn ProvideCredentialDyn<Credential =
AwsCredential>>);
+
+impl Clone for CustomAwsCredentialLoader {
+ fn clone(&self) -> Self {
+ Self(Arc::clone(&self.0))
+ }
+}
impl std::fmt::Debug for CustomAwsCredentialLoader {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -166,21 +174,9 @@ impl std::fmt::Debug for CustomAwsCredentialLoader {
}
impl CustomAwsCredentialLoader {
- /// Create a new custom AWS credential loader.
- pub fn new(loader: Arc<dyn AwsCredentialLoad>) -> Self {
- Self(loader)
- }
-
- /// Convert this loader into an opendal compatible loader for customized
AWS credentials.
- pub fn into_opendal_loader(self) -> Box<dyn AwsCredentialLoad> {
- Box::new(self)
- }
-}
-
-#[async_trait]
-impl AwsCredentialLoad for CustomAwsCredentialLoader {
- async fn load_credential(&self, client: Client) ->
anyhow::Result<Option<AwsCredential>> {
- self.0.load_credential(client).await
+ /// Create a new custom AWS credential loader from any
[`ProvideCredential`] implementation.
+ pub fn new(provider: impl ProvideCredential<Credential = AwsCredential> +
'static) -> Self {
+ Self(Arc::new(provider) as Arc<dyn ProvideCredentialDyn<Credential =
AwsCredential>>)
}
}
diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs
b/crates/storage/opendal/tests/file_io_s3_test.rs
index d6dd8a3b4..d5858e18f 100644
--- a/crates/storage/opendal/tests/file_io_s3_test.rs
+++ b/crates/storage/opendal/tests/file_io_s3_test.rs
@@ -23,16 +23,16 @@
mod tests {
use std::sync::Arc;
- use async_trait::async_trait;
use futures::StreamExt;
use iceberg::io::{
FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT,
S3_PATH_STYLE_ACCESS, S3_REGION,
S3_SECRET_ACCESS_KEY,
};
- use iceberg_storage_opendal::{CustomAwsCredentialLoader,
OpenDalStorageFactory};
+ use iceberg_storage_opendal::{
+ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory,
ProvideCredential,
+ };
use iceberg_test_utils::{get_minio_endpoint,
normalize_test_name_with_parts, set_up};
- use reqsign::{AwsCredential, AwsCredentialLoad};
- use reqwest::Client;
+ use reqsign_core::Context;
async fn get_file_io() -> FileIO {
set_up();
@@ -99,6 +99,7 @@ mod tests {
}
// Mock credential loader for testing
+ #[derive(Debug)]
struct MockCredentialLoader {
credential: Option<AwsCredential>,
}
@@ -118,9 +119,13 @@ mod tests {
}
}
- #[async_trait]
- impl AwsCredentialLoad for MockCredentialLoader {
- async fn load_credential(&self, _client: Client) ->
anyhow::Result<Option<AwsCredential>> {
+ impl ProvideCredential for MockCredentialLoader {
+ type Credential = AwsCredential;
+
+ async fn provide_credential(
+ &self,
+ _ctx: &Context,
+ ) -> reqsign_core::Result<Option<AwsCredential>> {
Ok(self.credential.clone())
}
}
@@ -129,7 +134,7 @@ mod tests {
fn test_custom_aws_credential_loader_instantiation() {
// Test creating CustomAwsCredentialLoader with mock loader
let mock_loader = MockCredentialLoader::new_minio();
- let custom_loader =
CustomAwsCredentialLoader::new(Arc::new(mock_loader));
+ let custom_loader = CustomAwsCredentialLoader::new(mock_loader);
// Test that the loader can be used in FileIOBuilder with
OpenDalStorageFactory
let _builder = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 {
@@ -149,7 +154,7 @@ mod tests {
// Create a mock credential loader
let mock_loader = MockCredentialLoader::new_minio();
- let custom_loader =
CustomAwsCredentialLoader::new(Arc::new(mock_loader));
+ let custom_loader = CustomAwsCredentialLoader::new(mock_loader);
let minio_endpoint = get_minio_endpoint();
@@ -177,7 +182,7 @@ mod tests {
// Create a mock credential loader with no credentials
let mock_loader = MockCredentialLoader::new(None);
- let custom_loader =
CustomAwsCredentialLoader::new(Arc::new(mock_loader));
+ let custom_loader = CustomAwsCredentialLoader::new(mock_loader);
let minio_endpoint = get_minio_endpoint();
@@ -199,8 +204,8 @@ mod tests {
),
Err(e) => {
assert!(
- e.to_string()
- .contains("no valid credential found and anonymous
access is not allowed")
+ e.to_string().contains("failed to load signing
credential"),
+ "unexpected error: {e}"
);
}
}
diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs
b/crates/storage/opendal/tests/resolving_storage_test.rs
index c23508950..1853a796d 100644
--- a/crates/storage/opendal/tests/resolving_storage_test.rs
+++ b/crates/storage/opendal/tests/resolving_storage_test.rs
@@ -257,19 +257,21 @@ mod tests {
#[cfg(feature = "opendal-s3")]
#[tokio::test]
async fn test_with_custom_credential_loader() {
- use async_trait::async_trait;
- use iceberg_storage_opendal::CustomAwsCredentialLoader;
- use reqsign::{AwsCredential, AwsCredentialLoad};
- use reqwest::Client;
+ use iceberg_storage_opendal::{
+ AwsCredential, CustomAwsCredentialLoader, ProvideCredential,
+ };
+ use reqsign_core::Context;
+ #[derive(Debug)]
struct MinioCredentialLoader;
- #[async_trait]
- impl AwsCredentialLoad for MinioCredentialLoader {
- async fn load_credential(
+ impl ProvideCredential for MinioCredentialLoader {
+ type Credential = AwsCredential;
+
+ async fn provide_credential(
&self,
- _client: Client,
- ) -> anyhow::Result<Option<AwsCredential>> {
+ _ctx: &Context,
+ ) -> reqsign_core::Result<Option<AwsCredential>> {
Ok(Some(AwsCredential {
access_key_id: "admin".to_string(),
secret_access_key: "password".to_string(),
@@ -282,9 +284,8 @@ mod tests {
set_up();
let minio_endpoint = get_minio_endpoint();
- let factory =
OpenDalResolvingStorageFactory::new().with_s3_credential_loader(
- CustomAwsCredentialLoader::new(Arc::new(MinioCredentialLoader)),
- );
+ let factory = OpenDalResolvingStorageFactory::new()
+
.with_s3_credential_loader(CustomAwsCredentialLoader::new(MinioCredentialLoader));
let file_io = FileIOBuilder::new(Arc::new(factory))
.with_props(vec