This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new bd4264985 deps: update to latest iceberg-rust to pick up
get_byte_ranges [iceberg] (#3635)
bd4264985 is described below
commit bd42649850151d1378c387113beb34816c959d28
Author: Matt Butrovich <[email protected]>
AuthorDate: Fri Mar 6 09:06:51 2026 -0500
deps: update to latest iceberg-rust to pick up get_byte_ranges [iceberg]
(#3635)
---
native/Cargo.lock | 364 ++++++++++++++++++---
native/Cargo.toml | 3 +-
native/core/Cargo.toml | 1 +
.../core/src/execution/operators/iceberg_scan.rs | 31 +-
4 files changed, 346 insertions(+), 53 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 78fa3fa12..5fe82f8a0 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -17,6 +17,17 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
+[[package]]
+name = "aes"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
+dependencies = [
+ "cfg-if",
+ "cipher",
+ "cpufeatures 0.2.17",
+]
+
[[package]]
name = "ahash"
version = "0.8.12"
@@ -547,9 +558,9 @@ checksum =
"c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-config"
-version = "1.8.14"
+version = "1.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2"
+checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -567,7 +578,7 @@ dependencies = [
"fastrand",
"hex",
"http 1.4.0",
- "ring",
+ "sha1",
"time",
"tokio",
"tracing",
@@ -577,9 +588,9 @@ dependencies = [
[[package]]
name = "aws-credential-types"
-version = "1.2.13"
+version = "1.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6d203b0bf2626dcba8665f5cd0871d7c2c0930223d6b6be9097592fea21242d0"
+checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -589,9 +600,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.16.0"
+version = "1.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d9a7b350e3bb1767102698302bc37256cbd48422809984b98d292c40e2579aa9"
+checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -599,9 +610,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.37.1"
+version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b092fe214090261288111db7a2b2c2118e5a7f30dc2569f1732c4069a6840549"
+checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
dependencies = [
"cc",
"cmake",
@@ -611,9 +622,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
-version = "1.7.1"
+version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ede2ddc593e6c8acc6ce3358c28d6677a6dc49b65ba4b37a2befe14a11297e75"
+checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -636,9 +647,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.95.0"
+version = "1.96.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "00c5ff27c6ba2cbd95e6e26e2e736676fdf6bcf96495b187733f521cfe4ce448"
+checksum = "f64a6eded248c6b453966e915d32aeddb48ea63ad17932682774eb026fbef5b1"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -660,9 +671,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.97.0"
+version = "1.98.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4d186f1e5a3694a188e5a0640b3115ccc6e084d104e16fd6ba968dca072ffef8"
+checksum = "db96d720d3c622fcbe08bae1c4b04a72ce6257d8b0584cb5418da00ae20a344f"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -684,9 +695,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.99.0"
+version = "1.100.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9acba7c62f3d4e2408fa998a3a8caacd8b9a5b5549cf36e2372fbdae329d5449"
+checksum = "fafbdda43b93f57f699c5dfe8328db590b967b8a820a13ccdd6687355dfcc7ca"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -709,9 +720,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
-version = "1.4.1"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "37411f8e0f4bea0c3ca0958ce7f18f6439db24d555dbd809787262cd00926aa9"
+checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
@@ -889,9 +900,9 @@ dependencies = [
[[package]]
name = "aws-types"
-version = "1.3.13"
+version = "1.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0470cc047657c6e286346bdf10a8719d26efd6a91626992e0e64481e44323e96"
+checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -943,6 +954,12 @@ dependencies = [
"vsimd",
]
+[[package]]
+name = "base64ct"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
+
[[package]]
name = "bigdecimal"
version = "0.4.10"
@@ -1049,6 +1066,15 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "block-padding"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
+dependencies = [
+ "generic-array",
+]
+
[[package]]
name = "blocking"
version = "1.6.2"
@@ -1167,6 +1193,15 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+[[package]]
+name = "cbc"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "cc"
version = "1.2.56"
@@ -1268,6 +1303,16 @@ dependencies = [
"half",
]
+[[package]]
+name = "cipher"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
+dependencies = [
+ "crypto-common",
+ "inout",
+]
+
[[package]]
name = "clang-sys"
version = "1.8.1"
@@ -1804,6 +1849,7 @@ dependencies = [
"hdrs",
"hex",
"iceberg",
+ "iceberg-storage-opendal",
"itertools 0.14.0",
"jni",
"lazy_static",
@@ -2474,6 +2520,17 @@ dependencies = [
"uuid",
]
+[[package]]
+name = "der"
+version = "0.7.10"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
+dependencies = [
+ "const-oid",
+ "pem-rfc7468",
+ "zeroize",
+]
+
[[package]]
name = "deranged"
version = "0.5.8"
@@ -2926,20 +2983,20 @@ dependencies = [
"cfg-if",
"js-sys",
"libc",
- "r-efi",
+ "r-efi 5.3.0",
"wasip2",
"wasm-bindgen",
]
[[package]]
name = "getrandom"
-version = "0.4.1"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec"
+checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
"libc",
- "r-efi",
+ "r-efi 6.0.0",
"rand_core 0.10.0",
"wasip2",
"wasip3",
@@ -3249,7 +3306,7 @@ dependencies = [
[[package]]
name = "iceberg"
version = "0.8.0"
-source =
"git+https://github.com/apache/iceberg-rust?rev=b24ab63#b24ab6310235f71907f4b6b6dc14a8e5d9291acc";
+source =
"git+https://github.com/apache/iceberg-rust?rev=854f66d#854f66d371a59d218498de72b02db7d3282f2643";
dependencies = [
"anyhow",
"apache-avro",
@@ -3279,11 +3336,9 @@ dependencies = [
"moka",
"murmur3",
"once_cell",
- "opendal",
"ordered-float 4.6.0",
"parquet",
"rand 0.8.5",
- "reqsign",
"reqwest",
"roaring",
"serde",
@@ -3301,6 +3356,24 @@ dependencies = [
"zstd",
]
+[[package]]
+name = "iceberg-storage-opendal"
+version = "0.8.0"
+source =
"git+https://github.com/apache/iceberg-rust?rev=854f66d#854f66d371a59d218498de72b02db7d3282f2643";
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "bytes",
+ "cfg-if",
+ "iceberg",
+ "opendal",
+ "reqsign",
+ "reqwest",
+ "serde",
+ "typetag",
+ "url",
+]
+
[[package]]
name = "icu_collections"
version = "2.1.1"
@@ -3456,6 +3529,16 @@ dependencies = [
"str_stack",
]
+[[package]]
+name = "inout"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
+dependencies = [
+ "block-padding",
+ "generic-array",
+]
+
[[package]]
name = "integer-encoding"
version = "3.0.4"
@@ -3473,9 +3556,9 @@ dependencies = [
[[package]]
name = "ipnet"
-version = "2.11.0"
+version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
+checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
@@ -3533,9 +3616,9 @@ dependencies = [
[[package]]
name = "jiff"
-version = "0.2.22"
+version = "0.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "819b44bc7c87d9117eb522f14d46e918add69ff12713c475946b0a29363ed1c2"
+checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359"
dependencies = [
"jiff-static",
"jiff-tzdb-platform",
@@ -3548,9 +3631,9 @@ dependencies = [
[[package]]
name = "jiff-static"
-version = "0.2.22"
+version = "0.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "470252db18ecc35fd766c0891b1e3ec6cbbcd62507e85276c01bf75d8e94d4a1"
+checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4"
dependencies = [
"proc-macro2",
"quote",
@@ -3559,9 +3642,9 @@ dependencies = [
[[package]]
name = "jiff-tzdb"
-version = "0.1.5"
+version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2"
+checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076"
[[package]]
name = "jiff-tzdb-platform"
@@ -3616,6 +3699,21 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "jsonwebtoken"
+version = "9.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde"
+dependencies = [
+ "base64",
+ "js-sys",
+ "pem",
+ "ring",
+ "serde",
+ "serde_json",
+ "simple_asn1",
+]
+
[[package]]
name = "kv-log-macro"
version = "1.0.7"
@@ -3630,6 +3728,9 @@ name = "lazy_static"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+dependencies = [
+ "spin 0.9.8",
+]
[[package]]
name = "lazycell"
@@ -4000,6 +4101,22 @@ dependencies = [
"serde",
]
+[[package]]
+name = "num-bigint-dig"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7"
+dependencies = [
+ "lazy_static",
+ "libm",
+ "num-integer",
+ "num-iter",
+ "num-traits",
+ "rand 0.8.5",
+ "smallvec",
+ "zeroize",
+]
+
[[package]]
name = "num-complex"
version = "0.4.6"
@@ -4331,12 +4448,41 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+[[package]]
+name = "pbkdf2"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
+dependencies = [
+ "digest",
+ "hmac",
+]
+
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+[[package]]
+name = "pem"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be"
+dependencies = [
+ "base64",
+ "serde_core",
+]
+
+[[package]]
+name = "pem-rfc7468"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412"
+dependencies = [
+ "base64ct",
+]
+
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -4416,6 +4562,44 @@ dependencies = [
"futures-io",
]
+[[package]]
+name = "pkcs1"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
+dependencies = [
+ "der",
+ "pkcs8",
+ "spki",
+]
+
+[[package]]
+name = "pkcs5"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6"
+dependencies = [
+ "aes",
+ "cbc",
+ "der",
+ "pbkdf2",
+ "scrypt",
+ "sha2",
+ "spki",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
+dependencies = [
+ "der",
+ "pkcs5",
+ "rand_core 0.6.4",
+ "spki",
+]
+
[[package]]
name = "pkg-config"
version = "0.3.32"
@@ -4510,7 +4694,7 @@ dependencies = [
"nix",
"once_cell",
"smallvec",
- "spin",
+ "spin 0.10.0",
"symbolic-demangle",
"tempfile",
"thiserror 2.0.18",
@@ -4711,9 +4895,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.44"
+version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
@@ -4724,6 +4908,12 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
+[[package]]
+name = "r-efi"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
+
[[package]]
name = "rand"
version = "0.8.5"
@@ -4752,7 +4942,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8"
dependencies = [
"chacha20",
- "getrandom 0.4.1",
+ "getrandom 0.4.2",
"rand_core 0.10.0",
]
@@ -4900,11 +5090,14 @@ dependencies = [
"hmac",
"home",
"http 1.4.0",
+ "jsonwebtoken",
"log",
+ "once_cell",
"percent-encoding",
"quick-xml 0.37.5",
"rand 0.8.5",
"reqwest",
+ "rsa",
"rust-ini",
"serde",
"serde_json",
@@ -4989,6 +5182,27 @@ dependencies = [
"byteorder",
]
+[[package]]
+name = "rsa"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d"
+dependencies = [
+ "const-oid",
+ "digest",
+ "num-bigint-dig",
+ "num-integer",
+ "num-traits",
+ "pkcs1",
+ "pkcs8",
+ "rand_core 0.6.4",
+ "sha2",
+ "signature",
+ "spki",
+ "subtle",
+ "zeroize",
+]
+
[[package]]
name = "rust-ini"
version = "0.21.3"
@@ -5122,6 +5336,15 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
+[[package]]
+name = "salsa20"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213"
+dependencies = [
+ "cipher",
+]
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -5170,6 +5393,17 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "scrypt"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f"
+dependencies = [
+ "pbkdf2",
+ "salsa20",
+ "sha2",
+]
+
[[package]]
name = "security-framework"
version = "3.7.0"
@@ -5382,6 +5616,16 @@ dependencies = [
"libc",
]
+[[package]]
+name = "signature"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
+dependencies = [
+ "digest",
+ "rand_core 0.6.4",
+]
+
[[package]]
name = "simd-adler32"
version = "0.3.8"
@@ -5394,6 +5638,18 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+[[package]]
+name = "simple_asn1"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d"
+dependencies = [
+ "num-bigint",
+ "num-traits",
+ "thiserror 2.0.18",
+ "time",
+]
+
[[package]]
name = "siphasher"
version = "1.0.2"
@@ -5428,6 +5684,12 @@ dependencies = [
"windows-sys 0.60.2",
]
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
[[package]]
name = "spin"
version = "0.10.0"
@@ -5437,6 +5699,16 @@ dependencies = [
"lock_api",
]
+[[package]]
+name = "spki"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
+dependencies = [
+ "base64ct",
+ "der",
+]
+
[[package]]
name = "sqlparser"
version = "0.59.0"
@@ -5581,7 +5853,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
dependencies = [
"fastrand",
- "getrandom 0.4.1",
+ "getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.61.2",
@@ -5756,9 +6028,9 @@ checksum =
"1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.49.0"
+version = "1.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86"
+checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
dependencies = [
"bytes",
"libc",
@@ -6032,11 +6304,11 @@ checksum =
"b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
-version = "1.21.0"
+version = "1.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb"
+checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
dependencies = [
- "getrandom 0.4.1",
+ "getrandom 0.4.2",
"js-sys",
"serde_core",
"wasm-bindgen",
diff --git a/native/Cargo.toml b/native/Cargo.toml
index d5a6aeabc..7e0f9d884 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -55,7 +55,8 @@ object_store = { version = "0.12.3", features = ["gcp",
"azure", "aws", "http"]
url = "2.2"
aws-config = "1.8.14"
aws-credential-types = "1.2.13"
-iceberg = { git = "https://github.com/apache/iceberg-rust";, rev = "b24ab63" }
+iceberg = { git = "https://github.com/apache/iceberg-rust";, rev = "854f66d" }
+iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust";,
rev = "854f66d", features = ["opendal-all"] }
[profile.release]
debug = true
diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml
index cbe397b12..37be0f282 100644
--- a/native/core/Cargo.toml
+++ b/native/core/Cargo.toml
@@ -80,6 +80,7 @@ object_store_opendal = {version = "0.55.0", optional = true}
hdfs-sys = {version = "0.3", optional = true, features = ["hdfs_3_3"]}
opendal = { version ="0.55.0", optional = true, features = ["services-hdfs"] }
iceberg = { workspace = true }
+iceberg-storage-opendal = { workspace = true }
serde_json = "1.0"
uuid = "1.21.0"
diff --git a/native/core/src/execution/operators/iceberg_scan.rs
b/native/core/src/execution/operators/iceberg_scan.rs
index 720a4c09a..b76e8d587 100644
--- a/native/core/src/execution/operators/iceberg_scan.rs
+++ b/native/core/src/execution/operators/iceberg_scan.rs
@@ -38,7 +38,8 @@ use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
};
use futures::{Stream, StreamExt, TryStreamExt};
-use iceberg::io::FileIO;
+use iceberg::io::{FileIO, FileIOBuilder, StorageFactory};
+use iceberg_storage_opendal::OpenDalStorageFactory;
use crate::execution::operators::ExecutionError;
use crate::parquet::parquet_support::SparkParquetOptions;
@@ -191,20 +192,38 @@ impl IcebergScanExec {
Ok(Box::pin(wrapped_stream))
}
+ fn storage_factory_for(path: &str) -> Result<Arc<dyn StorageFactory>,
DataFusionError> {
+ let scheme = if path.contains("://") {
+ path.split("://").next().unwrap_or("file")
+ } else {
+ "file"
+ };
+ match scheme {
+ "file" => Ok(Arc::new(OpenDalStorageFactory::Fs)),
+ "s3" | "s3a" => Ok(Arc::new(OpenDalStorageFactory::S3 {
+ configured_scheme: scheme.to_string(),
+ customized_credential_load: None,
+ })),
+ "gs" => Ok(Arc::new(OpenDalStorageFactory::Gcs)),
+ "oss" => Ok(Arc::new(OpenDalStorageFactory::Oss)),
+ _ => Err(DataFusionError::Execution(format!(
+ "Unsupported storage scheme: {scheme}"
+ ))),
+ }
+ }
+
fn load_file_io(
catalog_properties: &HashMap<String, String>,
metadata_location: &str,
) -> Result<FileIO, DataFusionError> {
- let mut file_io_builder = FileIO::from_path(metadata_location)
- .map_err(|e| DataFusionError::Execution(format!("Failed to create
FileIO: {}", e)))?;
+ let factory = Self::storage_factory_for(metadata_location)?;
+ let mut file_io_builder = FileIOBuilder::new(factory);
for (key, value) in catalog_properties {
file_io_builder = file_io_builder.with_prop(key, value);
}
- file_io_builder
- .build()
- .map_err(|e| DataFusionError::Execution(format!("Failed to build
FileIO: {}", e)))
+ Ok(file_io_builder.build())
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
