This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new de9e16b113 Update to apache-avro 0.17, fix compatibility changes
schema handling (#13727)
de9e16b113 is described below
commit de9e16b113419313516b08628836aa5ebe80fe15
Author: Marc Droogh <[email protected]>
AuthorDate: Thu Dec 12 15:04:27 2024 +0100
Update to apache-avro 0.17, fix compatibility changes schema handling
(#13727)
* Update apache-avro requirement from 0.16 to 0.17
---
updated-dependencies:
- dependency-name: apache-avro
dependency-type: direct:production
...
Signed-off-by: dependabot[bot] <[email protected]>
* Fix compatibility changes schema handling apache-avro 0.17
- Handle ArraySchema struct
- Handle MapSchema struct
- Map BigDecimal => LargeBinary
- Map TimestampNanos => Timestamp(TimeUnit::Nanosecond, None)
- Map LocalTimestampNanos => todo!()
- Add Default to FixedSchema test
* Update Cargo.lock file for apache-avro 0.17
---------
Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot]
<49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Marc Droogh <[email protected]>
Co-authored-by: Andrew Lamb <[email protected]>
---
datafusion-cli/Cargo.lock | 241 ++++++++++-----------
datafusion/common/Cargo.toml | 2 +-
datafusion/core/Cargo.toml | 2 +-
.../datasource/avro_to_arrow/arrow_array_reader.rs | 6 +-
.../core/src/datasource/avro_to_arrow/schema.rs | 14 +-
5 files changed, 128 insertions(+), 137 deletions(-)
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index 015bc1e0c3..67880c55c2 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -133,14 +133,14 @@ dependencies = [
[[package]]
name = "apache-avro"
-version = "0.16.0"
+version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ceb7c683b2f8f40970b70e39ff8be514c95b96fcb9c4af87e1ed2cb2e10801a0"
+checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13"
dependencies = [
+ "bigdecimal",
"bzip2",
"crc32fast",
"digest",
- "lazy_static",
"libflate",
"log",
"num-bigint",
@@ -148,15 +148,16 @@ dependencies = [
"rand",
"regex-lite",
"serde",
+ "serde_bytes",
"serde_json",
"snap",
- "strum 0.25.0",
- "strum_macros 0.25.3",
+ "strum",
+ "strum_macros",
"thiserror 1.0.69",
"typed-builder",
"uuid",
"xz2",
- "zstd 0.12.4",
+ "zstd",
]
[[package]]
@@ -418,8 +419,8 @@ dependencies = [
"pin-project-lite",
"tokio",
"xz2",
- "zstd 0.13.2",
- "zstd-safe 7.2.1",
+ "zstd",
+ "zstd-safe",
]
[[package]]
@@ -800,6 +801,20 @@ dependencies = [
"vsimd",
]
+[[package]]
+name = "bigdecimal"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c"
+dependencies = [
+ "autocfg",
+ "libm",
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+ "serde",
+]
+
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -926,9 +941,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.2"
+version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
+checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d"
dependencies = [
"jobserver",
"libc",
@@ -955,9 +970,9 @@ checksum =
"613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
-version = "0.4.38"
+version = "0.4.39"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
+checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
dependencies = [
"android-tzdata",
"iana-time-zone",
@@ -989,9 +1004,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.22"
+version = "4.5.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b"
+checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84"
dependencies = [
"clap_builder",
"clap_derive",
@@ -999,9 +1014,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.22"
+version = "4.5.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1"
+checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838"
dependencies = [
"anstream",
"anstyle",
@@ -1015,7 +1030,7 @@ version = "4.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"syn",
@@ -1023,9 +1038,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.7.3"
+version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "clipboard-win"
@@ -1048,8 +1063,8 @@ version = "7.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9"
dependencies = [
- "strum 0.26.3",
- "strum_macros 0.26.4",
+ "strum",
+ "strum_macros",
"unicode-width 0.2.0",
]
@@ -1254,7 +1269,7 @@ dependencies = [
"url",
"uuid",
"xz2",
- "zstd 0.13.2",
+ "zstd",
]
[[package]]
@@ -1375,8 +1390,8 @@ dependencies = [
"recursive",
"serde_json",
"sqlparser",
- "strum 0.26.3",
- "strum_macros 0.26.4",
+ "strum",
+ "strum_macros",
]
[[package]]
@@ -1761,9 +1776,9 @@ checksum =
"a5d9305ccc6942a704f4335694ecd3de2ea531b114ac2d51f5f843750787a92f"
[[package]]
name = "fastrand"
-version = "2.2.0"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fd-lock"
@@ -2021,12 +2036,6 @@ version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
-[[package]]
-name = "heck"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-
[[package]]
name = "heck"
version = "0.5.0"
@@ -2205,7 +2214,7 @@ dependencies = [
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
"tower-service",
]
@@ -2444,9 +2453,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.74"
+version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705"
+checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7"
dependencies = [
"once_cell",
"wasm-bindgen",
@@ -2460,9 +2469,9 @@ checksum =
"bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "lexical-core"
-version = "1.0.2"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458"
+checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958"
dependencies = [
"lexical-parse-float",
"lexical-parse-integer",
@@ -2473,9 +2482,9 @@ dependencies = [
[[package]]
name = "lexical-parse-float"
-version = "1.0.2"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0"
+checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2"
dependencies = [
"lexical-parse-integer",
"lexical-util",
@@ -2484,9 +2493,9 @@ dependencies = [
[[package]]
name = "lexical-parse-integer"
-version = "1.0.2"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61"
+checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e"
dependencies = [
"lexical-util",
"static_assertions",
@@ -2494,18 +2503,18 @@ dependencies = [
[[package]]
name = "lexical-util"
-version = "1.0.3"
+version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0"
+checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3"
dependencies = [
"static_assertions",
]
[[package]]
name = "lexical-write-float"
-version = "1.0.2"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809"
+checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd"
dependencies = [
"lexical-util",
"lexical-write-integer",
@@ -2514,9 +2523,9 @@ dependencies = [
[[package]]
name = "lexical-write-integer"
-version = "1.0.2"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162"
+checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978"
dependencies = [
"lexical-util",
"static_assertions",
@@ -2524,9 +2533,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.167"
+version = "0.2.168"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
+checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d"
[[package]]
name = "libflate"
@@ -2726,6 +2735,7 @@ checksum =
"a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
+ "serde",
]
[[package]]
@@ -2912,7 +2922,7 @@ dependencies = [
"thrift",
"tokio",
"twox-hash",
- "zstd 0.13.2",
+ "zstd",
"zstd-sys",
]
@@ -3104,7 +3114,7 @@ dependencies = [
"rustc-hash",
"rustls 0.23.19",
"socket2",
- "thiserror 2.0.4",
+ "thiserror 2.0.6",
"tokio",
"tracing",
]
@@ -3123,7 +3133,7 @@ dependencies = [
"rustls 0.23.19",
"rustls-pki-types",
"slab",
- "thiserror 2.0.4",
+ "thiserror 2.0.6",
"tinyvec",
"tracing",
"web-time",
@@ -3131,9 +3141,9 @@ dependencies = [
[[package]]
name = "quinn-udp"
-version = "0.5.7"
+version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da"
+checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527"
dependencies = [
"cfg_aliases 0.2.1",
"libc",
@@ -3307,7 +3317,7 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper",
"tokio",
- "tokio-rustls 0.26.0",
+ "tokio-rustls 0.26.1",
"tokio-util",
"tower-service",
"url",
@@ -3392,15 +3402,15 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.38.41"
+version = "0.38.42"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6"
+checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85"
dependencies = [
"bitflags 2.6.0",
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -3619,18 +3629,27 @@ checksum =
"a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
-version = "1.0.215"
+version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
+checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e"
dependencies = [
"serde_derive",
]
+[[package]]
+name = "serde_bytes"
+version = "0.11.15"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a"
+dependencies = [
+ "serde",
+]
+
[[package]]
name = "serde_derive"
-version = "1.0.215"
+version = "1.0.216"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
+checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e"
dependencies = [
"proc-macro2",
"quote",
@@ -3723,7 +3742,7 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"syn",
@@ -3803,32 +3822,13 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-[[package]]
-name = "strum"
-version = "0.25.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
-
[[package]]
name = "strum"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
dependencies = [
- "strum_macros 0.26.4",
-]
-
-[[package]]
-name = "strum_macros"
-version = "0.25.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
-dependencies = [
- "heck 0.4.1",
- "proc-macro2",
- "quote",
- "rustversion",
- "syn",
+ "strum_macros",
]
[[package]]
@@ -3837,7 +3837,7 @@ version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
dependencies = [
- "heck 0.5.0",
+ "heck",
"proc-macro2",
"quote",
"rustversion",
@@ -3911,11 +3911,11 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "2.0.4"
+version = "2.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490"
+checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47"
dependencies = [
- "thiserror-impl 2.0.4",
+ "thiserror-impl 2.0.6",
]
[[package]]
@@ -3931,9 +3931,9 @@ dependencies = [
[[package]]
name = "thiserror-impl"
-version = "2.0.4"
+version = "2.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061"
+checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312"
dependencies = [
"proc-macro2",
"quote",
@@ -4056,12 +4056,11 @@ dependencies = [
[[package]]
name = "tokio-rustls"
-version = "0.26.0"
+version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
dependencies = [
"rustls 0.23.19",
- "rustls-pki-types",
"tokio",
]
@@ -4150,18 +4149,18 @@ dependencies = [
[[package]]
name = "typed-builder"
-version = "0.16.2"
+version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16"
+checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600"
dependencies = [
"typed-builder-macro",
]
[[package]]
name = "typed-builder-macro"
-version = "0.16.2"
+version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e"
+checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8"
dependencies = [
"proc-macro2",
"quote",
@@ -4297,9 +4296,9 @@ checksum =
"9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
-version = "0.2.97"
+version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c"
+checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396"
dependencies = [
"cfg-if",
"once_cell",
@@ -4308,13 +4307,12 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.97"
+version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd"
+checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79"
dependencies = [
"bumpalo",
"log",
- "once_cell",
"proc-macro2",
"quote",
"syn",
@@ -4323,9 +4321,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.47"
+version = "0.4.49"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d"
+checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2"
dependencies = [
"cfg-if",
"js-sys",
@@ -4336,9 +4334,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.97"
+version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051"
+checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -4346,9 +4344,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.97"
+version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d"
+checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2"
dependencies = [
"proc-macro2",
"quote",
@@ -4359,9 +4357,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.97"
+version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49"
+checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6"
[[package]]
name = "wasm-streams"
@@ -4378,9 +4376,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.74"
+version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c"
+checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -4722,32 +4720,13 @@ dependencies = [
"syn",
]
-[[package]]
-name = "zstd"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c"
-dependencies = [
- "zstd-safe 6.0.6",
-]
-
[[package]]
name = "zstd"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
dependencies = [
- "zstd-safe 7.2.1",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "6.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581"
-dependencies = [
- "libc",
- "zstd-sys",
+ "zstd-safe",
]
[[package]]
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index d76848dfe9..50bf523d13 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -43,7 +43,7 @@ force_hash_collisions = []
[dependencies]
ahash = { workspace = true }
-apache-avro = { version = "0.16", default-features = false, features = [
+apache-avro = { version = "0.17", default-features = false, features = [
"bzip",
"snappy",
"xz",
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index 4706afc897..458af9ad21 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -77,7 +77,7 @@ unicode_expressions = [
]
[dependencies]
-apache-avro = { version = "0.16", optional = true }
+apache-avro = { version = "0.17", optional = true }
arrow = { workspace = true }
arrow-array = { workspace = true }
arrow-ipc = { workspace = true }
diff --git a/datafusion/core/src/datasource/avro_to_arrow/arrow_array_reader.rs
b/datafusion/core/src/datasource/avro_to_arrow/arrow_array_reader.rs
index f3358bce76..8f0e3792ff 100644
--- a/datafusion/core/src/datasource/avro_to_arrow/arrow_array_reader.rs
+++ b/datafusion/core/src/datasource/avro_to_arrow/arrow_array_reader.rs
@@ -138,7 +138,11 @@ impl<R: Read> AvroArrowArrayReader<'_, R> {
}
AvroSchema::Array(schema) => {
let sub_parent_field_name = format!("{}.element",
parent_field_name);
- Self::child_schema_lookup(&sub_parent_field_name, schema,
schema_lookup)?;
+ Self::child_schema_lookup(
+ &sub_parent_field_name,
+ &schema.items,
+ schema_lookup,
+ )?;
}
_ => (),
}
diff --git a/datafusion/core/src/datasource/avro_to_arrow/schema.rs
b/datafusion/core/src/datasource/avro_to_arrow/schema.rs
index 039a6aacc0..991f648e58 100644
--- a/datafusion/core/src/datasource/avro_to_arrow/schema.rs
+++ b/datafusion/core/src/datasource/avro_to_arrow/schema.rs
@@ -73,11 +73,15 @@ fn schema_to_field_with_props(
AvroSchema::Bytes => DataType::Binary,
AvroSchema::String => DataType::Utf8,
AvroSchema::Array(item_schema) => DataType::List(Arc::new(
- schema_to_field_with_props(item_schema, Some("element"), false,
None)?,
+ schema_to_field_with_props(&item_schema.items, Some("element"),
false, None)?,
)),
AvroSchema::Map(value_schema) => {
- let value_field =
- schema_to_field_with_props(value_schema, Some("value"), false,
None)?;
+ let value_field = schema_to_field_with_props(
+ &value_schema.types,
+ Some("value"),
+ false,
+ None,
+ )?;
DataType::Dictionary(
Box::new(DataType::Utf8),
Box::new(value_field.data_type().clone()),
@@ -144,14 +148,17 @@ fn schema_to_field_with_props(
AvroSchema::Decimal(DecimalSchema {
precision, scale, ..
}) => DataType::Decimal128(*precision as u8, *scale as i8),
+ AvroSchema::BigDecimal => DataType::LargeBinary,
AvroSchema::Uuid => DataType::FixedSizeBinary(16),
AvroSchema::Date => DataType::Date32,
AvroSchema::TimeMillis => DataType::Time32(TimeUnit::Millisecond),
AvroSchema::TimeMicros => DataType::Time64(TimeUnit::Microsecond),
AvroSchema::TimestampMillis =>
DataType::Timestamp(TimeUnit::Millisecond, None),
AvroSchema::TimestampMicros =>
DataType::Timestamp(TimeUnit::Microsecond, None),
+ AvroSchema::TimestampNanos =>
DataType::Timestamp(TimeUnit::Nanosecond, None),
AvroSchema::LocalTimestampMillis => todo!(),
AvroSchema::LocalTimestampMicros => todo!(),
+ AvroSchema::LocalTimestampNanos => todo!(),
AvroSchema::Duration => DataType::Duration(TimeUnit::Millisecond),
};
@@ -371,6 +378,7 @@ mod test {
aliases: Some(vec![alias("foofixed"), alias("barfixed")]),
size: 1,
doc: None,
+ default: None,
attributes: Default::default(),
});
let props = external_props(&fixed_schema);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
