This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 823a007577 Update to arrow/parquet `52.2.0` (#11691)
823a007577 is described below
commit 823a00757711993e12d3602543bf2da7ecfd7442
Author: Andrew Lamb <[email protected]>
AuthorDate: Sun Jul 28 19:56:46 2024 -0400
Update to arrow/parquet `52.2.0` (#11691)
* Update to arrow/parquet 52.2.0
* Update datafuion-cli Cargo.lock
* Update to avoid deprecated feature
---
Cargo.toml | 18 ++--
datafusion-cli/Cargo.lock | 97 +++++++++++-----------
datafusion-cli/Cargo.toml | 4 +-
.../core/src/datasource/file_format/parquet.rs | 2 +-
4 files changed, 60 insertions(+), 61 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index cdf3d2f93b..90aff3f715 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,22 +66,22 @@ version = "40.0.0"
ahash = { version = "0.8", default-features = false, features = [
"runtime-rng",
] }
-arrow = { version = "52.1.0", features = [
+arrow = { version = "52.2.0", features = [
"prettyprint",
] }
-arrow-array = { version = "52.1.0", default-features = false, features = [
+arrow-array = { version = "52.2.0", default-features = false, features = [
"chrono-tz",
] }
-arrow-buffer = { version = "52.1.0", default-features = false }
-arrow-flight = { version = "52.1.0", features = [
+arrow-buffer = { version = "52.2.0", default-features = false }
+arrow-flight = { version = "52.2.0", features = [
"flight-sql-experimental",
] }
-arrow-ipc = { version = "52.1.0", default-features = false, features = [
+arrow-ipc = { version = "52.2.0", default-features = false, features = [
"lz4",
] }
-arrow-ord = { version = "52.1.0", default-features = false }
-arrow-schema = { version = "52.1.0", default-features = false }
-arrow-string = { version = "52.1.0", default-features = false }
+arrow-ord = { version = "52.2.0", default-features = false }
+arrow-schema = { version = "52.2.0", default-features = false }
+arrow-string = { version = "52.2.0", default-features = false }
async-trait = "0.1.73"
bigdecimal = "=0.4.1"
bytes = "1.4"
@@ -118,7 +118,7 @@ log = "^0.4"
num_cpus = "1.13.0"
object_store = { version = "0.10.1", default-features = false }
parking_lot = "0.12"
-parquet = { version = "52.1.0", default-features = false, features = [
+parquet = { version = "52.2.0", default-features = false, features = [
"arrow",
"async",
"object_store",
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index 89ea13cf5b..e2851cfb40 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -84,9 +84,9 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.7"
+version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
[[package]]
name = "apache-avro"
@@ -130,9 +130,9 @@ checksum =
"96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
[[package]]
name = "arrow"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6127ea5e585a12ec9f742232442828ebaf264dfa5eefdd71282376c599562b77"
+checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -151,9 +151,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7add7f39210b7d726e2a8efc0083e7bf06e8f2d15bdb4896b564dce4410fbf5d"
+checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -166,9 +166,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "81c16ec702d3898c2f5cfdc148443c6cd7dbe5bac28399859eb0a3d38f072827"
+checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c"
dependencies = [
"ahash",
"arrow-buffer",
@@ -183,9 +183,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cae6970bab043c4fbc10aee1660ceb5b306d0c42c8cc5f6ae564efcd9759b663"
+checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1"
dependencies = [
"bytes",
"half",
@@ -194,9 +194,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1c7ef44f26ef4f8edc392a048324ed5d757ad09135eff6d5509e6450d39e0398"
+checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -215,9 +215,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5f843490bd258c5182b66e888161bb6f198f49f3792f7c7f98198b924ae0f564"
+checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -234,9 +234,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a769666ffac256dd301006faca1ca553d0ae7cffcf4cd07095f73f95eb226514"
+checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -246,9 +246,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dbf9c3fb57390a1af0b7bb3b5558c1ee1f63905f3eccf49ae7676a8d1e6e5a72"
+checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -261,9 +261,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "654e7f3724176b66ddfacba31af397c48e106fbe4d281c8144e7d237df5acfd7"
+checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -281,9 +281,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e8008370e624e8e3c68174faaf793540287106cfda8ad1da862fdc53d8e096b4"
+checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -296,9 +296,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ca5e3a6b7fda8d9fe03f3b18a2d946354ea7f3c8e4076dbdb502ad50d9d44824"
+checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c"
dependencies = [
"ahash",
"arrow-array",
@@ -306,20 +306,19 @@ dependencies = [
"arrow-data",
"arrow-schema",
"half",
- "hashbrown 0.14.5",
]
[[package]]
name = "arrow-schema"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dab1c12b40e29d9f3b699e0203c2a73ba558444c05e388a4377208f8f9c97eee"
+checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8"
[[package]]
name = "arrow-select"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e80159088ffe8c48965cb9b1a7c968b2729f29f37363df7eca177fc3281fe7c3"
+checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3"
dependencies = [
"ahash",
"arrow-array",
@@ -331,9 +330,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0fd04a6ea7de183648edbcb7a6dd925bbd04c210895f6384c780e27a9b54afcd"
+checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -348,9 +347,9 @@ dependencies = [
[[package]]
name = "assert_cmd"
-version = "2.0.14"
+version = "2.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8"
+checksum = "bc65048dd435533bb1baf2ed9956b9a278fbfdcf90301b39ee117f06c0199d37"
dependencies = [
"anstyle",
"bstr",
@@ -815,9 +814,9 @@ dependencies = [
[[package]]
name = "bstr"
-version = "1.9.1"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
+checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
dependencies = [
"memchr",
"regex-automata",
@@ -2621,9 +2620,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "52.1.0"
+version = "52.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0f22ba0d95db56dde8685e3fadcb915cdaadda31ab8abbe3ff7f0ad1ef333267"
+checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd"
dependencies = [
"ahash",
"arrow-array",
@@ -2776,9 +2775,9 @@ checksum =
"5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "predicates"
-version = "3.1.0"
+version = "3.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8"
+checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97"
dependencies = [
"anstyle",
"difflib",
@@ -2790,15 +2789,15 @@ dependencies = [
[[package]]
name = "predicates-core"
-version = "1.0.6"
+version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174"
+checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931"
[[package]]
name = "predicates-tree"
-version = "1.0.9"
+version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf"
+checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13"
dependencies = [
"predicates-core",
"termtree",
@@ -3703,9 +3702,9 @@ checksum =
"1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.39.1"
+version = "1.39.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a"
+checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
dependencies = [
"backtrace",
"bytes",
@@ -3958,9 +3957,9 @@ dependencies = [
[[package]]
name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "vsimd"
@@ -4370,9 +4369,9 @@ dependencies = [
[[package]]
name = "zstd-sys"
-version = "2.0.11+zstd.1.5.6"
+version = "2.0.12+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4"
+checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13"
dependencies = [
"cc",
"pkg-config",
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index 860dc123fa..ff6673dc97 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -30,7 +30,7 @@ rust-version = "1.76"
readme = "README.md"
[dependencies]
-arrow = { version = "52.1.0" }
+arrow = { version = "52.2.0" }
async-trait = "0.1.41"
aws-config = "0.55"
aws-credential-types = "0.55"
@@ -51,7 +51,7 @@ futures = "0.3"
mimalloc = { version = "0.1", default-features = false }
object_store = { version = "0.10.1", features = ["aws", "gcp", "http"] }
parking_lot = { version = "0.12" }
-parquet = { version = "52.1.0", default-features = false }
+parquet = { version = "52.2.0", default-features = false }
regex = "1.8"
rustyline = "11.0"
tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread",
"sync", "parking_lot", "signal"] }
diff --git a/datafusion/core/src/datasource/file_format/parquet.rs
b/datafusion/core/src/datasource/file_format/parquet.rs
index 3250b59fa1..b50e9389ad 100644
--- a/datafusion/core/src/datasource/file_format/parquet.rs
+++ b/datafusion/core/src/datasource/file_format/parquet.rs
@@ -1321,7 +1321,7 @@ mod tests {
.map(|i| i.to_string())
.collect();
let coll: Vec<_> = schema
- .all_fields()
+ .flattened_fields()
.into_iter()
.map(|i| i.name().to_string())
.collect();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
