This is an automated email from the ASF dual-hosted git repository.
jonah pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new a2811fc85d Update arrow/parquet to arrow/parquet `53.3.0` (#13508)
a2811fc85d is described below
commit a2811fc85d469c879e3d4db6ceb3fa13fbf263be
Author: Andrew Lamb <[email protected]>
AuthorDate: Thu Nov 21 00:49:55 2024 -0500
Update arrow/parquet to arrow/parquet `53.3.0` (#13508)
* Update arrow/parquet to arrow 53.3.0
* Update Cargo.lock
* fix ci test
Signed-off-by: jayzhan211 <[email protected]>
---------
Signed-off-by: jayzhan211 <[email protected]>
Co-authored-by: jayzhan211 <[email protected]>
---
Cargo.toml | 18 ++--
datafusion-cli/Cargo.lock | 100 ++++++++++-----------
.../test_files/string/string_literal.slt | 90 +++++++++----------
.../sqllogictest/test_files/string/string_view.slt | 13 ++-
4 files changed, 116 insertions(+), 105 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
index 0011539156..e947afff8f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -74,22 +74,22 @@ version = "43.0.0"
ahash = { version = "0.8", default-features = false, features = [
"runtime-rng",
] }
-arrow = { version = "53.2.0", features = [
+arrow = { version = "53.3.0", features = [
"prettyprint",
] }
-arrow-array = { version = "53.2.0", default-features = false, features = [
+arrow-array = { version = "53.3.0", default-features = false, features = [
"chrono-tz",
] }
-arrow-buffer = { version = "53.2.0", default-features = false }
-arrow-flight = { version = "53.2.0", features = [
+arrow-buffer = { version = "53.3.0", default-features = false }
+arrow-flight = { version = "53.3.0", features = [
"flight-sql-experimental",
] }
-arrow-ipc = { version = "53.2.0", default-features = false, features = [
+arrow-ipc = { version = "53.3.0", default-features = false, features = [
"lz4",
] }
-arrow-ord = { version = "53.2.0", default-features = false }
-arrow-schema = { version = "53.2.0", default-features = false }
-arrow-string = { version = "53.2.0", default-features = false }
+arrow-ord = { version = "53.3.0", default-features = false }
+arrow-schema = { version = "53.3.0", default-features = false }
+arrow-string = { version = "53.3.0", default-features = false }
async-trait = "0.1.73"
bigdecimal = "=0.4.1"
bytes = "1.4"
@@ -131,7 +131,7 @@ log = "^0.4"
num_cpus = "1.13.0"
object_store = { version = "0.11.0", default-features = false }
parking_lot = "0.12"
-parquet = { version = "53.2.0", default-features = false, features = [
+parquet = { version = "53.3.0", default-features = false, features = [
"arrow",
"async",
"object_store",
diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock
index c5576b7e7d..8afb096df5 100644
--- a/datafusion-cli/Cargo.lock
+++ b/datafusion-cli/Cargo.lock
@@ -173,9 +173,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3"
+checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -194,9 +194,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a"
+checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -209,9 +209,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8"
+checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f"
dependencies = [
"ahash",
"arrow-buffer",
@@ -220,15 +220,15 @@ dependencies = [
"chrono",
"chrono-tz",
"half",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.1",
"num",
]
[[package]]
name = "arrow-buffer"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec"
+checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16"
dependencies = [
"bytes",
"half",
@@ -237,9 +237,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1"
+checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -258,9 +258,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d"
+checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -277,9 +277,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5"
+checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -289,9 +289,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee"
+checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -304,9 +304,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c"
+checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -324,9 +324,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52"
+checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -339,9 +339,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be"
+checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff"
dependencies = [
"ahash",
"arrow-array",
@@ -353,15 +353,15 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1"
+checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678"
[[package]]
name = "arrow-select"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125"
+checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722"
dependencies = [
"ahash",
"arrow-array",
@@ -373,9 +373,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c"
+checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -1158,9 +1158,9 @@ dependencies = [
[[package]]
name = "ctor"
-version = "0.2.8"
+version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f"
+checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501"
dependencies = [
"quote",
"syn",
@@ -1934,9 +1934,9 @@ dependencies = [
[[package]]
name = "h2"
-version = "0.4.6"
+version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
+checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e"
dependencies = [
"atomic-waker",
"bytes",
@@ -2120,14 +2120,14 @@ dependencies = [
[[package]]
name = "hyper"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a"
+checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
- "h2 0.4.6",
+ "h2 0.4.7",
"http 1.1.0",
"http-body 1.0.1",
"httparse",
@@ -2162,7 +2162,7 @@ checksum =
"08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
dependencies = [
"futures-util",
"http 1.1.0",
- "hyper 1.5.0",
+ "hyper 1.5.1",
"hyper-util",
"rustls 0.23.17",
"rustls-native-certs 0.8.0",
@@ -2183,7 +2183,7 @@ dependencies = [
"futures-util",
"http 1.1.0",
"http-body 1.0.1",
- "hyper 1.5.0",
+ "hyper 1.5.1",
"pin-project-lite",
"socket2",
"tokio",
@@ -2392,9 +2392,9 @@ dependencies = [
[[package]]
name = "itoa"
-version = "1.0.11"
+version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2"
[[package]]
name = "jobserver"
@@ -2778,7 +2778,7 @@ dependencies = [
"chrono",
"futures",
"humantime",
- "hyper 1.5.0",
+ "hyper 1.5.1",
"itertools",
"md-5",
"parking_lot",
@@ -2855,9 +2855,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "53.2.0"
+version = "53.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e"
+checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191"
dependencies = [
"ahash",
"arrow-array",
@@ -2874,7 +2874,7 @@ dependencies = [
"flate2",
"futures",
"half",
- "hashbrown 0.14.5",
+ "hashbrown 0.15.1",
"lz4_flex",
"num",
"num-bigint",
@@ -3256,11 +3256,11 @@ dependencies = [
"bytes",
"futures-core",
"futures-util",
- "h2 0.4.6",
+ "h2 0.4.7",
"http 1.1.0",
"http-body 1.0.1",
"http-body-util",
- "hyper 1.5.0",
+ "hyper 1.5.1",
"hyper-rustls 0.27.3",
"hyper-util",
"ipnet",
@@ -3824,9 +3824,9 @@ dependencies = [
[[package]]
name = "sync_wrapper"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
@@ -4137,9 +4137,9 @@ checksum =
"42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "unicode-ident"
-version = "1.0.13"
+version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
+checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "unicode-segmentation"
diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt
b/datafusion/sqllogictest/test_files/string/string_literal.slt
index 493da64063..145081f91a 100644
--- a/datafusion/sqllogictest/test_files/string/string_literal.slt
+++ b/datafusion/sqllogictest/test_files/string/string_literal.slt
@@ -901,7 +901,7 @@ SELECT
'\' LIKE '\\',
'\\' LIKE '\\'
----
-false true false true false false true
+true false false true false true false
# if "%%" in the pattern was simplified to "%", the pattern semantics would
change
query BBBBB
@@ -1002,7 +1002,7 @@ NULL \%abc NULL
\ NULL NULL
\ (empty) false
\ \ true
-\ \\ false
+\ \\ true
\ \\\ false
\ \\\\ false
\ a false
@@ -1010,10 +1010,10 @@ NULL \%abc NULL
\ \\a false
\ % true
\ \% false
-\ \\% false
+\ \\% true
\ %% true
\ \%% false
-\ \\%% false
+\ \\%% true
\ _ true
\ \_ false
\ \\_ false
@@ -1028,21 +1028,21 @@ NULL \%abc NULL
\\ NULL NULL
\\ (empty) false
\\ \ false
-\\ \\ true
-\\ \\\ false
-\\ \\\\ false
+\\ \\ false
+\\ \\\ true
+\\ \\\\ true
\\ a false
\\ \a false
\\ \\a false
\\ % true
\\ \% false
-\\ \\% false
+\\ \\% true
\\ %% true
\\ \%% false
-\\ \\%% false
+\\ \\%% true
\\ _ false
\\ \_ false
-\\ \\_ false
+\\ \\_ true
\\ __ true
\\ \__ false
\\ \\__ false
@@ -1055,23 +1055,23 @@ NULL \%abc NULL
\\\ (empty) false
\\\ \ false
\\\ \\ false
-\\\ \\\ true
+\\\ \\\ false
\\\ \\\\ false
\\\ a false
\\\ \a false
\\\ \\a false
\\\ % true
\\\ \% false
-\\\ \\% false
+\\\ \\% true
\\\ %% true
\\\ \%% false
-\\\ \\%% false
+\\\ \\%% true
\\\ _ false
\\\ \_ false
\\\ \\_ false
\\\ __ false
\\\ \__ false
-\\\ \\__ false
+\\\ \\__ true
\\\ abc false
\\\ a_c false
\\\ a\_c false
@@ -1082,16 +1082,16 @@ NULL \%abc NULL
\\\\ \ false
\\\\ \\ false
\\\\ \\\ false
-\\\\ \\\\ true
+\\\\ \\\\ false
\\\\ a false
\\\\ \a false
\\\\ \\a false
\\\\ % true
\\\\ \% false
-\\\\ \\% false
+\\\\ \\% true
\\\\ %% true
\\\\ \%% false
-\\\\ \\%% false
+\\\\ \\%% true
\\\\ _ false
\\\\ \_ false
\\\\ \\_ false
@@ -1110,7 +1110,7 @@ a \\ false
a \\\ false
a \\\\ false
a a true
-a \a false
+a \a true
a \\a false
a % true
a \% false
@@ -1136,17 +1136,17 @@ a \%abc false
\a \\\ false
\a \\\\ false
\a a false
-\a \a true
-\a \\a false
+\a \a false
+\a \\a true
\a % true
\a \% false
-\a \\% false
+\a \\% true
\a %% true
\a \%% false
-\a \\%% false
+\a \\%% true
\a _ false
\a \_ false
-\a \\_ false
+\a \\_ true
\a __ true
\a \__ false
\a \\__ false
@@ -1163,19 +1163,19 @@ a \%abc false
\\a \\\\ false
\\a a false
\\a \a false
-\\a \\a true
+\\a \\a false
\\a % true
\\a \% false
-\\a \\% false
+\\a \\% true
\\a %% true
\\a \%% false
-\\a \\%% false
+\\a \\%% true
\\a _ false
\\a \_ false
\\a \\_ false
\\a __ false
\\a \__ false
-\\a \\__ false
+\\a \\__ true
\\a abc false
\\a a_c false
\\a a\_c false
@@ -1224,7 +1224,7 @@ a \%abc false
\% \\%% true
\% _ false
\% \_ false
-\% \\_ false
+\% \\_ true
\% __ true
\% \__ false
\% \\__ false
@@ -1244,16 +1244,16 @@ a \%abc false
\\% \\a false
\\% % true
\\% \% false
-\\% \\% false
+\\% \\% true
\\% %% true
\\% \%% false
-\\% \\%% false
+\\% \\%% true
\\% _ false
\\% \_ false
\\% \\_ false
\\% __ false
\\% \__ false
-\\% \\__ false
+\\% \\__ true
\\% abc false
\\% a_c false
\\% a\_c false
@@ -1296,7 +1296,7 @@ a \%abc false
\%% \\a false
\%% % true
\%% \% false
-\%% \\% false
+\%% \\% true
\%% %% true
\%% \%% false
\%% \\%% true
@@ -1305,7 +1305,7 @@ a \%abc false
\%% \\_ false
\%% __ false
\%% \__ false
-\%% \\__ false
+\%% \\__ true
\%% abc false
\%% a_c false
\%% a\_c false
@@ -1322,10 +1322,10 @@ a \%abc false
\\%% \\a false
\\%% % true
\\%% \% false
-\\%% \\% false
+\\%% \\% true
\\%% %% true
\\%% \%% false
-\\%% \\%% false
+\\%% \\%% true
\\%% _ false
\\%% \_ false
\\%% \\_ false
@@ -1374,10 +1374,10 @@ _ \%abc false
\_ \\a false
\_ % true
\_ \% false
-\_ \\% false
+\_ \\% true
\_ %% true
\_ \%% false
-\_ \\%% false
+\_ \\%% true
\_ _ false
\_ \_ false
\_ \\_ true
@@ -1400,16 +1400,16 @@ _ \%abc false
\\_ \\a false
\\_ % true
\\_ \% false
-\\_ \\% false
+\\_ \\% true
\\_ %% true
\\_ \%% false
-\\_ \\%% false
+\\_ \\%% true
\\_ _ false
\\_ \_ false
\\_ \\_ false
\\_ __ false
\\_ \__ false
-\\_ \\__ false
+\\_ \\__ true
\\_ abc false
\\_ a_c false
\\_ a\_c false
@@ -1452,10 +1452,10 @@ __ \%abc false
\__ \\a false
\__ % true
\__ \% false
-\__ \\% false
+\__ \\% true
\__ %% true
\__ \%% false
-\__ \\%% false
+\__ \\%% true
\__ _ false
\__ \_ false
\__ \\_ false
@@ -1478,10 +1478,10 @@ __ \%abc false
\\__ \\a false
\\__ % true
\\__ \% false
-\\__ \\% false
+\\__ \\% true
\\__ %% true
\\__ \%% false
-\\__ \\%% false
+\\__ \\%% true
\\__ _ false
\\__ \_ false
\\__ \\_ false
@@ -1608,7 +1608,7 @@ a\_c \%abc false
\%abc \\a false
\%abc % true
\%abc \% false
-\%abc \\% false
+\%abc \\% true
\%abc %% true
\%abc \%% false
\%abc \\%% true
diff --git a/datafusion/sqllogictest/test_files/string/string_view.slt
b/datafusion/sqllogictest/test_files/string/string_view.slt
index 5a08f3f544..aa41cbb811 100644
--- a/datafusion/sqllogictest/test_files/string/string_view.slt
+++ b/datafusion/sqllogictest/test_files/string/string_view.slt
@@ -39,8 +39,19 @@ drop table test_source
# TODO: Revisit this issue after upgrading to the arrow-rs version that
includes apache/arrow-rs#6671.
# see issue https://github.com/apache/datafusion/issues/13329
-query error DataFusion error: Arrow error: Compute error: bit_length not
supported for Utf8View
+query IIII
select bit_length(ascii_1), bit_length(ascii_2), bit_length(unicode_1),
bit_length(unicode_2) from test_basic_operator;
+----
+48 8 144 32
+72 72 176 176
+56 8 240 64
+88 88 104 256
+56 24 216 288
+0 8 0 0
+8 16 0 0
+8 16 0 0
+NULL 8 NULL NULL
+NULL 8 NULL 32
#
# common test for string-like functions and operators
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
