This is an automated email from the ASF dual-hosted git repository.
kazuyukitanimura pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 591f45a0 chore: Bump arrow-rs to 53.1.0 and datafusion (#1001)
591f45a0 is described below
commit 591f45a07e3e9132770737cad99d97bff17d72c4
Author: KAZUYUKI TANIMURA <[email protected]>
AuthorDate: Mon Oct 14 15:44:23 2024 -0700
chore: Bump arrow-rs to 53.1.0 and datafusion (#1001)
## Which issue does this PR close?
## Rationale for this change
Arrow-rs 53.1.0 includes performance improvements
## What changes are included in this PR?
Bumping arrow-rs to 53.1.0 and datafusion to a revision
## How are these changes tested?
existing tests
---
native/Cargo.lock | 179 ++++++++++-----------
native/Cargo.toml | 28 ++--
native/core/Cargo.toml | 2 +-
native/core/src/execution/jni_api.rs | 2 +-
native/core/src/execution/operators/filter.rs | 4 +-
.../core/src/parquet/util/test_common/page_util.rs | 2 +-
.../src/scalar_funcs/hash_expressions.rs | 14 +-
7 files changed, 112 insertions(+), 119 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index c9301c6e..86171890 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -114,9 +114,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba"
+checksum = "a9ba0d7248932f4e2a12fb37f0a2e3ec82b3bdedbac2a1dce186e036843b8f8c"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -135,9 +135,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a"
+checksum = "d60afcdc004841a5c8d8da4f4fa22d64eb19c0c01ef4bcedd77f175a7cf6e38f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -150,16 +150,16 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f"
+checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da"
dependencies = [
"ahash",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
- "chrono-tz 0.9.0",
+ "chrono-tz 0.10.0",
"half",
"hashbrown",
"num",
@@ -167,9 +167,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d"
+checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80"
dependencies = [
"bytes",
"half",
@@ -178,9 +178,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785"
+checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -199,9 +199,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740"
+checksum = "d3a4e4d63830a341713e35d9a42452fbc6241d5f42fa5cf6a4681b8ad91370c4"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -218,9 +218,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65"
+checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -230,9 +230,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a"
+checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -245,9 +245,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6"
+checksum = "b198b9c6fcf086501730efbbcb483317b39330a116125af7bb06467d04b352a3"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -265,9 +265,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b"
+checksum = "2427f37b4459a4b9e533045abe87a5183a5e0995a3fc2c2fd45027ae2cc4ef3f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -280,9 +280,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb"
+checksum = "15959657d92e2261a7a323517640af87f5afd9fd8a6492e424ebee2203c567f6"
dependencies = [
"ahash",
"arrow-array",
@@ -294,18 +294,18 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858"
+checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794"
dependencies = [
"bitflags 2.6.0",
]
[[package]]
name = "arrow-select"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba"
+checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a"
dependencies = [
"ahash",
"arrow-array",
@@ -317,9 +317,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "53.0.0"
+version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230"
+checksum = "7ab3db7c09dd826e74079661d84ed01ed06547cf75d52c2818ef776d0d852305"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -527,12 +527,12 @@ dependencies = [
[[package]]
name = "chrono-tz"
-version = "0.9.0"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb"
+checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6"
dependencies = [
"chrono",
- "chrono-tz-build 0.3.0",
+ "chrono-tz-build 0.4.0",
"phf",
]
@@ -549,12 +549,11 @@ dependencies = [
[[package]]
name = "chrono-tz-build"
-version = "0.3.0"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1"
+checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7"
dependencies = [
"parse-zoneinfo",
- "phf",
"phf_codegen",
]
@@ -807,8 +806,7 @@ dependencies = [
[[package]]
name = "datafusion"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ee907b081e45e1d14e1f327e89ef134f91fcebad0bfc2dc229fa9f6044379682"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -856,8 +854,7 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6c2b914f6e33c429af7d8696c72a47ed9225d7e2b82c747ebdfa2408ed53579f"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow-schema",
"async-trait",
@@ -953,8 +950,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3a84f8e76330c582a6b8ada0b2c599ca46cfe46b7585e458fc3f4092bc722a18"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -976,8 +972,7 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cf08cc30d92720d557df13bd5a5696213bd5ea0f38a866d8d85055d866fba774"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"log",
"tokio",
@@ -986,8 +981,7 @@ dependencies = [
[[package]]
name = "datafusion-execution"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "86bc4183d5c45b9f068a6f351678a0d1eb1225181424542bb75db18ec280b822"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow",
"chrono",
@@ -1007,8 +1001,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "202119ce58e4d103e37ae64aab40d4e574c97bdd2bea994bf307b175fcbfa74d"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -1018,7 +1011,9 @@ dependencies = [
"datafusion-common",
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
+ "datafusion-functions-window-common",
"datafusion-physical-expr-common",
+ "indexmap",
"paste",
"serde_json",
"sqlparser",
@@ -1029,8 +1024,7 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f8b181ce8569216abb01ef3294aa16c0a40d7d39350c2ff01ede00f167a535f2"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow",
"datafusion-common",
@@ -1040,8 +1034,7 @@ dependencies = [
[[package]]
name = "datafusion-functions"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6e4124b8066444e05a24472f852e94cf56546c0f4d92d00f018f207216902712"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow",
"arrow-buffer",
@@ -1067,8 +1060,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b94acdac235ea21810150a89751617ef2db7e32eba27f54be48a81bde2bfe119"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -1080,16 +1072,15 @@ dependencies = [
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"half",
+ "indexmap",
"log",
"paste",
- "sqlparser",
]
[[package]]
name = "datafusion-functions-aggregate-common"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5c9ea085bbf900bf16e2ca0f56fc56236b2e4f2e1a2cccb67bcd83c5ab4ad0ef"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -1102,8 +1093,7 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6c882e61665ed60c5ce9b061c1e587aeb8ae5ae4bcb5e5f2465139ab25328e0f"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow",
"arrow-array",
@@ -1125,20 +1115,29 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "98a354ce96df3ca6d025093adac9fd55ca09931c9b6f2630140721a95873fde4"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"datafusion-common",
"datafusion-expr",
+ "datafusion-functions-window-common",
"datafusion-physical-expr-common",
"log",
+ "paste",
+]
+
+[[package]]
+name = "datafusion-functions-window-common"
+version = "42.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+dependencies = [
+ "datafusion-common",
+ "datafusion-physical-expr-common",
]
[[package]]
name = "datafusion-optimizer"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "baf677c74fb7b5a1899ef52709e4a70fff3ed80bdfb4bbe495909810e83d5f39"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow",
"async-trait",
@@ -1157,8 +1156,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "30b077999f6eb6c43d6b25bc66332a3be2f693c382840f008dd763b8540f9530"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -1167,30 +1165,25 @@ dependencies = [
"arrow-ord",
"arrow-schema",
"arrow-string",
- "base64",
"chrono",
"datafusion-common",
- "datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
"hashbrown",
- "hex",
"indexmap",
"itertools 0.13.0",
"log",
"paste",
"petgraph",
- "regex",
]
[[package]]
name = "datafusion-physical-expr-common"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dce847f885c2b13bbe29f5c8b7948797131aa470af6e16d2a94f4428b4f4f1bd"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -1203,12 +1196,13 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d13238e3b9fdd62a4c18760bfef714bb990d1e1d3430e9f416aae4b3cfaa71af"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
+ "arrow",
"arrow-schema",
"datafusion-common",
"datafusion-execution",
+ "datafusion-expr-common",
"datafusion-physical-expr",
"datafusion-physical-plan",
"itertools 0.13.0",
@@ -1217,8 +1211,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "faba6f55a7eaf0241d07d12c2640de52742646b10f754485d5192bdfe2c9ceae"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"ahash",
"arrow",
@@ -1232,8 +1225,8 @@ dependencies = [
"datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
- "datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
+ "datafusion-functions-window-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"futures",
@@ -1252,8 +1245,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "42.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dad8d96a9b52e1aa24f9373696a815be828193efce7cb0bbd2140b6bb67d1819"
+source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
dependencies = [
"arrow",
"arrow-array",
@@ -1767,9 +1759,9 @@ checksum =
"bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "lexical-core"
-version = "0.8.5"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46"
+checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458"
dependencies = [
"lexical-parse-float",
"lexical-parse-integer",
@@ -1780,9 +1772,9 @@ dependencies = [
[[package]]
name = "lexical-parse-float"
-version = "0.8.5"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f"
+checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0"
dependencies = [
"lexical-parse-integer",
"lexical-util",
@@ -1791,9 +1783,9 @@ dependencies = [
[[package]]
name = "lexical-parse-integer"
-version = "0.8.6"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9"
+checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61"
dependencies = [
"lexical-util",
"static_assertions",
@@ -1801,18 +1793,18 @@ dependencies = [
[[package]]
name = "lexical-util"
-version = "0.8.5"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc"
+checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0"
dependencies = [
"static_assertions",
]
[[package]]
name = "lexical-write-float"
-version = "0.8.5"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862"
+checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809"
dependencies = [
"lexical-util",
"lexical-write-integer",
@@ -1821,9 +1813,9 @@ dependencies = [
[[package]]
name = "lexical-write-integer"
-version = "0.8.5"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446"
+checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162"
dependencies = [
"lexical-util",
"static_assertions",
@@ -2178,8 +2170,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "52.0.0"
-source =
"git+https://github.com/apache/arrow-rs.git?rev=0a4d8a1#0a4d8a14b58e45ef92e31541f0b51a5b25de5f10";
+version = "53.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e"
dependencies = [
"ahash",
"bytes",
@@ -2734,9 +2727,9 @@ checksum =
"1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
[[package]]
name = "sqlparser"
-version = "0.50.0"
+version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac"
+checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7"
dependencies = [
"log",
"sqlparser_derive",
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 87c988aa..943d591d 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -33,20 +33,20 @@ edition = "2021"
rust-version = "1.75"
[workspace.dependencies]
-arrow = { version = "53.0.0", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { version = "53.0.0" }
-arrow-buffer = { version = "53.0.0" }
-arrow-data = { version = "53.0.0" }
-arrow-schema = { version = "53.0.0" }
-parquet = { version = "53.0.0", default-features = false, features =
["experimental"] }
-datafusion-common = { version = "42.0.0" }
-datafusion = { default-features = false, version = "42.0.0", features =
["unicode_expressions", "crypto_expressions"] }
-datafusion-functions = { version = "42.0.0", features = ["crypto_expressions"]
}
-datafusion-functions-nested = { version = "42.0.0", default-features = false }
-datafusion-expr = { version = "42.0.0", default-features = false }
-datafusion-execution = { version = "42.0.0", default-features = false }
-datafusion-physical-plan = { version = "42.0.0", default-features = false }
-datafusion-physical-expr = { version = "42.0.0", default-features = false }
+arrow = { version = "53.1.0", features = ["prettyprint", "ffi", "chrono-tz"] }
+arrow-array = { version = "53.1.0" }
+arrow-buffer = { version = "53.1.0" }
+arrow-data = { version = "53.1.0" }
+arrow-schema = { version = "53.1.0" }
+parquet = { version = "53.1.0", default-features = false, features =
["experimental"] }
+datafusion-common = { git = "https://github.com/apache/datafusion.git";, rev =
"3b6aac2" }
+datafusion = { git = "https://github.com/apache/datafusion.git";, rev =
"3b6aac2", default-features = false, features = ["unicode_expressions",
"crypto_expressions"] }
+datafusion-functions = { git = "https://github.com/apache/datafusion.git";, rev
= "3b6aac2", features = ["crypto_expressions"] }
+datafusion-functions-nested = { git =
"https://github.com/apache/datafusion.git";, rev = "3b6aac2", default-features =
false }
+datafusion-expr = { git = "https://github.com/apache/datafusion.git";, rev =
"3b6aac2", default-features = false }
+datafusion-execution = { git = "https://github.com/apache/datafusion.git";, rev
= "3b6aac2", default-features = false }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git";,
rev = "3b6aac2", default-features = false }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git";,
rev = "3b6aac2", default-features = false }
datafusion-comet-spark-expr = { path = "spark-expr", version = "0.4.0" }
datafusion-comet-proto = { path = "proto", version = "0.4.0" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml
index 13f6b135..30dce574 100644
--- a/native/core/Cargo.toml
+++ b/native/core/Cargo.toml
@@ -40,7 +40,7 @@ arrow-array = { workspace = true }
arrow-buffer = { workspace = true }
arrow-data = { workspace = true }
arrow-schema = { workspace = true }
-parquet = { git = "https://github.com/apache/arrow-rs.git";, rev = "0a4d8a1",
default-features = false, features = ["experimental"] }
+parquet = { workspace = true, default-features = false, features =
["experimental"] }
half = { version = "2.4.1", default-features = false }
futures = "0.3.28"
mimalloc = { version = "*", default-features = false, optional = true }
diff --git a/native/core/src/execution/jni_api.rs
b/native/core/src/execution/jni_api.rs
index e45897a9..8347b794 100644
--- a/native/core/src/execution/jni_api.rs
+++ b/native/core/src/execution/jni_api.rs
@@ -241,7 +241,7 @@ fn prepare_datafusion_session_context(
session_config = session_config.set_str(key, value);
}
- let runtime = RuntimeEnv::new(rt_config).unwrap();
+ let runtime = RuntimeEnv::try_new(rt_config).unwrap();
let mut session_ctx = SessionContext::new_with_config_rt(session_config,
Arc::new(runtime));
diff --git a/native/core/src/execution/operators/filter.rs
b/native/core/src/execution/operators/filter.rs
index 6dfaf6bc..d9a54712 100644
--- a/native/core/src/execution/operators/filter.rs
+++ b/native/core/src/execution/operators/filter.rs
@@ -203,10 +203,10 @@ impl FilterExec {
ConstExpr::new(expr).with_across_partitions(true)
});
// this is for statistics
- eq_properties = eq_properties.add_constants(constants);
+ eq_properties = eq_properties.with_constants(constants);
// this is for logical constant (for example: a = '1', then a could be
marked as a constant)
// to do: how to deal with multiple situation to represent = (for
example c1 between 0 and 0)
- eq_properties =
eq_properties.add_constants(Self::extend_constants(input, predicate));
+ eq_properties =
eq_properties.with_constants(Self::extend_constants(input, predicate));
Ok(PlanProperties::new(
eq_properties,
input.output_partitioning().clone(), // Output Partitioning
diff --git a/native/core/src/parquet/util/test_common/page_util.rs
b/native/core/src/parquet/util/test_common/page_util.rs
index 6303943c..8fe5e65c 100644
--- a/native/core/src/parquet/util/test_common/page_util.rs
+++ b/native/core/src/parquet/util/test_common/page_util.rs
@@ -122,7 +122,7 @@ impl DataPageBuilder for DataPageBuilderImpl {
);
self.encoding = Some(encoding);
let mut encoder: Box<dyn Encoder<T>> =
- get_encoder::<T>(encoding).expect("get_encoder() should be OK");
+ get_encoder::<T>(encoding, &self.desc).expect("get_encoder()
should be OK");
encoder.put(values).expect("put() should be OK");
let encoded_values = encoder
.flush_buffer()
diff --git a/native/spark-expr/src/scalar_funcs/hash_expressions.rs
b/native/spark-expr/src/scalar_funcs/hash_expressions.rs
index e76314f4..33c8cc5e 100644
--- a/native/spark-expr/src/scalar_funcs/hash_expressions.rs
+++ b/native/spark-expr/src/scalar_funcs/hash_expressions.rs
@@ -22,7 +22,7 @@ use arrow_array::{ArrayRef, Int32Array, Int64Array,
StringArray};
use datafusion::functions::crypto::{sha224, sha256, sha384, sha512};
use datafusion_common::cast::as_binary_array;
use datafusion_common::{exec_err, internal_err, DataFusionError, ScalarValue};
-use datafusion_expr::{ColumnarValue, ScalarFunctionImplementation};
+use datafusion_expr::{ColumnarValue, ScalarUDF};
use std::sync::Arc;
/// Spark compatible murmur3 hash (just `hash` in Spark) in vectorized
execution fashion
@@ -115,31 +115,31 @@ pub fn spark_xxhash64(args: &[ColumnarValue]) ->
Result<ColumnarValue, DataFusio
/// `sha224` function that simulates Spark's `sha2` expression with bit width
224
pub fn spark_sha224(args: &[ColumnarValue]) -> Result<ColumnarValue,
DataFusionError> {
- wrap_digest_result_as_hex_string(args, sha224().fun())
+ wrap_digest_result_as_hex_string(args, sha224())
}
/// `sha256` function that simulates Spark's `sha2` expression with bit width
0 or 256
pub fn spark_sha256(args: &[ColumnarValue]) -> Result<ColumnarValue,
DataFusionError> {
- wrap_digest_result_as_hex_string(args, sha256().fun())
+ wrap_digest_result_as_hex_string(args, sha256())
}
/// `sha384` function that simulates Spark's `sha2` expression with bit width
384
pub fn spark_sha384(args: &[ColumnarValue]) -> Result<ColumnarValue,
DataFusionError> {
- wrap_digest_result_as_hex_string(args, sha384().fun())
+ wrap_digest_result_as_hex_string(args, sha384())
}
/// `sha512` function that simulates Spark's `sha2` expression with bit width
512
pub fn spark_sha512(args: &[ColumnarValue]) -> Result<ColumnarValue,
DataFusionError> {
- wrap_digest_result_as_hex_string(args, sha512().fun())
+ wrap_digest_result_as_hex_string(args, sha512())
}
// Spark requires hex string as the result of sha2 functions, we have to wrap
the
// result of digest functions as hex string
fn wrap_digest_result_as_hex_string(
args: &[ColumnarValue],
- digest: ScalarFunctionImplementation,
+ digest: Arc<ScalarUDF>,
) -> Result<ColumnarValue, DataFusionError> {
- let value = digest(args)?;
+ let value = digest.invoke(args)?;
match value {
ColumnarValue::Array(array) => {
let binary_array = as_binary_array(&array)?;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
