This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 7c0c596c chore: Upgrade to DataFusion 43.0.0-rc1 (#1057)
7c0c596c is described below
commit 7c0c596c4ee42cbde68c09caa973834ad055876d
Author: Andy Grove <[email protected]>
AuthorDate: Tue Nov 5 13:23:02 2024 -0700
chore: Upgrade to DataFusion 43.0.0-rc1 (#1057)
* Use DataFusion 43.0.0-rc1
* Fix compilation issues
* clippy
* fix
* fix
* ignore failing test
---
native/Cargo.lock | 140 +++++++++++----------
native/Cargo.toml | 30 ++---
native/core/benches/aggregate.rs | 2 +-
native/core/benches/bloom_filter_agg.rs | 2 +-
.../datafusion/expressions/sum_decimal.rs | 2 +-
native/core/src/execution/datafusion/planner.rs | 6 +-
.../src/scalar_funcs/hash_expressions.rs | 8 +-
.../org/apache/comet/exec/CometJoinSuite.scala | 3 +-
8 files changed, 102 insertions(+), 91 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 39db7f9d..9799358c 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -114,9 +114,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a9ba0d7248932f4e2a12fb37f0a2e3ec82b3bdedbac2a1dce186e036843b8f8c"
+checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -135,9 +135,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d60afcdc004841a5c8d8da4f4fa22d64eb19c0c01ef4bcedd77f175a7cf6e38f"
+checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -150,9 +150,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da"
+checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8"
dependencies = [
"ahash",
"arrow-buffer",
@@ -167,9 +167,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80"
+checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec"
dependencies = [
"bytes",
"half",
@@ -178,9 +178,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4"
+checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -199,9 +199,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d3a4e4d63830a341713e35d9a42452fbc6241d5f42fa5cf6a4681b8ad91370c4"
+checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -218,9 +218,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634"
+checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -230,9 +230,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2"
+checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -245,9 +245,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b198b9c6fcf086501730efbbcb483317b39330a116125af7bb06467d04b352a3"
+checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -265,9 +265,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2427f37b4459a4b9e533045abe87a5183a5e0995a3fc2c2fd45027ae2cc4ef3f"
+checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -280,9 +280,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "15959657d92e2261a7a323517640af87f5afd9fd8a6492e424ebee2203c567f6"
+checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be"
dependencies = [
"ahash",
"arrow-array",
@@ -294,18 +294,18 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794"
+checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1"
dependencies = [
"bitflags 2.6.0",
]
[[package]]
name = "arrow-select"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a"
+checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125"
dependencies = [
"ahash",
"arrow-array",
@@ -317,9 +317,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7ab3db7c09dd826e74079661d84ed01ed06547cf75d52c2818ef776d0d852305"
+checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -805,8 +805,8 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -853,8 +853,8 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow-schema",
"async-trait",
@@ -949,8 +949,8 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -960,6 +960,7 @@ dependencies = [
"chrono",
"half",
"hashbrown",
+ "indexmap",
"instant",
"libc",
"num_cpus",
@@ -971,8 +972,8 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"log",
"tokio",
@@ -980,8 +981,8 @@ dependencies = [
[[package]]
name = "datafusion-execution"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"chrono",
@@ -1000,8 +1001,8 @@ dependencies = [
[[package]]
name = "datafusion-expr"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -1023,18 +1024,19 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"datafusion-common",
+ "itertools 0.13.0",
"paste",
]
[[package]]
name = "datafusion-functions"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"arrow-buffer",
@@ -1059,8 +1061,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -1079,8 +1081,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -1092,8 +1094,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"arrow-array",
@@ -1114,12 +1116,13 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"datafusion-common",
"datafusion-expr",
"datafusion-functions-window-common",
+ "datafusion-physical-expr",
"datafusion-physical-expr-common",
"log",
"paste",
@@ -1127,8 +1130,8 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -1136,8 +1139,8 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"async-trait",
@@ -1155,8 +1158,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -1182,8 +1185,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -1195,8 +1198,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"arrow-schema",
@@ -1210,8 +1213,8 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"ahash",
"arrow",
@@ -1244,14 +1247,15 @@ dependencies = [
[[package]]
name = "datafusion-sql"
-version = "42.0.0"
-source =
"git+https://github.com/apache/datafusion.git?rev=3b6aac2#3b6aac2fcecdb003427f9475f061ed2cc52e8558";
+version = "43.0.0"
+source =
"git+https://github.com/apache/datafusion.git?rev=43.0.0-rc1#88f58bf929167c5c5e2250ad87caa88d4dff11e5";
dependencies = [
"arrow",
"arrow-array",
"arrow-schema",
"datafusion-common",
"datafusion-expr",
+ "indexmap",
"log",
"regex",
"sqlparser",
@@ -2170,9 +2174,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "53.1.0"
+version = "53.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e"
+checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e"
dependencies = [
"ahash",
"bytes",
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 943d591d..85146afe 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -30,23 +30,23 @@ license = "Apache-2.0"
edition = "2021"
# Comet uses the same minimum Rust version as DataFusion
-rust-version = "1.75"
+rust-version = "1.79"
[workspace.dependencies]
-arrow = { version = "53.1.0", features = ["prettyprint", "ffi", "chrono-tz"] }
-arrow-array = { version = "53.1.0" }
-arrow-buffer = { version = "53.1.0" }
-arrow-data = { version = "53.1.0" }
-arrow-schema = { version = "53.1.0" }
-parquet = { version = "53.1.0", default-features = false, features =
["experimental"] }
-datafusion-common = { git = "https://github.com/apache/datafusion.git";, rev =
"3b6aac2" }
-datafusion = { git = "https://github.com/apache/datafusion.git";, rev =
"3b6aac2", default-features = false, features = ["unicode_expressions",
"crypto_expressions"] }
-datafusion-functions = { git = "https://github.com/apache/datafusion.git";, rev
= "3b6aac2", features = ["crypto_expressions"] }
-datafusion-functions-nested = { git =
"https://github.com/apache/datafusion.git";, rev = "3b6aac2", default-features =
false }
-datafusion-expr = { git = "https://github.com/apache/datafusion.git";, rev =
"3b6aac2", default-features = false }
-datafusion-execution = { git = "https://github.com/apache/datafusion.git";, rev
= "3b6aac2", default-features = false }
-datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git";,
rev = "3b6aac2", default-features = false }
-datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git";,
rev = "3b6aac2", default-features = false }
+arrow = { version = "53.2.0", features = ["prettyprint", "ffi", "chrono-tz"] }
+arrow-array = { version = "53.2.0" }
+arrow-buffer = { version = "53.2.0" }
+arrow-data = { version = "53.2.0" }
+arrow-schema = { version = "53.2.0" }
+parquet = { version = "53.2.0", default-features = false, features =
["experimental"] }
+datafusion-common = { git = "https://github.com/apache/datafusion.git";, rev =
"43.0.0-rc1" }
+datafusion = { git = "https://github.com/apache/datafusion.git";, rev =
"43.0.0-rc1", default-features = false, features = ["unicode_expressions",
"crypto_expressions"] }
+datafusion-functions = { git = "https://github.com/apache/datafusion.git";, rev
= "43.0.0-rc1", features = ["crypto_expressions"] }
+datafusion-functions-nested = { git =
"https://github.com/apache/datafusion.git";, rev = "43.0.0-rc1",
default-features = false }
+datafusion-expr = { git = "https://github.com/apache/datafusion.git";, rev =
"43.0.0-rc1", default-features = false }
+datafusion-execution = { git = "https://github.com/apache/datafusion.git";, rev
= "43.0.0-rc1", default-features = false }
+datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git";,
rev = "43.0.0-rc1", default-features = false }
+datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git";,
rev = "43.0.0-rc1", default-features = false }
datafusion-comet-spark-expr = { path = "spark-expr", version = "0.4.0" }
datafusion-comet-proto = { path = "proto", version = "0.4.0" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
diff --git a/native/core/benches/aggregate.rs b/native/core/benches/aggregate.rs
index 14425f76..c6209406 100644
--- a/native/core/benches/aggregate.rs
+++ b/native/core/benches/aggregate.rs
@@ -151,7 +151,7 @@ fn create_aggregate(
AggregateExec::try_new(
AggregateMode::Partial,
PhysicalGroupBy::new_single(vec![(c0, "c0".to_string())]),
- vec![aggr_expr],
+ vec![aggr_expr.into()],
vec![None], // no filter expressions
scan,
Arc::clone(schema),
diff --git a/native/core/benches/bloom_filter_agg.rs
b/native/core/benches/bloom_filter_agg.rs
index 90e3e3f6..af3eb919 100644
--- a/native/core/benches/bloom_filter_agg.rs
+++ b/native/core/benches/bloom_filter_agg.rs
@@ -121,7 +121,7 @@ fn create_aggregate(
AggregateExec::try_new(
mode,
PhysicalGroupBy::new_single(vec![]),
- vec![aggr_expr],
+ vec![aggr_expr.into()],
vec![None],
scan,
Arc::clone(schema),
diff --git a/native/core/src/execution/datafusion/expressions/sum_decimal.rs
b/native/core/src/execution/datafusion/expressions/sum_decimal.rs
index a3ce96b6..d885ff90 100644
--- a/native/core/src/execution/datafusion/expressions/sum_decimal.rs
+++ b/native/core/src/execution/datafusion/expressions/sum_decimal.rs
@@ -536,7 +536,7 @@ mod tests {
let aggregate = Arc::new(AggregateExec::try_new(
AggregateMode::Partial,
PhysicalGroupBy::new_single(vec![(c0, "c0".to_string())]),
- vec![aggr_expr],
+ vec![aggr_expr.into()],
vec![None], // no filter expressions
scan,
Arc::clone(&schema),
diff --git a/native/core/src/execution/datafusion/planner.rs
b/native/core/src/execution/datafusion/planner.rs
index e2ea3863..6f41bf0a 100644
--- a/native/core/src/execution/datafusion/planner.rs
+++ b/native/core/src/execution/datafusion/planner.rs
@@ -111,6 +111,7 @@ use datafusion_expr::{
};
use datafusion_physical_expr::expressions::{Literal, StatsType};
use datafusion_physical_expr::window::WindowExpr;
+use datafusion_physical_expr::LexOrdering;
use itertools::Itertools;
use jni::objects::GlobalRef;
use num::{BigInt, ToPrimitive};
@@ -873,11 +874,12 @@ impl PhysicalPlanner {
.collect();
let num_agg = agg.agg_exprs.len();
+ let aggr_expr = agg_exprs?.into_iter().map(Arc::new).collect();
let aggregate = Arc::new(
datafusion::physical_plan::aggregates::AggregateExec::try_new(
mode,
group_by,
- agg_exprs?,
+ aggr_expr,
vec![None; num_agg], // no filter expressions
Arc::clone(&child),
Arc::clone(&schema),
@@ -935,7 +937,7 @@ impl PhysicalPlanner {
Ok((
scans,
- Arc::new(SortExec::new(exprs?, child).with_fetch(fetch)),
+ Arc::new(SortExec::new(LexOrdering::new(exprs?),
child).with_fetch(fetch)),
))
}
OpStruct::Scan(scan) => {
diff --git a/native/spark-expr/src/scalar_funcs/hash_expressions.rs
b/native/spark-expr/src/scalar_funcs/hash_expressions.rs
index 33c8cc5e..af423677 100644
--- a/native/spark-expr/src/scalar_funcs/hash_expressions.rs
+++ b/native/spark-expr/src/scalar_funcs/hash_expressions.rs
@@ -18,7 +18,7 @@
use crate::scalar_funcs::hex::hex_strings;
use crate::spark_hash::{create_murmur3_hashes, create_xxhash64_hashes};
-use arrow_array::{ArrayRef, Int32Array, Int64Array, StringArray};
+use arrow_array::{Array, ArrayRef, Int32Array, Int64Array, StringArray};
use datafusion::functions::crypto::{sha224, sha256, sha384, sha512};
use datafusion_common::cast::as_binary_array;
use datafusion_common::{exec_err, internal_err, DataFusionError, ScalarValue};
@@ -139,7 +139,11 @@ fn wrap_digest_result_as_hex_string(
args: &[ColumnarValue],
digest: Arc<ScalarUDF>,
) -> Result<ColumnarValue, DataFusionError> {
- let value = digest.invoke(args)?;
+ let row_count = match &args[0] {
+ ColumnarValue::Array(array) => array.len(),
+ ColumnarValue::Scalar(_) => 1,
+ };
+ let value = digest.invoke_batch(args, row_count)?;
match value {
ColumnarValue::Array(array) => {
let binary_array = as_binary_array(&array)?;
diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala
b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala
index d787a9b1..ad1aef4a 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala
@@ -338,7 +338,8 @@ class CometJoinSuite extends CometTestBase {
}
}
- test("SortMergeJoin with join filter") {
+ // https://github.com/apache/datafusion-comet/issues/398
+ ignore("SortMergeJoin with join filter") {
withSQLConf(
CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key ->
"true",
SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
