This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new b8df3f4db Add support for expm1 expression from datafusion-spark
(#1711)
b8df3f4db is described below
commit b8df3f4db4156ded1678cd8a60f77b2bd80272f8
Author: Andy Grove <[email protected]>
AuthorDate: Thu May 22 12:17:05 2025 -0600
Add support for expm1 expression from datafusion-spark (#1711)
---
native/Cargo.lock | 269 +++++++++++----------
native/Cargo.toml | 1 +
native/core/Cargo.toml | 1 +
native/core/src/execution/planner.rs | 9 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 5 +
.../org/apache/comet/CometExpressionSuite.scala | 17 ++
.../scala/org/apache/spark/sql/CometTestBase.scala | 4 +-
7 files changed, 170 insertions(+), 136 deletions(-)
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 824be7f32..e7340b96b 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -25,7 +25,7 @@ checksum =
"5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"const-random",
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
"once_cell",
"version_check",
"zerocopy",
@@ -301,7 +301,7 @@ version = "55.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"serde",
"serde_json",
]
@@ -439,9 +439,9 @@ checksum =
"bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
-version = "2.9.0"
+version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
+checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
[[package]]
name = "blake2"
@@ -527,9 +527,9 @@ checksum =
"37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.2.21"
+version = "1.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0"
+checksum = "5f4ac86a9e5bc1e2b3449ab9d7d3a6a405e3d1bb28d7b9be8614f55846ae3766"
dependencies = [
"jobserver",
"libc",
@@ -632,23 +632,23 @@ checksum =
"0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
dependencies = [
"glob",
"libc",
- "libloading 0.8.6",
+ "libloading 0.8.7",
]
[[package]]
name = "clap"
-version = "4.5.37"
+version = "4.5.38"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071"
+checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
-version = "4.5.37"
+version = "4.5.38"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2"
+checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120"
dependencies = [
"anstyle",
"clap_lex",
@@ -975,6 +975,7 @@ dependencies = [
"datafusion-comet-proto",
"datafusion-comet-spark-expr",
"datafusion-functions-nested",
+ "datafusion-spark",
"futures",
"hex",
"itertools 0.14.0",
@@ -1489,6 +1490,21 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "datafusion-spark"
+version = "47.0.0"
+source =
"git+https://github.com/apache/datafusion?rev=efd9587#efd9587b2f271dd54c31e19416f04810f42675ca";
+dependencies = [
+ "arrow",
+ "datafusion-catalog",
+ "datafusion-common",
+ "datafusion-execution",
+ "datafusion-expr",
+ "datafusion-functions",
+ "datafusion-macros",
+ "log",
+]
+
[[package]]
name = "datafusion-sql"
version = "47.0.0"
@@ -1586,9 +1602,9 @@ checksum =
"877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
-version = "0.3.11"
+version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
+checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18"
dependencies = [
"libc",
"windows-sys 0.59.0",
@@ -1624,7 +1640,7 @@ version = "25.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"rustc_version",
]
@@ -1802,9 +1818,9 @@ dependencies = [
[[package]]
name = "getrandom"
-version = "0.3.2"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
+checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
dependencies = [
"cfg-if",
"js-sys",
@@ -1990,9 +2006,9 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.11"
+version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2"
+checksum = "cf9f1e950e0d9d1d3c47184416723cf29c0d1f93bd8cccf37e4beb6b44f31710"
dependencies = [
"bytes",
"futures-channel",
@@ -2034,21 +2050,22 @@ dependencies = [
[[package]]
name = "icu_collections"
-version = "1.5.0"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
+checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47"
dependencies = [
"displaydoc",
+ "potential_utf",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
-name = "icu_locid"
-version = "1.5.0"
+name = "icu_locale_core"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
+checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a"
dependencies = [
"displaydoc",
"litemap",
@@ -2057,31 +2074,11 @@ dependencies = [
"zerovec",
]
-[[package]]
-name = "icu_locid_transform"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_locid_transform_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform_data"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d"
-
[[package]]
name = "icu_normalizer"
-version = "1.5.0"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
+checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979"
dependencies = [
"displaydoc",
"icu_collections",
@@ -2089,67 +2086,54 @@ dependencies = [
"icu_properties",
"icu_provider",
"smallvec",
- "utf16_iter",
- "utf8_iter",
- "write16",
"zerovec",
]
[[package]]
name = "icu_normalizer_data"
-version = "1.5.1"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7"
+checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3"
[[package]]
name = "icu_properties"
-version = "1.5.1"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
+checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b"
dependencies = [
"displaydoc",
"icu_collections",
- "icu_locid_transform",
+ "icu_locale_core",
"icu_properties_data",
"icu_provider",
- "tinystr",
+ "potential_utf",
+ "zerotrie",
"zerovec",
]
[[package]]
name = "icu_properties_data"
-version = "1.5.1"
+version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2"
+checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632"
[[package]]
name = "icu_provider"
-version = "1.5.0"
+version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
+checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af"
dependencies = [
"displaydoc",
- "icu_locid",
- "icu_provider_macros",
+ "icu_locale_core",
"stable_deref_trait",
"tinystr",
"writeable",
"yoke",
"zerofrom",
+ "zerotrie",
"zerovec",
]
-[[package]]
-name = "icu_provider_macros"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.101",
-]
-
[[package]]
name = "idna"
version = "1.0.3"
@@ -2163,9 +2147,9 @@ dependencies = [
[[package]]
name = "idna_adapter"
-version = "1.2.0"
+version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
+checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
dependencies = [
"icu_normalizer",
"icu_properties",
@@ -2285,7 +2269,7 @@ version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a"
dependencies = [
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
"libc",
]
@@ -2393,12 +2377,12 @@ dependencies = [
[[package]]
name = "libloading"
-version = "0.8.6"
+version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
+checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c"
dependencies = [
"cfg-if",
- "windows-targets 0.52.6",
+ "windows-targets 0.53.0",
]
[[package]]
@@ -2440,9 +2424,9 @@ checksum =
"cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "litemap"
-version = "0.7.5"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
+checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956"
[[package]]
name = "lock_api"
@@ -2497,6 +2481,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+
[[package]]
name = "lz4_flex"
version = "0.11.3"
@@ -2574,9 +2564,9 @@ dependencies = [
[[package]]
name = "multimap"
-version = "0.10.0"
+version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03"
+checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
[[package]]
name = "nix"
@@ -2694,9 +2684,9 @@ dependencies = [
[[package]]
name = "object_store"
-version = "0.12.0"
+version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e"
+checksum = "d94ac16b433c0ccf75326388c893d2835ab7457ea35ab8ba5d745c053ef5fa16"
dependencies = [
"async-trait",
"base64",
@@ -2714,7 +2704,7 @@ dependencies = [
"parking_lot",
"percent-encoding",
"quick-xml 0.37.5",
- "rand 0.8.5",
+ "rand 0.9.1",
"reqwest",
"ring",
"rustls-pemfile",
@@ -2726,6 +2716,8 @@ dependencies = [
"tracing",
"url",
"walkdir",
+ "wasm-bindgen-futures",
+ "web-time",
]
[[package]]
@@ -2947,6 +2939,15 @@ dependencies = [
"plotters-backend",
]
+[[package]]
+name = "potential_utf"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585"
+dependencies = [
+ "zerovec",
+]
+
[[package]]
name = "pprof"
version = "0.14.0"
@@ -3003,7 +3004,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"chrono",
"flate2",
"hex",
@@ -3017,7 +3018,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"chrono",
"hex",
]
@@ -3104,9 +3105,9 @@ dependencies = [
[[package]]
name = "quinn"
-version = "0.11.7"
+version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012"
+checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8"
dependencies = [
"bytes",
"cfg_aliases",
@@ -3124,12 +3125,13 @@ dependencies = [
[[package]]
name = "quinn-proto"
-version = "0.11.11"
+version = "0.11.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bcbafbbdbb0f638fe3f35f3c56739f77a8a1d070cb25603226c83339b391472b"
+checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e"
dependencies = [
"bytes",
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
+ "lru-slab",
"rand 0.9.1",
"ring",
"rustc-hash 2.1.1",
@@ -3227,7 +3229,7 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
dependencies = [
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
]
[[package]]
@@ -3276,7 +3278,7 @@ version = "0.5.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
]
[[package]]
@@ -3410,7 +3412,7 @@ version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"errno",
"libc",
"linux-raw-sys 0.4.15",
@@ -3423,7 +3425,7 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"errno",
"libc",
"linux-raw-sys 0.9.4",
@@ -3477,9 +3479,9 @@ dependencies = [
[[package]]
name = "rustls-webpki"
-version = "0.103.2"
+version = "0.103.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7149975849f1abb3832b246010ef62ccc80d3a76169517ada7188252b9cfb437"
+checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
dependencies = [
"ring",
"rustls-pki-types",
@@ -3528,7 +3530,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
"core-foundation",
"core-foundation-sys",
"libc",
@@ -3816,12 +3818,12 @@ dependencies = [
[[package]]
name = "tempfile"
-version = "3.19.1"
+version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
+checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
dependencies = [
"fastrand",
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
"once_cell",
"rustix 1.0.7",
"windows-sys 0.59.0",
@@ -3930,9 +3932,9 @@ dependencies = [
[[package]]
name = "tinystr"
-version = "0.7.6"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b"
dependencies = [
"displaydoc",
"zerovec",
@@ -4162,12 +4164,6 @@ dependencies = [
"percent-encoding",
]
-[[package]]
-name = "utf16_iter"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
-
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -4180,7 +4176,7 @@ version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9"
dependencies = [
- "getrandom 0.3.2",
+ "getrandom 0.3.3",
"js-sys",
"wasm-bindgen",
]
@@ -4374,15 +4370,15 @@ checksum =
"712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
-version = "0.61.0"
+version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980"
+checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link",
"windows-result",
- "windows-strings 0.4.0",
+ "windows-strings 0.4.2",
]
[[package]]
@@ -4426,9 +4422,9 @@ dependencies = [
[[package]]
name = "windows-result"
-version = "0.3.2"
+version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252"
+checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
dependencies = [
"windows-link",
]
@@ -4444,9 +4440,9 @@ dependencies = [
[[package]]
name = "windows-strings"
-version = "0.4.0"
+version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97"
+checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
dependencies = [
"windows-link",
]
@@ -4669,26 +4665,20 @@ version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
- "bitflags 2.9.0",
+ "bitflags 2.9.1",
]
-[[package]]
-name = "write16"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
-
[[package]]
name = "writeable"
-version = "0.5.5"
+version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
+checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb"
[[package]]
name = "yoke"
-version = "0.7.5"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
+checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc"
dependencies = [
"serde",
"stable_deref_trait",
@@ -4698,9 +4688,9 @@ dependencies = [
[[package]]
name = "yoke-derive"
-version = "0.7.5"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
+checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6"
dependencies = [
"proc-macro2",
"quote",
@@ -4755,11 +4745,22 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
+[[package]]
+name = "zerotrie"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595"
+dependencies = [
+ "displaydoc",
+ "yoke",
+ "zerofrom",
+]
+
[[package]]
name = "zerovec"
-version = "0.10.4"
+version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
+checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
dependencies = [
"yoke",
"zerofrom",
@@ -4768,9 +4769,9 @@ dependencies = [
[[package]]
name = "zerovec-derive"
-version = "0.10.3"
+version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
+checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f"
dependencies = [
"proc-macro2",
"quote",
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 8a674b121..00680b18f 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -39,6 +39,7 @@ async-trait = { version = "0.1" }
bytes = { version = "1.10.0" }
parquet = { version = "55.1.0", default-features = false, features =
["experimental"] }
datafusion = { git = "https://github.com/apache/datafusion";, rev = "efd9587",
default-features = false, features = ["unicode_expressions",
"crypto_expressions", "nested_expressions", "parquet"] }
+datafusion-spark = { git = "https://github.com/apache/datafusion";, rev =
"efd9587" }
datafusion-comet-spark-expr = { path = "spark-expr" }
datafusion-comet-proto = { path = "proto" }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml
index a598421d3..2a573a0b3 100644
--- a/native/core/Cargo.toml
+++ b/native/core/Cargo.toml
@@ -60,6 +60,7 @@ tempfile = "3.8.0"
itertools = "0.14.0"
paste = "1.0.14"
datafusion = { workspace = true }
+datafusion-spark = { workspace = true }
once_cell = "1.18.0"
regex = { workspace = true }
crc32fast = "1.3.2"
diff --git a/native/core/src/execution/planner.rs
b/native/core/src/execution/planner.rs
index bb5879d9d..c7ee749cf 100644
--- a/native/core/src/execution/planner.rs
+++ b/native/core/src/execution/planner.rs
@@ -78,7 +78,7 @@ use datafusion::common::{
use datafusion::datasource::listing::PartitionedFile;
use
datafusion::logical_expr::type_coercion::other::get_coerce_type_for_case_expression;
use datafusion::logical_expr::{
- AggregateUDF, ReturnFieldArgs, WindowFrame, WindowFrameBound,
WindowFrameUnits,
+ AggregateUDF, ReturnFieldArgs, ScalarUDF, WindowFrame, WindowFrameBound,
WindowFrameUnits,
WindowFunctionDefinition,
};
use datafusion::physical_expr::expressions::{Literal, StatsType};
@@ -108,6 +108,7 @@ use datafusion_comet_spark_expr::{
SparkCastOptions, StartsWith, Stddev, StringSpaceExpr, SubstringExpr,
SumDecimal,
TimestampTruncExpr, ToJson, UnboundColumn, Variance,
};
+use datafusion_spark::function::math::expm1::SparkExpm1;
use itertools::Itertools;
use jni::objects::GlobalRef;
use num::{BigInt, ToPrimitive};
@@ -146,6 +147,10 @@ pub struct PhysicalPlanner {
impl Default for PhysicalPlanner {
fn default() -> Self {
let session_ctx = Arc::new(SessionContext::new());
+
+ // register UDFs from datafusion-spark crate
+
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkExpm1::default()));
+
Self {
exec_context_id: TEST_EXEC_CONTEXT_ID,
session_ctx,
@@ -155,6 +160,8 @@ impl Default for PhysicalPlanner {
impl PhysicalPlanner {
pub fn new(session_ctx: Arc<SessionContext>) -> Self {
+ // register UDFs from datafusion-spark crate
+
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkExpm1::default()));
Self {
exec_context_id: TEST_EXEC_CONTEXT_ID,
session_ctx,
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index ad9683918..8f1971d44 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -1432,6 +1432,11 @@ object QueryPlanSerde extends Logging with CometExprShim
{
val optExpr = scalarFunctionExprToProto("ascii", childExpr)
optExprWithInfo(optExpr, expr, castExpr)
+ case Expm1(child) =>
+ val childExpr = exprToProtoInternal(child, inputs, binding)
+ val optExpr = scalarFunctionExprToProto("expm1", childExpr)
+ optExprWithInfo(optExpr, expr, child)
+
case s: StringDecode =>
// Right child is the encoding expression.
s.right match {
diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
index 43fbe57c5..e866e8d56 100644
--- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
@@ -1146,6 +1146,23 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
}
+ test("expm1") {
+ val testValues = Seq(
+ -1,
+ 0,
+ +1,
+ Double.MinValue,
+ Double.MaxValue,
+ Double.NaN,
+ Double.MinPositiveValue,
+ Double.PositiveInfinity,
+ Double.NegativeInfinity)
+ val testValuesRepeated = testValues.flatMap(v => Seq.fill(1000)(v))
+ withParquetTable(testValuesRepeated.map(n => (n, n)), "tbl") {
+ checkSparkAnswerWithTol("SELECT expm1(_1) FROM tbl")
+ }
+ }
+
// https://github.com/apache/datafusion-comet/issues/666
ignore("abs") {
Seq(true, false).foreach { dictionaryEnabled =>
diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
index fbfdbcda6..1695fc241 100644
--- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala
@@ -119,7 +119,9 @@ abstract class CometTestBase
actualAnswer.toSeq.zip(expectedAnswer.toSeq).foreach {
case (actual: Double, expected: Double) =>
- if (!actual.isNaN && !expected.isNaN) {
+ if (actual.isInfinity || expected.isInfinity) {
+ assert(actual.isInfinity == expected.isInfinity, s"actual answer
$actual != $expected")
+ } else if (!actual.isNaN && !expected.isNaN) {
assert(
math.abs(actual - expected) < absTol,
s"actual answer $actual not within $absTol of correct answer
$expected")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
