This is an automated email from the ASF dual-hosted git repository.
comphead pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 7979527bee chore: use DF `array_repeat` and `array_compact` (#4741)
7979527bee is described below
commit 7979527bee394ee2bda24fb20b12b57408387a93
Author: Oleks V <[email protected]>
AuthorDate: Wed Jul 22 10:43:44 2026 -0700
chore: use DF `array_repeat` and `array_compact` (#4741)
---
.../expression-audits/array_funcs.md | 6 +-
native/Cargo.lock | 450 +++++++++++----------
native/core/src/execution/jni_api.rs | 7 +-
native/core/src/execution/planner.rs | 1 -
native/spark-expr/src/array_funcs/array_compact.rs | 165 --------
native/spark-expr/src/array_funcs/mod.rs | 2 -
native/spark-expr/src/comet_scalar_funcs.rs | 7 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 5 +-
.../main/scala/org/apache/comet/serde/arrays.scala | 54 +--
.../apache/comet/shims/Spark4xCometExprShim.scala | 28 +-
.../sql-tests/expressions/array/array_compact.sql | 113 +++++-
.../sql-tests/expressions/array/array_repeat.sql | 93 ++++-
12 files changed, 435 insertions(+), 496 deletions(-)
diff --git a/docs/source/contributor-guide/expression-audits/array_funcs.md
b/docs/source/contributor-guide/expression-audits/array_funcs.md
index a4c0741a6a..bd10f55c23 100644
--- a/docs/source/contributor-guide/expression-audits/array_funcs.md
+++ b/docs/source/contributor-guide/expression-audits/array_funcs.md
@@ -37,9 +37,9 @@
## array_compact
-- Spark 3.4.3 (audited 2026-05-27): `RuntimeReplaceable` -> `ArrayFilter(arr,
IsNotNull(lambda))`. Comet receives the rewritten form, dispatches through
`CometArrayFilter`, which delegates back to `CometArrayCompact.convert` for the
actual proto emission. The native path uses Comet's `spark_array_compact` UDF
rather than DataFusion's `array_remove_all` because DataFusion 53 changed
`array_remove_all`'s NULL semantics.
+- Spark 3.4.3 (audited 2026-05-27): `RuntimeReplaceable` -> `ArrayFilter(arr,
IsNotNull(lambda))`. Comet receives the rewritten form, dispatches through
`CometArrayFilter`, which emits a call to DataFusion's built-in `array_compact`
(from `datafusion-functions-nested`) via `CometScalarFunction("array_compact")`.
- Spark 3.5.8 (audited 2026-05-27): identical to 3.4.3.
-- Spark 4.0.1 (audited 2026-05-27): the replacement is wrapped in
`KnownNotContainsNull(...)` (analysis-only hint, no semantic change).
+- Spark 4.0.1 (audited 2026-05-27): the replacement is wrapped in
`KnownNotContainsNull(...)`. The 4.x `Spark4xCometExprShim` strips the wrapper
and emits the same `array_compact` call.
- Spark 4.1.1 (audited 2026-05-27): identical to 4.0.1.
## array_contains
@@ -128,7 +128,7 @@
## array_repeat
- Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8.
-- Spark 3.5.8 (audited 2026-05-27): baseline. `ArrayRepeat(left, right)
extends BinaryExpression with ExpectsInputTypes`; `inputTypes =
Seq(AnyDataType, IntegerType)`. NULL count yields NULL; count <= 0 yields empty
array; count > `MAX_ROUNDED_ARRAY_LENGTH` throws at runtime. Comet wraps the
call in `CaseWhen(IsNotNull(right), array_repeat(...), null)`.
+- Spark 3.5.8 (audited 2026-05-27): baseline. `ArrayRepeat(left, right)
extends BinaryExpression with ExpectsInputTypes`; `inputTypes =
Seq(AnyDataType, IntegerType)`. NULL count yields NULL; count <= 0 yields empty
array; count > `MAX_ROUNDED_ARRAY_LENGTH` throws at runtime. Wired as
`CometScalarFunction("array_repeat")` against `datafusion-spark`'s
`SparkArrayRepeat`, which returns NULL for NULL count and repeats NULL elements
(matching Spark).
- Spark 4.0.1 (audited 2026-05-27): error message uses
`createArrayWithElementsExceedLimitError(prettyName, count)`; semantics
unchanged.
- Spark 4.1.1 (audited 2026-05-27): identical to 4.0.1.
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 0b5d1dd615..29f13df281 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -172,9 +172,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.103"
+version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
[[package]]
name = "apache-avro"
@@ -198,7 +198,7 @@ dependencies = [
"snap",
"strum",
"strum_macros",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
"uuid",
"zstd",
]
@@ -419,7 +419,7 @@ version = "58.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"serde_core",
"serde_json",
]
@@ -598,13 +598,13 @@ checksum =
"8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
[[package]]
name = "async-trait"
-version = "0.1.89"
+version = "0.1.91"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
+checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
@@ -630,9 +630,9 @@ checksum =
"f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "aws-config"
-version = "1.9.0"
+version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
+checksum = "701418aa459dac33e50a0f8e818e5662a16bc018a6ac7423659b70f3799d67a8"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -673,9 +673,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.17.1"
+version = "1.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
+checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -683,9 +683,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.42.0"
+version = "0.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
+checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c"
dependencies = [
"cc",
"cmake",
@@ -696,9 +696,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
-version = "1.8.1"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d"
+checksum = "a6b50a43f3ccdf331521c6d6c68b7cc9668b6e09d439ebda9569df5722324d76"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -712,7 +712,7 @@ dependencies = [
"bytes-utils",
"fastrand",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"percent-encoding",
"pin-project-lite",
"tracing",
@@ -721,9 +721,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.103.0"
+version = "1.104.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401"
+checksum = "b53416d16c278234845392e38d93bd4481d2f09daa0f005a2277f0aa91f59c22"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -747,9 +747,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.105.0"
+version = "1.106.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a"
+checksum = "cc9b706c3305ed0285d5b1b696c747aa34950f830fb03e3e6c76890f99b9f188"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -773,9 +773,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.108.0"
+version = "1.109.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f"
+checksum = "32d214cdfa5bbe17f117e76a7643fadf32a5234fb597322ef8b1fb4b2f17dbbd"
dependencies = [
"arc-swap",
"aws-credential-types",
@@ -844,7 +844,7 @@ dependencies = [
"futures-core",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"percent-encoding",
"pin-project-lite",
@@ -898,11 +898,14 @@ dependencies = [
[[package]]
name = "aws-smithy-query"
-version = "0.61.1"
+version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77"
+checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62"
dependencies = [
+ "aws-smithy-runtime-api",
+ "aws-smithy-schema",
"aws-smithy-types",
+ "aws-smithy-xml",
"urlencoding",
]
@@ -924,7 +927,7 @@ dependencies = [
"http 0.2.12",
"http 1.4.2",
"http-body 0.4.6",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"pin-project-lite",
"pin-utils",
@@ -958,7 +961,7 @@ checksum =
"221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -984,7 +987,7 @@ dependencies = [
"http 0.2.12",
"http 1.4.2",
"http-body 0.4.6",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"itoa",
"num-integer",
@@ -997,9 +1000,9 @@ dependencies = [
[[package]]
name = "aws-smithy-xml"
-version = "0.61.1"
+version = "0.62.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957"
+checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-schema",
@@ -1009,9 +1012,9 @@ dependencies = [
[[package]]
name = "aws-types"
-version = "1.4.0"
+version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86"
+checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -1098,9 +1101,9 @@ checksum =
"bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
-version = "2.13.0"
+version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
+checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
[[package]]
name = "blake2"
@@ -1197,7 +1200,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustversion",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -1238,9 +1241,9 @@ checksum =
"72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
-version = "1.25.1"
+version = "1.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424"
+checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797"
[[package]]
name = "byteorder"
@@ -1290,9 +1293,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.67"
+version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38"
+checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1314,9 +1317,9 @@ checksum =
"9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
-version = "0.2.1"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
[[package]]
name = "chacha20"
@@ -1392,9 +1395,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.6.2"
+version = "4.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011"
+checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1414,14 +1417,14 @@ dependencies = [
[[package]]
name = "clap_derive"
-version = "4.6.1"
+version = "4.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
@@ -1718,9 +1721,9 @@ dependencies = [
[[package]]
name = "ctor"
-version = "1.0.8"
+version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fb22e947478ccf9dc44d8922042c677a63fbb88f2cb468521d1145816e5087cb"
+checksum = "e2e30e509674ef0ec91e21a7735766db37d163d46151b6a361d8b83dd79116bd"
dependencies = [
"link-section",
"linktime-proc-macro",
@@ -1775,7 +1778,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -1788,7 +1791,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -1799,7 +1802,7 @@ checksum =
"fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core 0.20.11",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -1810,7 +1813,7 @@ checksum =
"ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core 0.23.0",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -1987,7 +1990,7 @@ dependencies = [
"datafusion",
"serde",
"serde_json",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
]
[[package]]
@@ -2005,7 +2008,7 @@ dependencies = [
"paste",
"prost",
"regex",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
]
[[package]]
@@ -2445,7 +2448,7 @@ checksum =
"1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb"
dependencies = [
"datafusion-doc",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -2677,7 +2680,7 @@ dependencies = [
"defmt-parser",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -2686,7 +2689,7 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e"
dependencies = [
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
]
[[package]]
@@ -2727,7 +2730,7 @@ dependencies = [
"darling 0.20.11",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -2737,7 +2740,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -2758,7 +2761,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustc_version",
- "syn",
+ "syn 2.0.119",
"unicode-xid",
]
@@ -2800,7 +2803,7 @@ checksum =
"1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -2853,7 +2856,7 @@ checksum =
"44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -2934,9 +2937,9 @@ dependencies = [
[[package]]
name = "fastrand"
-version = "2.4.1"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
+checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
[[package]]
name = "find-msvc-tools"
@@ -2968,7 +2971,7 @@ version = "25.12.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"rustc_version",
]
@@ -3018,9 +3021,9 @@ checksum =
"42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futures"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
+checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218"
dependencies = [
"futures-channel",
"futures-core",
@@ -3033,9 +3036,9 @@ dependencies = [
[[package]]
name = "futures-channel"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
+checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae"
dependencies = [
"futures-core",
"futures-sink",
@@ -3043,15 +3046,15 @@ dependencies = [
[[package]]
name = "futures-core"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7"
[[package]]
name = "futures-executor"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
+checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458"
dependencies = [
"futures-core",
"futures-task",
@@ -3060,9 +3063,9 @@ dependencies = [
[[package]]
name = "futures-io"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
+checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a"
[[package]]
name = "futures-lite"
@@ -3079,32 +3082,32 @@ dependencies = [
[[package]]
name = "futures-macro"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
+checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
name = "futures-sink"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
+checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307"
[[package]]
name = "futures-task"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
+checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109"
[[package]]
name = "futures-util"
-version = "0.3.32"
+version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
+checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa"
dependencies = [
"futures-channel",
"futures-core",
@@ -3184,9 +3187,9 @@ checksum =
"e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
[[package]]
name = "glob"
-version = "0.3.3"
+version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "gloo-timers"
@@ -3357,9 +3360,9 @@ dependencies = [
[[package]]
name = "http-body"
-version = "1.0.1"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
dependencies = [
"bytes",
"http 1.4.2",
@@ -3367,14 +3370,14 @@ dependencies = [
[[package]]
name = "http-body-util"
-version = "0.1.3"
+version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2"
dependencies = [
"bytes",
"futures-core",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"pin-project-lite",
]
@@ -3401,9 +3404,9 @@ dependencies = [
[[package]]
name = "hyper"
-version = "1.10.1"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
+checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
dependencies = [
"atomic-waker",
"bytes",
@@ -3411,7 +3414,7 @@ dependencies = [
"futures-core",
"h2",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"httparse",
"itoa",
"pin-project-lite",
@@ -3447,7 +3450,7 @@ dependencies = [
"futures-channel",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"hyper",
"ipnet",
"libc",
@@ -3800,11 +3803,12 @@ dependencies = [
[[package]]
name = "jiff"
-version = "0.2.32"
+version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e"
+checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16"
dependencies = [
"defmt",
+ "jiff-core",
"jiff-static",
"jiff-tzdb-platform",
"js-sys",
@@ -3816,15 +3820,25 @@ dependencies = [
"windows-link",
]
+[[package]]
+name = "jiff-core"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09"
+dependencies = [
+ "defmt",
+]
+
[[package]]
name = "jiff-static"
-version = "0.2.32"
+version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc"
+checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de"
dependencies = [
+ "jiff-core",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -3872,7 +3886,7 @@ dependencies = [
"libloading",
"log",
"simd_cesu8",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
"walkdir",
"windows-link",
]
@@ -3887,7 +3901,7 @@ dependencies = [
"quote",
"rustc_version",
"simd_cesu8",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -3915,7 +3929,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -3954,7 +3968,7 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
dependencies = [
- "spin 0.9.8",
+ "spin 0.9.9",
]
[[package]]
@@ -4022,9 +4036,9 @@ checksum =
"34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
[[package]]
name = "libc"
-version = "0.2.186"
+version = "0.2.189"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
+checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
[[package]]
name = "libloading"
@@ -4073,9 +4087,9 @@ dependencies = [
[[package]]
name = "link-section"
-version = "0.19.0"
+version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e333fe507b738576d6da5bb3f1a7d7a1c80307ed9ef31624c057d844c19c93e9"
+checksum = "8dc98458dfe90986c5e2f6ddcf68360c7e5c4252600153e06aa4ee8176c0f8d1"
[[package]]
name = "linktime-proc-macro"
@@ -4142,7 +4156,7 @@ dependencies = [
"serde-value",
"serde_json",
"serde_yaml",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
"thread-id",
"typemap-ors",
"unicode-segmentation",
@@ -4238,9 +4252,9 @@ dependencies = [
[[package]]
name = "mio"
-version = "1.2.1"
+version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
+checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427"
dependencies = [
"libc",
"wasi",
@@ -4442,7 +4456,7 @@ dependencies = [
"serde",
"serde_json",
"serde_urlencoded",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
"tokio",
"tracing",
"url",
@@ -4523,7 +4537,7 @@ dependencies = [
"bytes",
"futures",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"jiff",
"log",
"md-5 0.11.0",
@@ -4945,7 +4959,7 @@ checksum =
"c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -5071,9 +5085,9 @@ dependencies = [
[[package]]
name = "portable-atomic"
-version = "1.13.1"
+version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
+checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3"
[[package]]
name = "portable-atomic-util"
@@ -5115,10 +5129,10 @@ dependencies = [
"nix",
"once_cell",
"smallvec",
- "spin 0.10.0",
+ "spin 0.10.1",
"symbolic-demangle",
"tempfile",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
]
[[package]]
@@ -5137,14 +5151,14 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
- "syn",
+ "syn 2.0.119",
]
[[package]]
name = "proc-macro2"
-version = "1.0.106"
+version = "1.0.107"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
dependencies = [
"unicode-ident",
]
@@ -5155,7 +5169,7 @@ version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"chrono",
"flate2",
"procfs-core",
@@ -5168,7 +5182,7 @@ version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"chrono",
"hex",
]
@@ -5198,7 +5212,7 @@ dependencies = [
"prost",
"prost-types",
"regex",
- "syn",
+ "syn 2.0.119",
"tempfile",
]
@@ -5212,7 +5226,7 @@ dependencies = [
"itertools 0.14.0",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -5273,7 +5287,7 @@ dependencies = [
"rustc-hash",
"rustls",
"socket2",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
"tokio",
"tracing",
"web-time",
@@ -5296,7 +5310,7 @@ dependencies = [
"rustls",
"rustls-pki-types",
"slab",
- "thiserror 2.0.18",
+ "thiserror 2.0.19",
"tinyvec",
"tracing",
"web-time",
@@ -5318,9 +5332,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.46"
+version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
@@ -5447,27 +5461,27 @@ version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
]
[[package]]
name = "ref-cast"
-version = "1.0.25"
+version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d"
+checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d"
dependencies = [
"ref-cast-impl",
]
[[package]]
name = "ref-cast-impl"
-version = "1.0.25"
+version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
+checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
@@ -5631,7 +5645,7 @@ dependencies = [
"futures-util",
"h2",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper",
"hyper-rustls",
@@ -5672,7 +5686,7 @@ dependencies = [
"futures-core",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"http-body-util",
"hyper",
"hyper-rustls",
@@ -5790,7 +5804,7 @@ version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"errno",
"libc",
"linux-raw-sys",
@@ -5799,9 +5813,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.41"
+version = "0.23.42"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
+checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138"
dependencies = [
"aws-lc-rs",
"once_cell",
@@ -5959,7 +5973,7 @@ version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"core-foundation",
"core-foundation-sys",
"libc",
@@ -5990,9 +6004,9 @@ checksum =
"1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "serde"
-version = "1.0.228"
+version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
dependencies = [
"serde_core",
"serde_derive",
@@ -6029,29 +6043,29 @@ dependencies = [
[[package]]
name = "serde_core"
-version = "1.0.228"
+version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.228"
+version = "1.0.229"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
name = "serde_json"
-version = "1.0.150"
+version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"indexmap 2.14.0",
"itoa",
@@ -6063,13 +6077,13 @@ dependencies = [
[[package]]
name = "serde_repr"
-version = "0.1.20"
+version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c"
+checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
@@ -6113,7 +6127,7 @@ dependencies = [
"darling 0.23.0",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6207,9 +6221,9 @@ checksum =
"3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
[[package]]
name = "simd_cesu8"
-version = "1.1.1"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
+checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520"
dependencies = [
"rustc_version",
"simdutf8",
@@ -6247,9 +6261,9 @@ checksum =
"199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886"
[[package]]
name = "socket2"
-version = "0.6.4"
+version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
+checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
dependencies = [
"libc",
"windows-sys 0.61.2",
@@ -6257,15 +6271,15 @@ dependencies = [
[[package]]
name = "spin"
-version = "0.9.8"
+version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
[[package]]
name = "spin"
-version = "0.10.0"
+version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591"
+checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
dependencies = [
"lock_api",
]
@@ -6298,7 +6312,7 @@ checksum =
"a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6337,7 +6351,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6371,9 +6385,20 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.118"
+version = "2.0.119"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422"
+checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
dependencies = [
"proc-macro2",
"quote",
@@ -6397,7 +6422,7 @@ checksum =
"728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6430,11 +6455,11 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "2.0.18"
+version = "2.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
+checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9"
dependencies = [
- "thiserror-impl 2.0.18",
+ "thiserror-impl 2.0.19",
]
[[package]]
@@ -6445,18 +6470,18 @@ checksum =
"4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
name = "thiserror-impl"
-version = "2.0.18"
+version = "2.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
+checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
@@ -6513,9 +6538,9 @@ dependencies = [
[[package]]
name = "time"
-version = "0.3.53"
+version = "0.3.54"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
+checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244"
dependencies = [
"deranged",
"num-conv",
@@ -6533,9 +6558,9 @@ checksum =
"9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]]
name = "time-macros"
-version = "0.2.31"
+version = "0.2.32"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
+checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85"
dependencies = [
"num-conv",
"time-core",
@@ -6587,9 +6612,9 @@ checksum =
"1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
-version = "1.53.0"
+version = "1.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d988bcd52dbe076d3d46903332f58c912b87a2c49b1428419a5845154762ffee"
+checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
dependencies = [
"bytes",
"libc",
@@ -6603,13 +6628,13 @@ dependencies = [
[[package]]
name = "tokio-macros"
-version = "2.7.0"
+version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
+checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6636,13 +6661,14 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.18"
+version = "0.7.19"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
+checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
+ "libc",
"pin-project-lite",
"tokio",
]
@@ -6668,11 +6694,11 @@ version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
- "bitflags 2.13.0",
+ "bitflags 2.13.1",
"bytes",
"futures-util",
"http 1.4.2",
- "http-body 1.0.1",
+ "http-body 1.1.0",
"pin-project-lite",
"tower",
"tower-layer",
@@ -6711,7 +6737,7 @@ checksum =
"7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6731,11 +6757,11 @@ checksum =
"e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "twox-hash"
-version = "2.1.2"
+version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
+checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9"
dependencies = [
- "rand 0.9.5",
+ "rand 0.10.2",
]
[[package]]
@@ -6755,7 +6781,7 @@ checksum =
"3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -6781,9 +6807,9 @@ checksum =
"b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "typetag"
-version = "0.2.22"
+version = "0.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c5a897b12c6c1151ad0b138b8db50252dc301f93bc3b027db05eec82aeed298c"
+checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2"
dependencies = [
"erased-serde",
"inventory",
@@ -6794,13 +6820,13 @@ dependencies = [
[[package]]
name = "typetag-impl"
-version = "0.2.22"
+version = "0.2.23"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cf808357c6ed7e13ba0f3277ec8d8f21b2d501274895104263985330c726c1c5"
+checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 3.0.3",
]
[[package]]
@@ -6902,9 +6928,9 @@ dependencies = [
[[package]]
name = "value-bag"
-version = "1.13.0"
+version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5dd4ec1eb1d240636e354a30110a1dfcb37047169a4d9bd6d9d3469df574b5c4"
+checksum = "ef73bfbaf3216cb59c205d7176bee1194e0d84348979da31f4a71fefe3c2054e"
[[package]]
name = "version_check"
@@ -6994,7 +7020,7 @@ dependencies = [
"bumpalo",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
"wasm-bindgen-shared",
]
@@ -7055,9 +7081,9 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
-version = "1.0.8"
+version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
+checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
dependencies = [
"rustls-pki-types",
]
@@ -7114,7 +7140,7 @@ checksum =
"053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -7125,7 +7151,7 @@ checksum =
"3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -7347,28 +7373,28 @@ checksum =
"de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
"synstructure",
]
[[package]]
name = "zerocopy"
-version = "0.8.54"
+version = "0.8.55"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19"
+checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.54"
+version = "0.8.55"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5"
+checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
@@ -7388,7 +7414,7 @@ checksum =
"11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
"synstructure",
]
@@ -7428,7 +7454,7 @@ checksum =
"625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.119",
]
[[package]]
diff --git a/native/core/src/execution/jni_api.rs
b/native/core/src/execution/jni_api.rs
index 6837373500..9a82eb6746 100644
--- a/native/core/src/execution/jni_api.rs
+++ b/native/core/src/execution/jni_api.rs
@@ -44,6 +44,7 @@ use datafusion::{
use datafusion_comet_proto::spark_operator::Operator;
use datafusion_comet_spark_expr::url_funcs::{CometParseUrl, CometTryParseUrl};
use datafusion_spark::function::array::array_contains::SparkArrayContains;
+use datafusion_spark::function::array::repeat::SparkArrayRepeat;
use datafusion_spark::function::bitwise::bit_count::SparkBitCount;
use datafusion_spark::function::bitwise::bit_get::SparkBitGet;
use datafusion_spark::function::bitwise::bit_shift::SparkBitShift;
@@ -609,11 +610,6 @@ fn prepare_datafusion_session_context(
// register UDFs from datafusion-spark crate
fn register_datafusion_spark_function(session_ctx: &SessionContext) {
- // Don't register SparkArrayRepeat — it returns NULL when the element is
NULL
- // (e.g. array_repeat(null, 3) returns NULL instead of [null, null, null]).
- // Comet's Scala serde wraps the call in a CaseWhen for null count
handling,
- // so DataFusion's built-in ArrayRepeat is sufficient.
- // TODO: file upstream issue against datafusion-spark
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkExpm1::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSha2::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(CharFunc::default()));
@@ -634,6 +630,7 @@ fn register_datafusion_spark_function(session_ctx:
&SessionContext) {
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkSpace::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkBitCount::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkArrayContains::default()));
+
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkArrayRepeat::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkBin::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkStrToMap::default()));
session_ctx.register_udf(ScalarUDF::new_from_impl(SparkUrlDecode::default()));
diff --git a/native/core/src/execution/planner.rs
b/native/core/src/execution/planner.rs
index 339bfff156..75d916768a 100644
--- a/native/core/src/execution/planner.rs
+++ b/native/core/src/execution/planner.rs
@@ -4814,7 +4814,6 @@ mod tests {
#[test]
fn test_array_repeat() {
- // Use built-in ArrayRepeat, not SparkArrayRepeat (see jni_api.rs
comment)
let session_ctx = SessionContext::new();
let task_ctx = session_ctx.task_ctx();
let planner = PhysicalPlanner::new(Arc::from(session_ctx), 0);
diff --git a/native/spark-expr/src/array_funcs/array_compact.rs
b/native/spark-expr/src/array_funcs/array_compact.rs
deleted file mode 100644
index 18e28b9afa..0000000000
--- a/native/spark-expr/src/array_funcs/array_compact.rs
+++ /dev/null
@@ -1,165 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Spark-compatible array_compact: removes null elements from an array.
-//
-// DataFusion's array_remove_all(arr, null) returns NULL for the entire row
-// when the element-to-remove is NULL (DF 53, PR #21013). Spark's array_compact
-// needs to actually remove null elements, so we implement it directly.
-//
-// TODO: upstream this to datafusion-spark crate
-
-use arrow::array::{
- make_array, Array, ArrayRef, Capacities, GenericListArray,
MutableArrayData, NullBufferBuilder,
- OffsetSizeTrait,
-};
-use arrow::buffer::OffsetBuffer;
-use arrow::datatypes::{DataType, FieldRef};
-use datafusion::common::{exec_err, utils::take_function_args, Result};
-use datafusion::logical_expr::{
- ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignature, Volatility,
-};
-use std::sync::Arc;
-
-#[derive(Debug, PartialEq, Eq, Hash)]
-pub struct SparkArrayCompact {
- signature: Signature,
-}
-
-impl Default for SparkArrayCompact {
- fn default() -> Self {
- Self::new()
- }
-}
-
-impl SparkArrayCompact {
- pub fn new() -> Self {
- Self {
- signature: Signature::new(TypeSignature::Any(1),
Volatility::Immutable),
- }
- }
-}
-
-impl ScalarUDFImpl for SparkArrayCompact {
- fn name(&self) -> &str {
- "spark_array_compact"
- }
-
- fn signature(&self) -> &Signature {
- &self.signature
- }
-
- fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
- datafusion::common::internal_err!("return_field_from_args should be
used instead")
- }
-
- fn return_field_from_args(
- &self,
- args: datafusion::logical_expr::ReturnFieldArgs,
- ) -> Result<FieldRef> {
- Ok(Arc::clone(&args.arg_fields[0]))
- }
-
- fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
- let [array] = take_function_args(self.name(), &args.args)?;
- match array {
- ColumnarValue::Array(array) => match array.data_type() {
- DataType::List(_) =>
Ok(ColumnarValue::Array(compact_list::<i32>(
- array.as_any().downcast_ref().unwrap(),
- )?)),
- DataType::LargeList(_) =>
Ok(ColumnarValue::Array(compact_list::<i64>(
- array.as_any().downcast_ref().unwrap(),
- )?)),
- other => exec_err!("spark_array_compact does not support type
'{other}'"),
- },
- ColumnarValue::Scalar(scalar) => {
- let array = scalar.to_array()?;
- let result = match array.data_type() {
- DataType::List(_) => {
-
compact_list::<i32>(array.as_any().downcast_ref().unwrap())?
- }
- DataType::LargeList(_) => {
-
compact_list::<i64>(array.as_any().downcast_ref().unwrap())?
- }
- other => {
- return exec_err!("spark_array_compact does not support
type '{other}'")
- }
- };
- Ok(ColumnarValue::Array(result))
- }
- }
- }
-}
-
-/// Remove null elements from each row of a list array.
-fn compact_list<OffsetSize: OffsetSizeTrait>(
- list_array: &GenericListArray<OffsetSize>,
-) -> Result<ArrayRef> {
- let list_field = match list_array.data_type() {
- DataType::List(field) | DataType::LargeList(field) => field,
- other => {
- return exec_err!("Expected List or LargeList, got {other:?}");
- }
- };
-
- let values = list_array.values();
- let original_data = values.to_data();
- let mut offsets = Vec::<OffsetSize>::with_capacity(list_array.len() + 1);
- offsets.push(OffsetSize::zero());
- let mut mutable = MutableArrayData::with_capacities(
- vec![&original_data],
- false,
- Capacities::Array(original_data.len()),
- );
- let mut valid = NullBufferBuilder::new(list_array.len());
-
- // Use logical_nulls() instead of is_null() to correctly handle NullArray.
- // NullArray::nulls() returns None (which makes is_null() return false),
- // but logical_nulls() correctly reports all elements as null.
- let value_nulls = values.logical_nulls();
-
- for (row_index, offset_window) in
list_array.offsets().windows(2).enumerate() {
- if list_array.is_null(row_index) {
- offsets.push(offsets[row_index]);
- valid.append_null();
- continue;
- }
-
- let start = offset_window[0].to_usize().unwrap();
- let end = offset_window[1].to_usize().unwrap();
- let mut copied = 0usize;
-
- for i in start..end {
- let is_null = value_nulls.as_ref().map(|n|
n.is_null(i)).unwrap_or(false);
- if !is_null {
- mutable.extend(0, i, i + 1);
- copied += 1;
- }
- }
-
- offsets.push(offsets[row_index] + OffsetSize::usize_as(copied));
- valid.append_non_null();
- }
-
- let new_values = make_array(mutable.freeze());
- Ok(Arc::new(GenericListArray::<OffsetSize>::try_new(
- Arc::clone(list_field),
- OffsetBuffer::new(offsets.into()),
- new_values,
- valid.finish(),
- )?))
-}
diff --git a/native/spark-expr/src/array_funcs/mod.rs
b/native/spark-expr/src/array_funcs/mod.rs
index b1361e3b72..0c2c68dc6d 100644
--- a/native/spark-expr/src/array_funcs/mod.rs
+++ b/native/spark-expr/src/array_funcs/mod.rs
@@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
-mod array_compact;
mod array_insert;
mod array_position;
mod array_slice;
@@ -26,7 +25,6 @@ mod get_array_struct_fields;
mod list_extract;
mod size;
-pub use array_compact::SparkArrayCompact;
pub use array_insert::ArrayInsert;
pub use array_position::SparkArrayPositionFunc;
pub use array_slice::SparkArraySlice;
diff --git a/native/spark-expr/src/comet_scalar_funcs.rs
b/native/spark-expr/src/comet_scalar_funcs.rs
index c102b01849..98d78e0e67 100644
--- a/native/spark-expr/src/comet_scalar_funcs.rs
+++ b/native/spark-expr/src/comet_scalar_funcs.rs
@@ -26,9 +26,9 @@ use crate::{
spark_ceil, spark_day_name, spark_decimal_div, spark_decimal_integral_div,
spark_floor,
spark_isnan, spark_lpad, spark_make_decimal, spark_month_name,
spark_read_side_padding,
spark_round, spark_rpad, spark_to_time, spark_unhex, spark_unscaled_value,
EvalMode,
- SparkArrayCompact, SparkArrayPositionFunc, SparkArraySlice,
SparkArraysOverlap, SparkContains,
- SparkDateDiff, SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten,
SparkMakeDate,
- SparkMakeTime, SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc,
+ SparkArrayPositionFunc, SparkArraySlice, SparkArraysOverlap,
SparkContains, SparkDateDiff,
+ SparkDateFromUnixDate, SparkDateTrunc, SparkFlatten, SparkMakeDate,
SparkMakeTime,
+ SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc,
};
use arrow::datatypes::DataType;
use datafusion::common::{DataFusionError, Result as DataFusionResult};
@@ -243,7 +243,6 @@ pub fn create_comet_physical_fun_with_eval_mode(
fn all_scalar_functions() -> Vec<Arc<ScalarUDF>> {
vec![
- Arc::new(ScalarUDF::new_from_impl(SparkArrayCompact::default())),
Arc::new(ScalarUDF::new_from_impl(SparkArrayPositionFunc::default())),
Arc::new(ScalarUDF::new_from_impl(SparkArraySlice::default())),
Arc::new(ScalarUDF::new_from_impl(SparkArraysOverlap::default())),
diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
index 284094de35..e62c7eef56 100644
--- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
@@ -57,7 +57,8 @@ object QueryPlanSerde extends Logging with CometExprShim with
CometTypeShim {
classOf[ArrayAppend] -> CometArrayAppend,
// ArrayCompact is RuntimeReplaceable in all supported Spark versions
(rewritten to
// ArrayFilter(arr, IsNotNull(...))), so it never reaches serde directly;
dispatch flows
- // through CometArrayFilter -> CometArrayCompact instead. No direct
registration here.
+ // through CometArrayFilter -> CometArrayCompact -> DataFusion's
array_compact. On Spark
+ // 4.0+ the rewrite is wrapped in KnownNotContainsNull, stripped by
Spark4xCometExprShim.
classOf[ArrayContains] -> CometArrayContains,
classOf[ArrayDistinct] -> CometScalarFunction("array_distinct"),
classOf[ArrayExcept] -> CometArrayExcept,
@@ -69,7 +70,7 @@ object QueryPlanSerde extends Logging with CometExprShim with
CometTypeShim {
classOf[ArrayMin] -> CometArrayMin,
classOf[ArrayPosition] -> CometArrayPosition,
classOf[ArrayRemove] -> CometArrayRemove,
- classOf[ArrayRepeat] -> CometArrayRepeat,
+ classOf[ArrayRepeat] -> CometScalarFunction("array_repeat"),
classOf[Slice] -> CometSlice,
classOf[SortArray] -> CometSortArray,
classOf[ArraysOverlap] -> CometArraysOverlap,
diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
index 8bc639021c..dfaecd3786 100644
--- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
@@ -22,7 +22,7 @@ package org.apache.comet.serde
import scala.annotation.tailrec
import scala.jdk.CollectionConverters._
-import org.apache.spark.sql.catalyst.expressions.{And, ArrayAggregate,
ArrayAppend, ArrayContains, ArrayExcept, ArrayExists, ArrayFilter, ArrayForAll,
ArrayInsert, ArrayIntersect, ArrayJoin, ArrayMax, ArrayMin, ArrayPosition,
ArrayRemove, ArrayRepeat, ArraySort, ArraysOverlap, ArraysZip, ArrayTransform,
ArrayUnion, Attribute, Cast, CreateArray, ElementAt, EmptyRow, Expression,
Flatten, GetArrayItem, IsNotNull, LambdaFunction, Literal, NamedLambdaVariable,
Reverse, Sequence, Size, Slice, [...]
+import org.apache.spark.sql.catalyst.expressions.{And, ArrayAggregate,
ArrayAppend, ArrayContains, ArrayExcept, ArrayExists, ArrayFilter, ArrayForAll,
ArrayInsert, ArrayIntersect, ArrayJoin, ArrayMax, ArrayMin, ArrayPosition,
ArrayRemove, ArraySort, ArraysOverlap, ArraysZip, ArrayTransform, ArrayUnion,
Attribute, Cast, CreateArray, ElementAt, EmptyRow, Expression, Flatten,
GetArrayItem, IsNotNull, LambdaFunction, Literal, NamedLambdaVariable, Reverse,
Sequence, Size, Slice, SortArray, ZipWith}
import org.apache.spark.sql.catalyst.util.GenericArrayData
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
@@ -259,46 +259,6 @@ object CometArraysOverlap extends
CometExpressionSerde[ArraysOverlap] {
}
}
-object CometArrayRepeat extends CometExpressionSerde[ArrayRepeat] {
-
- override def convert(
- expr: ArrayRepeat,
- inputs: Seq[Attribute],
- binding: Boolean): Option[ExprOuterClass.Expr] = {
- val elementProto = exprToProto(expr.left, inputs, binding)
- val countProto = exprToProto(expr.right, inputs, binding)
- val returnType = ArrayType(elementType = expr.left.dataType)
- for {
- countIsNotNullExpr <- countIsNotNullExpr(expr, inputs, binding)
- arrayRepeatExprProto <- scalarFunctionExprToProto("array_repeat",
elementProto, countProto)
- nullLiteralExprProto <- exprToProtoInternal(Literal(null, returnType),
inputs, binding)
- } yield {
- val caseWhenProto = ExprOuterClass.CaseWhen
- .newBuilder()
- .addWhen(countIsNotNullExpr)
- .addThen(arrayRepeatExprProto)
- .setElseExpr(nullLiteralExprProto)
- .build()
- ExprOuterClass.Expr
- .newBuilder()
- .setCaseWhen(caseWhenProto)
- .build()
- }
- }
-
- private def countIsNotNullExpr(
- expr: ArrayRepeat,
- inputs: Seq[Attribute],
- binding: Boolean): Option[ExprOuterClass.Expr] = {
- createUnaryExpr(
- expr,
- expr.right,
- inputs,
- binding,
- (builder, countExpr) => builder.setIsNotNull(countExpr))
- }
-}
-
object CometArrayCompact extends CometExpressionSerde[Expression] {
override def convert(
@@ -306,19 +266,9 @@ object CometArrayCompact extends
CometExpressionSerde[Expression] {
inputs: Seq[Attribute],
binding: Boolean): Option[ExprOuterClass.Expr] = {
val child = expr.children.head
- val elementType = child.dataType.asInstanceOf[ArrayType].elementType
-
val arrayExprProto = exprToProto(child, inputs, binding)
- // Use Comet's SparkArrayCompact UDF instead of DataFusion's
array_remove_all.
- // DF 53 changed array_remove_all to return NULL when the element arg is
NULL,
- // which breaks the array_compact use case.
- // TODO: upstream to datafusion-spark crate
- val arrayCompactScalarExpr = scalarFunctionExprToProtoWithReturnType(
- "spark_array_compact",
- ArrayType(elementType = elementType),
- false,
- arrayExprProto)
+ val arrayCompactScalarExpr = scalarFunctionExprToProto("array_compact",
arrayExprProto)
optExprWithFallbackReason(arrayCompactScalarExpr, expr, expr.children: _*)
}
}
diff --git
a/spark/src/main/spark-4.x/org/apache/comet/shims/Spark4xCometExprShim.scala
b/spark/src/main/spark-4.x/org/apache/comet/shims/Spark4xCometExprShim.scala
index a5e77b4a68..b0d9fa8475 100644
--- a/spark/src/main/spark-4.x/org/apache/comet/shims/Spark4xCometExprShim.scala
+++ b/spark/src/main/spark-4.x/org/apache/comet/shims/Spark4xCometExprShim.scala
@@ -23,13 +23,12 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.json.{JsonExpressionUtils,
StructsToJsonEvaluator}
import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke}
import org.apache.spark.sql.catalyst.expressions.url.ParseUrlEvaluator
-import org.apache.spark.sql.types.ArrayType
import org.apache.comet.CometExplainInfo
import org.apache.comet.expressions.CometEvalMode
import org.apache.comet.serde.{CometExpressionSerde, CometMapSort,
CometToPrettyString, CometWidthBucket}
import org.apache.comet.serde.ExprOuterClass.Expr
-import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal,
optExprWithFallbackReason, scalarFunctionExprToProtoWithReturnType}
+import org.apache.comet.serde.QueryPlanSerde.exprToProtoInternal
/**
* Shared trait body for the Spark 4.x `CometExprShim` traits (4.0/4.1/4.2).
Holds the parts that
@@ -64,28 +63,9 @@ trait Spark4xCometExprShim extends CometExprShim4x {
case knc: KnownNotContainsNull =>
// On Spark 4.0+, array_compact rewrites to
KnownNotContainsNull(ArrayFilter(IsNotNull)).
- // Strip the wrapper and serialize the inner ArrayFilter as
spark_array_compact. The
- // guard requires the IsNotNull operand to be the lambda's own
variable (matched by
- // exprId to handle nested-lambda shadowing), not a captured column
(#4830).
- knc.child match {
- case filter: ArrayFilter =>
- filter.function match {
- case LambdaFunction(IsNotNull(v: NamedLambdaVariable),
Seq(lambdaVar), _)
- if v.exprId == lambdaVar.exprId =>
- val arrayChild = filter.left
- val elementType =
arrayChild.dataType.asInstanceOf[ArrayType].elementType
- val arrayExprProto = exprToProtoInternal(arrayChild, inputs,
binding)
- val returnType = ArrayType(elementType)
- val scalarExpr = scalarFunctionExprToProtoWithReturnType(
- "spark_array_compact",
- returnType,
- false,
- arrayExprProto)
- optExprWithFallbackReason(scalarExpr, knc, arrayChild)
- case _ => exprToProtoInternal(knc.child, inputs, binding)
- }
- case _ => exprToProtoInternal(knc.child, inputs, binding)
- }
+ // Strip the wrapper and recurse; CometArrayFilter's fast path detects
the
+ // `x -> x IS NOT NULL` lambda and dispatches to CometArrayCompact.
+ exprToProtoInternal(knc.child, inputs, binding)
// On Spark 4.0+, RuntimeReplaceable expressions (StructsToJson,
ParseUrl) become
// Invoke(Literal(Evaluator), "evaluate", ...). Reconstruct the original
expression and
diff --git
a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
index 4143d86d75..e66b65fa3e 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
@@ -15,55 +15,134 @@
-- specific language governing permissions and limitations
-- under the License.
+-- Spark's ArrayCompact is RuntimeReplaceable -> rewrites to
+-- ArrayFilter(child, x -> IsNotNull(x)). On Spark 4.0+ that's wrapped in
+-- KnownNotContainsNull. Both paths route to DataFusion's array_compact.
statement
CREATE TABLE test_array_compact(
ints array<int>,
+ longs array<bigint>,
+ shorts array<smallint>,
+ bytes array<tinyint>,
+ floats array<float>,
+ doubles array<double>,
+ decs array<decimal(38, 0)>,
+ bools array<boolean>,
strs array<string>,
- dbls array<double>,
- nested array<array<int>>
+ bins array<binary>,
+ dates array<date>,
+ tss array<timestamp>,
+ nested array<array<int>>,
+ structs array<struct<a:int, b:string>>
) USING parquet
statement
INSERT INTO test_array_compact VALUES
- (array(1, NULL, 2, NULL, 3), array('a', NULL, 'b', NULL, 'c'), array(1.0,
NULL, 2.0), array(array(1, NULL, 3), NULL, array(4, NULL, 6))),
- (array(), array(), array(), array()),
- (NULL, NULL, NULL, NULL),
- (array(NULL, NULL), array(NULL, NULL), array(NULL, NULL), array(NULL, NULL)),
- (array(1, 2, 3), array('x', 'y', 'z'), array(1.5, 2.5), array(array(1, 2),
array(3, 4)))
+ (
+ array(1, NULL, 2147483647, NULL, -2147483648),
+ array(CAST(1 AS BIGINT), NULL, 9223372036854775807, NULL,
-9223372036854775808),
+ array(CAST(1 AS SMALLINT), NULL, CAST(32767 AS SMALLINT), NULL,
CAST(-32768 AS SMALLINT)),
+ array(CAST(1 AS TINYINT), NULL, CAST(127 AS TINYINT), NULL, CAST(-128 AS
TINYINT)),
+ array(CAST(1.0 AS FLOAT), NULL, CAST('NaN' AS FLOAT), CAST('Infinity' AS
FLOAT), CAST(-0.0 AS FLOAT)),
+ array(CAST(1.0 AS DOUBLE), NULL, CAST('NaN' AS DOUBLE), CAST('-Infinity'
AS DOUBLE), CAST(0.0 AS DOUBLE)),
+ array(CAST(1 AS DECIMAL(38, 0)), NULL,
CAST(99999999999999999999999999999999999999 AS DECIMAL(38, 0))),
+ array(true, NULL, false, NULL),
+ array('a', NULL, 'é', NULL, '日本', ''),
+ array(X'01', NULL, X'02FF', NULL, X''),
+ array(DATE '1970-01-01', NULL, DATE '9999-12-31'),
+ array(TIMESTAMP '1970-01-01 00:00:00', NULL, TIMESTAMP '9999-12-31
23:59:59'),
+ array(array(1, NULL, 3), NULL, array(4, 5)),
+ array(named_struct('a', 1, 'b', 'x'), NULL, named_struct('a', 2, 'b', 'y'))
+ ),
+ (
+ array(), array(), array(), array(), array(), array(), array(),
+ array(), array(), array(), array(), array(), array(), array()
+ ),
+ (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL),
+ (
+ array(NULL, NULL), array(NULL, NULL), array(NULL, NULL), array(NULL, NULL),
+ array(NULL, NULL), array(NULL, NULL), array(NULL, NULL), array(NULL, NULL),
+ array(NULL, NULL), array(NULL, NULL), array(NULL, NULL), array(NULL, NULL),
+ array(NULL, NULL), array(NULL, NULL)
+ ),
+ (
+ array(1, 2, 3),
+ array(CAST(1 AS BIGINT), CAST(2 AS BIGINT)),
+ array(CAST(1 AS SMALLINT), CAST(2 AS SMALLINT)),
+ array(CAST(1 AS TINYINT), CAST(2 AS TINYINT)),
+ array(CAST(1.5 AS FLOAT), CAST(2.5 AS FLOAT)),
+ array(CAST(1.5 AS DOUBLE), CAST(2.5 AS DOUBLE)),
+ array(CAST(1 AS DECIMAL(38, 0)), CAST(2 AS DECIMAL(38, 0))),
+ array(true, false),
+ array('x', 'y', 'z'),
+ array(X'AA', X'BB'),
+ array(DATE '2024-02-29'),
+ array(TIMESTAMP '2024-02-29 12:34:56'),
+ array(array(1, 2), array(3, 4)),
+ array(named_struct('a', 5, 'b', 'z'))
+ )
--- integer column
query
SELECT array_compact(ints) FROM test_array_compact
--- string column
+query
+SELECT array_compact(longs) FROM test_array_compact
+
+query
+SELECT array_compact(shorts) FROM test_array_compact
+
+query
+SELECT array_compact(bytes) FROM test_array_compact
+
+query
+SELECT array_compact(floats) FROM test_array_compact
+
+query
+SELECT array_compact(doubles) FROM test_array_compact
+
+query
+SELECT array_compact(decs) FROM test_array_compact
+
+query
+SELECT array_compact(bools) FROM test_array_compact
+
query
SELECT array_compact(strs) FROM test_array_compact
--- double column
query
-SELECT array_compact(dbls) FROM test_array_compact
+SELECT array_compact(bins) FROM test_array_compact
--- nested array column: outer nulls removed, inner nulls preserved
+query
+SELECT array_compact(dates) FROM test_array_compact
+
+query
+SELECT array_compact(tss) FROM test_array_compact
+
+-- nested array<array<int>>: outer NULL elements removed, inner NULL elements
preserved
query
SELECT array_compact(nested) FROM test_array_compact
--- literal arguments
+-- array<struct<...>>: NULL struct elements removed, inner field NULLs
preserved
+query
+SELECT array_compact(structs) FROM test_array_compact
+
+-- literal arguments (CometSqlFileTestSuite disables constant folding, so
+-- literal-only queries still exercise the native eval path)
query
SELECT array_compact(array(1, NULL, 2, NULL, 3))
--- literal string array
query
SELECT array_compact(array('a', NULL, 'b'))
--- all-null literal array
query
SELECT array_compact(array(NULL, NULL, NULL))
--- double element type
+query
+SELECT array_compact(array(CAST(NULL AS INT)))
+
query
SELECT array_compact(array(1.0, NULL, 2.0, NULL, 3.0))
--- nested array type (removes null arrays from outer, preserves null elements
in inner)
query
SELECT array_compact(array(array(1, NULL, 3), NULL, array(NULL, 2, 3)))
diff --git
a/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql
b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql
index a265caf9b4..f682a167ba 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_repeat.sql
@@ -15,23 +15,98 @@
-- specific language governing permissions and limitations
-- under the License.
+-- Mixed-column table covers per-type element with parameterized count.
statement
-CREATE TABLE test_array_repeat(val int, cnt int) USING parquet
+CREATE TABLE test_array_repeat(
+ byte_v tinyint,
+ short_v smallint,
+ int_v int,
+ long_v bigint,
+ float_v float,
+ double_v double,
+ dec_v decimal(38, 0),
+ bool_v boolean,
+ str_v string,
+ bin_v binary,
+ date_v date,
+ ts_v timestamp,
+ arr_v array<int>,
+ cnt int
+) USING parquet
statement
-INSERT INTO test_array_repeat VALUES (1, 3), (NULL, 3), (1, 0), (1, -1), (1,
NULL)
+INSERT INTO test_array_repeat VALUES
+ (CAST(127 AS TINYINT), CAST(32767 AS SMALLINT), 2147483647,
9223372036854775807, CAST('Infinity' AS FLOAT), CAST('Infinity' AS DOUBLE),
CAST(99999999999999999999999999999999999999 AS DECIMAL(38, 0)), true, 'a',
X'01', DATE '1970-01-01', TIMESTAMP '1970-01-01 00:00:00', array(1, 2, 3), 3),
+ (CAST(-128 AS TINYINT), CAST(-32768 AS SMALLINT), -2147483648,
-9223372036854775808, CAST('-Infinity' AS FLOAT), CAST('-Infinity' AS DOUBLE),
CAST(-99999999999999999999999999999999999999 AS DECIMAL(38, 0)), false, 'é',
X'02FF', DATE '9999-12-31', TIMESTAMP '9999-12-31 23:59:59', array(), 2),
+ (CAST(0 AS TINYINT), CAST(0 AS SMALLINT), 0, 0, CAST('NaN' AS FLOAT),
CAST('NaN' AS DOUBLE), CAST(0 AS DECIMAL(38, 0)), false, '', X'', DATE
'2024-02-29', TIMESTAMP '2024-02-29 12:34:56', array(NULL, 1, NULL), 1),
+ (NULL, NULL, NULL, NULL, CAST(0.0 AS FLOAT), CAST(-0.0 AS DOUBLE), NULL,
NULL, '日本', X'00', NULL, NULL, NULL, 0),
+ (CAST(1 AS TINYINT), CAST(1 AS SMALLINT), 1, 1, CAST(-0.0 AS FLOAT),
CAST(0.0 AS DOUBLE), CAST(1 AS DECIMAL(38, 0)), true, NULL, NULL, DATE
'2000-01-01', TIMESTAMP '2000-01-01 00:00:00', array(7), -1),
+ (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL, NULL)
+-- per-type column repeated with a column count
query
-SELECT array_repeat(val, cnt) FROM test_array_repeat
+SELECT array_repeat(byte_v, cnt) FROM test_array_repeat
--- column + literal
query
-SELECT array_repeat(val, 3) FROM test_array_repeat
+SELECT array_repeat(short_v, cnt) FROM test_array_repeat
--- literal + column
query
-SELECT array_repeat(1, cnt) FROM test_array_repeat
+SELECT array_repeat(int_v, cnt) FROM test_array_repeat
--- literal + literal
query
-SELECT array_repeat(1, 3), array_repeat(NULL, 3), array_repeat(1, 0),
array_repeat(1, -1)
+SELECT array_repeat(long_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(float_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(double_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(dec_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(bool_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(str_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(bin_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(date_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(ts_v, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(arr_v, cnt) FROM test_array_repeat
+
+-- column element + literal count: exercise count edge cases
+-- (NULL -> row NULL, 0 / negative -> empty array, positive -> array of N)
+query
+SELECT array_repeat(int_v, 3) FROM test_array_repeat
+
+query
+SELECT array_repeat(int_v, 0) FROM test_array_repeat
+
+query
+SELECT array_repeat(int_v, -5) FROM test_array_repeat
+
+query
+SELECT array_repeat(int_v, CAST(NULL AS INT)) FROM test_array_repeat
+
+-- literal element + column count
+query
+SELECT array_repeat(42, cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat('hello', cnt) FROM test_array_repeat
+
+-- NULL element repeated count times -- result is array of NULLs, not a NULL
row
+query
+SELECT array_repeat(CAST(NULL AS INT), cnt) FROM test_array_repeat
+
+query
+SELECT array_repeat(CAST(NULL AS STRING), cnt) FROM test_array_repeat
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
