This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new f77df70 [SPARK-59116][FOLLOWUP] Default lag/lead offset to 1 for the
bare single-column form
f77df70 is described below
commit f77df70e33e53c3360d0f32489aa3f60d9df44a5
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Aug 30 07:24:01 2026 +0900
[SPARK-59116][FOLLOWUP] Default lag/lead offset to 1 for the bare
single-column form
Making functions::lag/lead variadic (for the offset/default params) dropped
the
default offset that PySpark's lag(col)/lead(col) emit (lag(col, 1)), so the
single-column form no longer matched tests/golden/functions.jsonl -> the
"Rust
build and test" / "WASM UDF feature" CI jobs failed the functions_golden
case
(cargo build --all-targets compiled it, but cargo test runs the assertion).
Append lit_int(1) only when a single column is passed; callers that supply
offset/default (the Python wrapper) already include them, so the multi-arg
path
is unchanged.
cargo test (default and --features wasm-udf) pass; behavioral batteries
remain
32/32 + 83/83 + 56/56.
Co-authored-by: Isaac <[email protected]>
---
crates/spark-connect/src/functions.rs | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
diff --git a/crates/spark-connect/src/functions.rs
b/crates/spark-connect/src/functions.rs
index 3020388..f2d972b 100644
--- a/crates/spark-connect/src/functions.rs
+++ b/crates/spark-connect/src/functions.rs
@@ -1047,7 +1047,13 @@ pub fn kurtosis(col1: Column) -> Column {
/// Mirrors `pyspark.sql.functions.lag`.
pub fn lag<C: Into<Column>>(cols: impl IntoIterator<Item = C>) -> Column {
let cols: Vec<Column> = cols.into_iter().map(Into::into).collect();
- func("lag", cols.iter().map(|c| c.expression().clone()).collect())
+ let mut args: Vec<Expression> = cols.iter().map(|c|
c.expression().clone()).collect();
+ // PySpark's lag(col) defaults the offset to 1 (lag(col, 1)); append it
only for the
+ // bare single-column form -- callers passing offset/default already
include them.
+ if args.len() == 1 {
+ args.push(lit_int(1));
+ }
+ func("lag", args)
}
/// Mirrors `pyspark.sql.functions.last`.
@@ -1073,10 +1079,13 @@ pub fn lcase(col1: Column) -> Column {
/// Mirrors `pyspark.sql.functions.lead`.
pub fn lead<C: Into<Column>>(cols: impl IntoIterator<Item = C>) -> Column {
let cols: Vec<Column> = cols.into_iter().map(Into::into).collect();
- func(
- "lead",
- cols.iter().map(|c| c.expression().clone()).collect(),
- )
+ let mut args: Vec<Expression> = cols.iter().map(|c|
c.expression().clone()).collect();
+ // PySpark's lead(col) defaults the offset to 1; append it only for the
bare
+ // single-column form -- callers passing offset/default already include
them.
+ if args.len() == 1 {
+ args.push(lit_int(1));
+ }
+ func("lead", args)
}
/// Mirrors `pyspark.sql.functions.length`.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
