This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9f07faf [SPARK-60952][CONNECT] Send the optional arguments of hmac,
variant_set, vector_norm and counter_diff as PySpark does
9f07faf is described below
commit 9f07faf1d77c3fb398a3e9e882c73ad5ef7029a7
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 12:10:57 2026 +0900
[SPARK-60952][CONNECT] Send the optional arguments of hmac, variant_set,
vector_norm and counter_diff as PySpark does
### What changes were proposed in this pull request?
[SPARK-60952](https://issues.apache.org/jira/browse/SPARK-60952): these
functions now send their optional arguments, as PySpark 4.3.0 does:
- `hmac(key, message, algorithm)`
- `variant_set(v, path, value, create_if_missing)` and `try_variant_set`
- `vector_norm(vector, degree)` and `vector_normalize(vector, degree)`
- `counter_diff(value, startTime)`
The Python wrappers take PySpark's signatures:
- An optional argument is sent when it is given, and `None` is not given.
- `variant_set` and `try_variant_set` always send `create_if_missing`,
`True` by default, as PySpark does.
The Rust core has `hmac_with_algorithm`,
`variant_set_with_create_if_missing`, `try_variant_set_with_create_if_missing`,
`vector_norm_with_degree`, `vector_normalize_with_degree` and
`counter_diff_with_start_time`, which the dispatch uses when the argument is
given.
### Why are the changes needed?
The wrappers took these arguments, but the dispatch sent only the leading
ones, so the results were silently wrong:
- `F.hmac(k, m, F.lit("SHA-1"))` computed an HMAC-SHA256.
- `variant_set(v, "$.b", lit(2), False)` inserted the missing path.
- `vector_norm(vec, one)` computed the L2 norm: 5.0 for `[3.0, 4.0]`, where
PySpark gives 7.0.
- `counter_diff(value, startTime)` ignored `startTime`.
An explicit `None`, as `vector_norm(v, None)`, was also sent as the
argument.
### Does this PR introduce _any_ user-facing change?
Yes, these functions now compute as in PySpark 4.3.0.
### How was this patch tested?
- Against a local Spark 4.3.0 server, `hmac` with `SHA-1`, `variant_set`
with `create_if_missing=False` and the default, and `vector_norm` and
`vector_normalize` of degree 1 give the same values as official PySpark 4.3.0;
before this change, they gave the defaults' values.
- The reprs of the fourteen calls of a probe have the same arguments as
official PySpark 4.3.0. Two print their literals differently, as `true` rather
than `True` and `1` rather than `1.0`, which
[SPARK-60948](https://issues.apache.org/jira/browse/SPARK-60948) tracks.
- New offline tests check that each optional argument is sent when given
and not when `None`; their cases agree with official PySpark 4.3.0. A new live
test checks the values, and it also passes with official PySpark 4.3.0.
- The lib tests and every integration test, e2e included, against the local
server (643 passed), the build of all tests with `wasm-udf`, the offline Python
suites (620 passed, 8 live cases skipped), and the live drop-in and Arrow
suites (58 passed) and live cases (10 passed). Clippy reports no diagnostics in
the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #174 from HyukjinKwon/SPARK-60952-optional-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dispatch_generated.rs | 12 ++---
crates/spark-connect/src/functions.rs | 72 +++++++++++++++++++++++++++++
python/pyspark/sql/functions.py | 32 +++++++++++++
python/tests/test_dropin_e2e.py | 18 ++++++++
python/tests/test_dropin_offline.py | 20 ++++++++
5 files changed, 148 insertions(+), 6 deletions(-)
diff --git a/crates/pyspark-rs/src/dispatch_generated.rs
b/crates/pyspark-rs/src/dispatch_generated.rs
index e6356c1..6873066 100644
--- a/crates/pyspark-rs/src/dispatch_generated.rs
+++ b/crates/pyspark-rs/src/dispatch_generated.rs
@@ -959,7 +959,7 @@ pub fn call_builtin(name: &str, args:
Vec<spark_connect::column::Column>) -> PyR
},
"counter_diff" => {
if args.is_empty() { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>("Missing required column
argument")); }
- Ok(spark_funcs::counter_diff(args[0].clone()))
+ if args.len() >= 2 {
Ok(spark_funcs::counter_diff_with_start_time(args[0].clone(), args[1].clone()))
} else { Ok(spark_funcs::counter_diff(args[0].clone())) }
},
"is_valid_variant" => {
if args.is_empty() { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>("Missing required column
argument")); }
@@ -1099,11 +1099,11 @@ pub fn call_builtin(name: &str, args:
Vec<spark_connect::column::Column>) -> PyR
},
"vector_norm" => {
if args.is_empty() { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>("Missing required column
argument")); }
- Ok(spark_funcs::vector_norm(args[0].clone()))
+ if args.len() >= 2 {
Ok(spark_funcs::vector_norm_with_degree(args[0].clone(), args[1].clone())) }
else { Ok(spark_funcs::vector_norm(args[0].clone())) }
},
"vector_normalize" => {
if args.is_empty() { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>("Missing required column
argument")); }
- Ok(spark_funcs::vector_normalize(args[0].clone()))
+ if args.len() >= 2 {
Ok(spark_funcs::vector_normalize_with_degree(args[0].clone(), args[1].clone()))
} else { Ok(spark_funcs::vector_normalize(args[0].clone())) }
},
"vector_sum" => {
if args.is_empty() { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>("Missing required column
argument")); }
@@ -1780,7 +1780,7 @@ pub fn call_builtin(name: &str, args:
Vec<spark_connect::column::Column>) -> PyR
},
"hmac" => {
if args.len() < 2 { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("Missing required
arguments for hmac: expected at least 2, got {}", args.len()))); }
- Ok(spark_funcs::hmac(args[0].clone(), args[1].clone()))
+ if args.len() >= 3 {
Ok(spark_funcs::hmac_with_algorithm(args[0].clone(), args[1].clone(),
args[2].clone())) } else { Ok(spark_funcs::hmac(args[0].clone(),
args[1].clone())) }
},
"jaro_winkler_similarity" => {
if args.len() < 2 { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("Missing required
arguments for jaro_winkler_similarity: expected at least 2, got {}",
args.len()))); }
@@ -1888,7 +1888,7 @@ pub fn call_builtin(name: &str, args:
Vec<spark_connect::column::Column>) -> PyR
},
"try_variant_set" => {
if args.len() < 3 { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("Missing required
arguments for try_variant_set: expected at least 3, got {}", args.len()))); }
- Ok(spark_funcs::try_variant_set(args[0].clone(),
args[1].clone(), args[2].clone()))
+ if args.len() >= 4 {
Ok(spark_funcs::try_variant_set_with_create_if_missing(args[0].clone(),
args[1].clone(), args[2].clone(), args[3].clone())) } else {
Ok(spark_funcs::try_variant_set(args[0].clone(), args[1].clone(),
args[2].clone())) }
},
"variant_array_append" => {
if args.len() < 3 { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("Missing required
arguments for variant_array_append: expected at least 3, got {}",
args.len()))); }
@@ -1900,7 +1900,7 @@ pub fn call_builtin(name: &str, args:
Vec<spark_connect::column::Column>) -> PyR
},
"variant_set" => {
if args.len() < 3 { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("Missing required
arguments for variant_set: expected at least 3, got {}", args.len()))); }
- Ok(spark_funcs::variant_set(args[0].clone(), args[1].clone(),
args[2].clone()))
+ if args.len() >= 4 {
Ok(spark_funcs::variant_set_with_create_if_missing(args[0].clone(),
args[1].clone(), args[2].clone(), args[3].clone())) } else {
Ok(spark_funcs::variant_set(args[0].clone(), args[1].clone(), args[2].clone()))
}
},
"vector_cosine_similarity" => {
if args.len() < 2 { return
Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(format!("Missing required
arguments for vector_cosine_similarity: expected at least 2, got {}",
args.len()))); }
diff --git a/crates/spark-connect/src/functions.rs
b/crates/spark-connect/src/functions.rs
index 3941ed5..0abc9e1 100644
--- a/crates/spark-connect/src/functions.rs
+++ b/crates/spark-connect/src/functions.rs
@@ -3618,6 +3618,14 @@ pub fn counter_diff(value: Column) -> Column {
func("counter_diff", vec![value.expression().clone()])
}
+/// Mirrors `pyspark.sql.functions.counter_diff(value, startTime)`.
+pub fn counter_diff_with_start_time(value: Column, start_time: Column) ->
Column {
+ func(
+ "counter_diff",
+ vec![value.expression().clone(), start_time.expression().clone()],
+ )
+}
+
/// Mirrors `pyspark.sql.functions.current_path`.
pub fn current_path() -> Column {
func("current_path", vec![])
@@ -3636,6 +3644,18 @@ pub fn hmac(key: Column, message: Column) -> Column {
)
}
+/// Mirrors `pyspark.sql.functions.hmac(key, message, algorithm)`.
+pub fn hmac_with_algorithm(key: Column, message: Column, algorithm: Column) ->
Column {
+ func(
+ "hmac",
+ vec![
+ key.expression().clone(),
+ message.expression().clone(),
+ algorithm.expression().clone(),
+ ],
+ )
+}
+
/// Mirrors `pyspark.sql.functions.is_valid_variant`.
pub fn is_valid_variant(v: Column) -> Column {
func("is_valid_variant", vec![v.expression().clone()])
@@ -4127,6 +4147,24 @@ pub fn try_variant_set(v: Column, path: Column, value:
Column) -> Column {
)
}
+/// Mirrors `pyspark.sql.functions.try_variant_set(v, path, value,
create_if_missing)`.
+pub fn try_variant_set_with_create_if_missing(
+ v: Column,
+ path: Column,
+ value: Column,
+ create_if_missing: Column,
+) -> Column {
+ func(
+ "try_variant_set",
+ vec![
+ v.expression().clone(),
+ path.expression().clone(),
+ value.expression().clone(),
+ create_if_missing.expression().clone(),
+ ],
+ )
+}
+
/// Mirrors `pyspark.sql.functions.unix_nanos`.
pub fn unix_nanos(col: Column) -> Column {
func("unix_nanos", vec![col.expression().clone()])
@@ -4185,6 +4223,24 @@ pub fn variant_set(v: Column, path: Column, value:
Column) -> Column {
)
}
+/// Mirrors `pyspark.sql.functions.variant_set(v, path, value,
create_if_missing)`.
+pub fn variant_set_with_create_if_missing(
+ v: Column,
+ path: Column,
+ value: Column,
+ create_if_missing: Column,
+) -> Column {
+ func(
+ "variant_set",
+ vec![
+ v.expression().clone(),
+ path.expression().clone(),
+ value.expression().clone(),
+ create_if_missing.expression().clone(),
+ ],
+ )
+}
+
/// Mirrors `pyspark.sql.functions.vector_avg`.
pub fn vector_avg(col: Column) -> Column {
func("vector_avg", vec![col.expression().clone()])
@@ -4219,11 +4275,27 @@ pub fn vector_norm(vector: Column) -> Column {
func("vector_norm", vec![vector.expression().clone()])
}
+/// Mirrors `pyspark.sql.functions.vector_norm(vector, degree)`.
+pub fn vector_norm_with_degree(vector: Column, degree: Column) -> Column {
+ func(
+ "vector_norm",
+ vec![vector.expression().clone(), degree.expression().clone()],
+ )
+}
+
/// Mirrors `pyspark.sql.functions.vector_normalize`.
pub fn vector_normalize(vector: Column) -> Column {
func("vector_normalize", vec![vector.expression().clone()])
}
+/// Mirrors `pyspark.sql.functions.vector_normalize(vector, degree)`.
+pub fn vector_normalize_with_degree(vector: Column, degree: Column) -> Column {
+ func(
+ "vector_normalize",
+ vec![vector.expression().clone(), degree.expression().clone()],
+ )
+}
+
/// Mirrors `pyspark.sql.functions.vector_sum`.
pub fn vector_sum(col: Column) -> Column {
func("vector_sum", vec![col.expression().clone()])
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index e8783c2..e81be66 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -510,6 +510,38 @@ for name in dir(_gen_module):
# generated single-arg wrapper cannot express (offset/default, variadic field
names).
# Defined AFTER the generated import so they take precedence.
Field/offset/default are
# literals; only the first arg is a column (PySpark ColumnOrName).
+# Functions whose optional arguments PySpark sends only when given (None is
not given), and
+# variant_set, whose create_if_missing it always sends.
+def hmac(key, message, algorithm=None):
+ """hmac(key, message, algorithm=None): the HMAC of message with key, by
algorithm."""
+ return _dispatch("hmac", [key, message] if algorithm is None else [key,
message, algorithm])
+
+
+def variant_set(v, path, value, create_if_missing=True):
+ """variant_set(v, path, value, create_if_missing=True): v with value set
at path."""
+ return _dispatch("variant_set", [v, path, value, create_if_missing])
+
+
+def try_variant_set(v, path, value, create_if_missing=True):
+ """try_variant_set(v, path, value, create_if_missing=True): variant_set,
or null on error."""
+ return _dispatch("try_variant_set", [v, path, value, create_if_missing])
+
+
+def vector_norm(vector, degree=None):
+ """vector_norm(vector, degree=None): the norm of vector, of degree (2.0 by
default)."""
+ return _dispatch("vector_norm", [vector] if degree is None else [vector,
degree])
+
+
+def vector_normalize(vector, degree=None):
+ """vector_normalize(vector, degree=None): vector divided by its norm of
degree."""
+ return _dispatch("vector_normalize", [vector] if degree is None else
[vector, degree])
+
+
+def counter_diff(value, startTime=None):
+ """counter_diff(value, startTime=None): the differences of a counter over
a window."""
+ return _dispatch("counter_diff", [value] if startTime is None else [value,
startTime])
+
+
def lag(col, offset=1, default=None):
"""lag(col, offset=1, default=None): value offset rows before the current
row."""
return _wrap(_call_function("lag", _to_col(col), _pyfunc_lit(offset),
_pyfunc_lit(default)))
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index c72dbdf..ba50415 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -633,3 +633,21 @@ def
test_create_dataframe_infers_by_the_session_configurations_as_pyspark(sessio
assert [tuple(r) for r in df.collect()] == rows, (key, data)
finally:
session.conf.unset(key)
+
+
+def test_functions_apply_their_optional_arguments_as_pyspark(session):
+ df = session.sql(
+ "SELECT 'key' AS k, 'msg' AS m, parse_json('{\"a\": 1}') AS v, "
+ "array(3.0F, 4.0F) AS vec, 1.0F AS one")
+ row = df.select(
+ F.hex(F.hmac("k", "m", F.lit("SHA-1"))).alias("sha1"),
+ F.to_json(F.variant_set("v", "$.b", F.lit(2), False)).alias("kept"),
+ F.to_json(F.try_variant_set("v", "$.b", F.lit(2))).alias("set"),
+ F.vector_norm("vec", F.col("one")).alias("l1"),
+ F.vector_normalize("vec", F.col("one")).alias("normalized"),
+ ).first()
+ assert row.sha1 == "102900B72B7BF1031EEC76B4804B66052376896B"
+ assert row.kept == '{"a":1}'
+ assert row.set == '{"a":1,"b":2}'
+ assert row.l1 == 7.0
+ assert row.normalized == pytest.approx([3 / 7, 4 / 7])
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index d87c9ad..cf347a1 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5315,3 +5315,23 @@ def test_sql_keyword_argument_errors_as_pyspark():
spark.sql("SELECT ? AS v", args=(1,))
assert error.value.getCondition() == "INVALID_TYPE"
assert error.value.getMessageParameters() == {"arg_name": "args",
"arg_type": "tuple"}
+
+
+# ---------------------------------------------------------------------------
optional args
+
+
[email protected]("column,expected", [
+ # The optional arguments of these functions are sent when given, and None
is not given,
+ # as PySpark does.
+ (lambda F: F.hmac("k", "m", "a"), "hmac(k, m, a)"),
+ (lambda F: F.hmac("k", "m", None), "hmac(k, m)"),
+ (lambda F: F.vector_norm("v", "d"), "vector_norm(v, d)"),
+ (lambda F: F.vector_norm("v", None), "vector_norm(v)"),
+ (lambda F: F.vector_normalize("v", degree="d"), "vector_normalize(v, d)"),
+ (lambda F: F.counter_diff("x", "t"), "counter_diff(x, t)"),
+ (lambda F: F.counter_diff("x", startTime=None), "counter_diff(x)"),
+])
+def test_functions_send_their_optional_arguments_as_pyspark(column, expected):
+ from pyspark.sql import functions as F
+
+ assert repr(column(F)) == f"Column<'{expected}'>"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]