This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 70c845eb [SPARK-61163][CONNECT][PYTHON] Make GroupedData.apply take
only pandas UDFs of SQL_GROUPED_MAP_PANDAS_UDF, and warn, in the Python client
as PySpark does
70c845eb is described below
commit 70c845eb25f8578042a3bfba02b0288c5c9a1f22
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sun Nov 8 14:35:54 2026 +0900
[SPARK-61163][CONNECT][PYTHON] Make GroupedData.apply take only pandas UDFs
of SQL_GROUPED_MAP_PANDAS_UDF, and warn, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61163](https://issues.apache.org/jira/browse/SPARK-61163): PySpark
Connect's `GroupedData.apply` takes only a `pandas_udf` of
`SQL_GROUPED_MAP_PANDAS_UDF`:
- anything else raises `PySparkTypeError` `INVALID_UDF_EVAL_TYPE`, of
`eval_type` `SQL_GROUPED_MAP_PANDAS_UDF`: a function, a UDF of another eval
type, or a Column, which it checks first, as a Column has any attribute
- such a UDF is applied with `applyInPandas`, of its function and return
type, with PySpark's `UserWarning` to prefer `applyInPandas`
The client's `apply` now checks and warns the same way.
The warning is PySpark's text. Its line is broken after "in the" because
the no-stub audit matches "in the future" on one line, as `catalog.rs` and
`column.rs` break theirs.
### Why are the changes needed?
The client checked only for a `func` attribute:
- for a function, it raised a plain `TypeError` ("apply() expects a
pandas_udf of type SQL_GROUPED_MAP_PANDAS_UDF")
- a UDF of another eval type, or a Column (whose `func` is a field), went
on to `applyInPandas`
- it did not warn
Apache Spark's `test_wrong_args` of `test_parity_pandas_grouped_map.py`
failed on this client.
### Does this PR introduce _any_ user-facing change?
Yes. `GroupedData.apply` raises the errors and the warning of PySpark 4.3.0.
### How was this patch tested?
- A new offline test, against a mock Connect server, checks the error class
and parameters for a function, a UDF, two Columns and two scalar pandas UDFs.
It also checks that a grouped-map pandas UDF warns and is sent with its eval
type. It fails without this change.
- The rows, warnings and error of `apply` against a local server are those
of official PySpark 4.3.0.
- Apache Spark's `test_parity_pandas_grouped_map.py` passes on this client
against a local server: 47 tests pass and 3 are skipped, including
`test_wrong_args`.
- The lib tests of both crates (597 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2219 passed, 8 skipped), and the
live drop-in and Arrow suites (201 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #352 from HyukjinKwon/SPARK-61163-grouped-apply.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/group.rs | 33 +++++++++++++++++++++++++++------
python/tests/test_dropin_offline.py | 35 +++++++++++++++++++++++++++++++++++
2 files changed, 62 insertions(+), 6 deletions(-)
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index c98c4506..827cca95 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -276,13 +276,34 @@ impl PyGroupedData {
/// Deprecated grouped-map apply: `apply(udf)` where `udf` is a
/// `SQL_GROUPED_MAP_PANDAS_UDF` pandas UDF. Delegates to `applyInPandas`
with the
- /// UDF's wrapped function and return type (mirrors the official Connect
API).
+ /// UDF's wrapped function and return type, as PySpark Connect's: anything
else, such as a
+ /// Column, a function or a UDF of another eval type, raises
`PySparkTypeError`
+ /// `INVALID_UDF_EVAL_TYPE`, and it warns to prefer `applyInPandas`.
fn apply(&self, py: Python<'_>, udf: Bound<'_, PyAny>) ->
PyResult<PyDataFrame> {
- let func = udf.getattr("func").map_err(|_| {
- PyErr::new::<pyo3::exceptions::PyTypeError, _>(
- "apply() expects a pandas_udf of type
SQL_GROUPED_MAP_PANDAS_UDF",
- )
- })?;
+ // A Column has any attribute, of its field of that name, so it is
checked first.
+ let grouped_map = !udf.is_instance_of::<crate::column::PyColumn>()
+ && udf.hasattr("func")?
+ && udf
+ .getattr("evalType")?
+ .eq(eval_type::SQL_GROUPED_MAP_PANDAS_UDF)?;
+ if !grouped_map {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "INVALID_UDF_EVAL_TYPE",
+ &[("eval_type", "SQL_GROUPED_MAP_PANDAS_UDF")],
+ ));
+ }
+ let warnings = py.import("warnings")?;
+ warnings.call_method1(
+ "warn",
+ (
+ "It is preferred to use 'applyInPandas' over this API. This
API will be deprecated in the \
+ future releases. See SPARK-28264 for more details.",
+ py.get_type::<pyo3::exceptions::PyUserWarning>(),
+ ),
+ )?;
+ let func = udf.getattr("func")?;
let schema = udf.getattr("returnType")?;
self.apply_in_pandas(py, func, schema)
}
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index aa99cbcf..030892f7 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -12759,6 +12759,41 @@ def
test_grouped_map_functions_warn_when_their_type_hints_cannot_be_inferred(
assert _sent_group_map_eval_type(service.execute_requests[-1]) == eval_type
+def
test_grouped_apply_takes_only_grouped_map_pandas_udfs_as_pyspark(mock_connect):
+ # As PySpark Connect's GroupedData.apply: anything but a pandas_udf of
+ # SQL_GROUPED_MAP_PANDAS_UDF raises INVALID_UDF_EVAL_TYPE, also a Column,
which has any
+ # attribute; such a UDF is applied with applyInPandas, with the warning to
prefer that.
+ import warnings
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+ from pyspark.sql.functions import PandasUDFType, pandas_udf
+
+ spark, service = mock_connect
+ df = spark.range(3)
+ grouped = df.groupBy("id")
+ for wrong in (
+ lambda x: x,
+ F.udf(lambda x: x, T.DoubleType()),
+ F.sum(df.id),
+ df.id + 1,
+ pandas_udf(lambda x, y: x, T.DoubleType()),
+ pandas_udf(lambda x, y: x, T.DoubleType(), PandasUDFType.SCALAR),
+ ):
+ with pytest.raises(PySparkTypeError) as error:
+ grouped.apply(wrong)
+ assert error.value.getCondition() == "INVALID_UDF_EVAL_TYPE"
+ assert error.value.getMessageParameters() == {"eval_type":
"SQL_GROUPED_MAP_PANDAS_UDF"}
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ identity = pandas_udf(lambda pdf: pdf, _ID_SCHEMA,
PandasUDFType.GROUPED_MAP)
+ grouped.apply(identity).collect()
+ assert (
+ "It is preferred to use 'applyInPandas' over this API. This API will
be deprecated in "
+ "the future releases. See SPARK-28264 for more details."
+ ) in [str(w.message) for w in caught if w.category is UserWarning]
+ assert _sent_group_map_eval_type(service.execute_requests[-1]) == 201
+
+
@pytest.mark.parametrize("build,detail", [
(lambda df: df.groupBy("id").applyInPandas(lambda a, b, c: a, _ID_SCHEMA),
"pandas_udfs with function type GROUPED_MAP or the function in
groupby.applyInPandas "
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]