This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 77f6822 [SPARK-61043][CONNECT][PYTHON] Take the arguments of count
and sum_distinct, and warn of sumDistinct, in the Python client as PySpark does
77f6822 is described below
commit 77f68224ffdd92b4e5c89022a5ccebc695006f5f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:03:34 2026 +0900
[SPARK-61043][CONNECT][PYTHON] Take the arguments of count and
sum_distinct, and warn of sumDistinct, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61043](https://issues.apache.org/jira/browse/SPARK-61043): `count`
and `sum_distinct` now take their arguments, and `sumDistinct` warns, as
PySpark Connect's do.
- `functions.count(col)` and `functions.sum_distinct(col)` take a Column or
a column name, and raise `NOT_EXPECTED_TYPE` for anything else, as PySpark's
`_to_col` does.
- `count` of a star Column, such as `count(col("*"))` or `count(df["*"])`,
counts the rows as `count(1)`. `count("*")` is still the count of a star, as in
PySpark.
- `sumDistinct`, deprecated, warns `FutureWarning: Deprecated in 3.4, use
sum_distinct instead.`
### Why are the changes needed?
The client:
- raised a `TypeError` for `count(1)`
- took `sum_distinct(1)` as the sum of the distinct values of the literal
`1`
- sent `count(col("*"))` as the count of a star
- did not warn of `sumDistinct`
A sweep of all 560 functions of `pyspark.sql.functions` with seven kinds of
arguments found these as the only differences in the errors and warnings at
call time, apart from the exception types of misused `col`, `expr` and
`call_function`. Those come from the implementation of PySpark, an
`AssertionError` or `AttributeError`, not from its API.
### Does this PR introduce _any_ user-facing change?
Yes. These functions take the arguments, and raise the errors and warnings,
of PySpark 4.3.0.
### How was this patch tested?
- A new offline test checks the columns of `count` and `sum_distinct`,
their errors, and the warning of `sumDistinct`.
- On a local server, 20 cases of these functions behave as in official
PySpark 4.3.0: their columns, errors, warnings and results.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the
live drop-in and Arrow suites (108 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #241 from HyukjinKwon/SPARK-61043-count-sum-distinct.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/functions.rs | 33 +++++++++++++++++++++++++++++++--
python/pyspark/sql/functions.py | 33 ++++++++++++++++++++++++++-------
python/tests/test_dropin_offline.py | 27 +++++++++++++++++++++++++++
3 files changed, 84 insertions(+), 9 deletions(-)
diff --git a/crates/pyspark-rs/src/functions.rs
b/crates/pyspark-rs/src/functions.rs
index a806563..31e1aca 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -376,9 +376,38 @@ fn pyfunc_sum(col: &PyColumn) -> PyColumn {
PyColumn::new(spark_funcs::sum(col.column.clone()))
}
+/// `count(col)`, as PySpark Connect's: of a Column or a column name, else
NOT_EXPECTED_TYPE,
+/// as its `_to_col`; a star Column, as `col("*")` or `df["*"]`, counts the
rows, as
+/// `count(1)`, while a name is the column of it, so that `count("*")` is the
count of a star.
#[pyfunction]
-fn pyfunc_count(col: &PyColumn) -> PyColumn {
- PyColumn::new(spark_funcs::count(col.column.clone()))
+fn pyfunc_count(col: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
+ let column = if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
+ if matches!(
+ column.column.expression().node(),
+ ExpressionNode::UnresolvedStar(..)
+ ) {
+ spark_connect::column::lit(1)
+ } else {
+ column.column.clone()
+ }
+ } else if col.is_instance_of::<pyo3::types::PyString>() {
+ // As PySpark's `_to_col`, by `col`, of its origin.
+ let _origin = crate::origin::with_origin("col");
+ spark_funcs::col(&col.extract::<String>()?)
+ } else {
+ let type_name = col.get_type().name()?.to_string();
+ return Err(crate::column::raise_pyspark(
+ col.py(),
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("expected_type", "Column or str"),
+ ("arg_name", "col"),
+ ("arg_type", &type_name),
+ ],
+ ));
+ };
+ Ok(PyColumn::new(spark_funcs::count(column)))
}
#[pyfunction]
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 8933608..f362437 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -42,12 +42,6 @@ def _to_col(obj):
return obj
-def _col_wrapper(native):
- """Wrap a native single-column function so it accepts a str column name
too."""
- def wrapper(col):
- return native(_to_col(col))
- return wrapper
-
def _wrap(obj):
"""Wrap the result from _call_function as a Column."""
return obj
@@ -361,7 +355,6 @@ def lit(col):
def expr(str):
"""Parses the expression string into the column that it represents."""
return _pyfunc_expr(str)
-count = _col_wrapper(_pyfunc_count)
def when(condition, value):
@@ -499,6 +492,32 @@ for name in _gen_module.__all__:
_column_or_name = _gen_module._to_col
+def count(col):
+ """count(col): the number of the rows of a non-null col.
+
+ Mirrors pyspark.sql.connect.functions.count: of a Column or a column name,
else
+ NOT_EXPECTED_TYPE, where a star Column counts the rows, as count(1).
+ """
+ return _pyfunc_count(col)
+
+
+def sum_distinct(col):
+ """sum_distinct(col): the sum of the distinct values of col.
+
+ Mirrors pyspark.sql.connect.functions.sum_distinct: of a Column or a
column name, else
+ NOT_EXPECTED_TYPE.
+ """
+ return _dispatch("sum_distinct", [_column_or_name(col)])
+
+
+def sumDistinct(col):
+ """sumDistinct(col): deprecated since 3.4, use sum_distinct."""
+ import warnings
+
+ warnings.warn("Deprecated in 3.4, use sum_distinct instead.",
FutureWarning)
+ return sum_distinct(col)
+
+
def count_distinct(col, *cols):
"""count_distinct(col, *cols): the number of distinct combinations of the
columns.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index fd9bfd9..af59898 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7001,6 +7001,33 @@ def test_function_calls_render_as_pyspark(column,
expected):
assert repr(column(F)) == f"Column<'{expected}'>"
+def test_count_and_sum_distinct_take_their_arguments_as_pyspark():
+ # As PySpark Connect's: a Column or a column name, else NOT_EXPECTED_TYPE;
count of a star
+ # Column counts the rows, as count(1), and sumDistinct warns that it is
deprecated.
+ import warnings
+
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ df = _offline_session().range(1)
+ assert repr(F.count(F.col("*"))) == "Column<'count(1)'>"
+ assert repr(F.count(df["*"])) == "Column<'count(1)'>"
+ assert repr(F.count(F.col("a.*"))) == "Column<'count(1)'>"
+ assert repr(F.count("a")) == repr(F.count(F.col("a"))) ==
"Column<'count(a)'>"
+ # A name is the column of it: count("*") is the count of a star, as in
PySpark.
+ assert repr(F.count("*")) == "Column<'count(unresolvedstar())'>"
+ assert repr(F.sum_distinct("a")) == "Column<'sum(distinct a)'>"
+ for call in (lambda: F.count(1), lambda: F.sum_distinct(1), lambda:
F.sum_distinct(None)):
+ with pytest.raises(PySparkTypeError) as error:
+ call()
+ assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+ assert error.value.getMessageParameters()["expected_type"] == "Column
or str"
+ with warnings.catch_warnings(record=True) as record:
+ warnings.simplefilter("always")
+ assert repr(F.sumDistinct("a")) == "Column<'sum(distinct a)'>"
+ assert [str(w.message) for w in record] == ["Deprecated in 3.4, use
sum_distinct instead."]
+
+
@pytest.mark.parametrize("build,expected", [
# As PySpark's lit: a list is the array of the literals of its elements.
(lambda F: F.lit([1, 2]), "array(1, 2)"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]