This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 77f6822  [SPARK-61043][CONNECT][PYTHON] Take the arguments of count 
and sum_distinct, and warn of sumDistinct, in the Python client as PySpark does
77f6822 is described below

commit 77f68224ffdd92b4e5c89022a5ccebc695006f5f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:03:34 2026 +0900

    [SPARK-61043][CONNECT][PYTHON] Take the arguments of count and 
sum_distinct, and warn of sumDistinct, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61043](https://issues.apache.org/jira/browse/SPARK-61043): `count` 
and `sum_distinct` now take their arguments, and `sumDistinct` warns, as 
PySpark Connect's do.
    
    - `functions.count(col)` and `functions.sum_distinct(col)` take a Column or 
a column name, and raise `NOT_EXPECTED_TYPE` for anything else, as PySpark's 
`_to_col` does.
    - `count` of a star Column, such as `count(col("*"))` or `count(df["*"])`, 
counts the rows as `count(1)`. `count("*")` is still the count of a star, as in 
PySpark.
    - `sumDistinct`, deprecated, warns `FutureWarning: Deprecated in 3.4, use 
sum_distinct instead.`
    
    ### Why are the changes needed?
    
    The client:
    - raised a `TypeError` for `count(1)`
    - took `sum_distinct(1)` as the sum of the distinct values of the literal 
`1`
    - sent `count(col("*"))` as the count of a star
    - did not warn of `sumDistinct`
    
    A sweep of all 560 functions of `pyspark.sql.functions` with seven kinds of 
arguments found these as the only differences in the errors and warnings at 
call time, apart from the exception types of misused `col`, `expr` and 
`call_function`. Those come from the implementation of PySpark, an 
`AssertionError` or `AttributeError`, not from its API.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These functions take the arguments, and raise the errors and warnings, 
of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks the columns of `count` and `sum_distinct`, 
their errors, and the warning of `sumDistinct`.
    - On a local server, 20 cases of these functions behave as in official 
PySpark 4.3.0: their columns, errors, warnings and results.
    - The lib tests of both crates (571 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the 
live drop-in and Arrow suites (108 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #241 from HyukjinKwon/SPARK-61043-count-sum-distinct.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/functions.rs  | 33 +++++++++++++++++++++++++++++++--
 python/pyspark/sql/functions.py     | 33 ++++++++++++++++++++++++++-------
 python/tests/test_dropin_offline.py | 27 +++++++++++++++++++++++++++
 3 files changed, 84 insertions(+), 9 deletions(-)

diff --git a/crates/pyspark-rs/src/functions.rs 
b/crates/pyspark-rs/src/functions.rs
index a806563..31e1aca 100644
--- a/crates/pyspark-rs/src/functions.rs
+++ b/crates/pyspark-rs/src/functions.rs
@@ -376,9 +376,38 @@ fn pyfunc_sum(col: &PyColumn) -> PyColumn {
     PyColumn::new(spark_funcs::sum(col.column.clone()))
 }
 
+/// `count(col)`, as PySpark Connect's: of a Column or a column name, else 
NOT_EXPECTED_TYPE,
+/// as its `_to_col`; a star Column, as `col("*")` or `df["*"]`, counts the 
rows, as
+/// `count(1)`, while a name is the column of it, so that `count("*")` is the 
count of a star.
 #[pyfunction]
-fn pyfunc_count(col: &PyColumn) -> PyColumn {
-    PyColumn::new(spark_funcs::count(col.column.clone()))
+fn pyfunc_count(col: &Bound<'_, PyAny>) -> PyResult<PyColumn> {
+    let column = if let Ok(column) = col.extract::<PyRef<'_, PyColumn>>() {
+        if matches!(
+            column.column.expression().node(),
+            ExpressionNode::UnresolvedStar(..)
+        ) {
+            spark_connect::column::lit(1)
+        } else {
+            column.column.clone()
+        }
+    } else if col.is_instance_of::<pyo3::types::PyString>() {
+        // As PySpark's `_to_col`, by `col`, of its origin.
+        let _origin = crate::origin::with_origin("col");
+        spark_funcs::col(&col.extract::<String>()?)
+    } else {
+        let type_name = col.get_type().name()?.to_string();
+        return Err(crate::column::raise_pyspark(
+            col.py(),
+            "PySparkTypeError",
+            "NOT_EXPECTED_TYPE",
+            &[
+                ("expected_type", "Column or str"),
+                ("arg_name", "col"),
+                ("arg_type", &type_name),
+            ],
+        ));
+    };
+    Ok(PyColumn::new(spark_funcs::count(column)))
 }
 
 #[pyfunction]
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 8933608..f362437 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -42,12 +42,6 @@ def _to_col(obj):
     return obj
 
 
-def _col_wrapper(native):
-    """Wrap a native single-column function so it accepts a str column name 
too."""
-    def wrapper(col):
-        return native(_to_col(col))
-    return wrapper
-
 def _wrap(obj):
     """Wrap the result from _call_function as a Column."""
     return obj
@@ -361,7 +355,6 @@ def lit(col):
 def expr(str):
     """Parses the expression string into the column that it represents."""
     return _pyfunc_expr(str)
-count = _col_wrapper(_pyfunc_count)
 
 
 def when(condition, value):
@@ -499,6 +492,32 @@ for name in _gen_module.__all__:
 _column_or_name = _gen_module._to_col
 
 
+def count(col):
+    """count(col): the number of the rows of a non-null col.
+
+    Mirrors pyspark.sql.connect.functions.count: of a Column or a column name, 
else
+    NOT_EXPECTED_TYPE, where a star Column counts the rows, as count(1).
+    """
+    return _pyfunc_count(col)
+
+
+def sum_distinct(col):
+    """sum_distinct(col): the sum of the distinct values of col.
+
+    Mirrors pyspark.sql.connect.functions.sum_distinct: of a Column or a 
column name, else
+    NOT_EXPECTED_TYPE.
+    """
+    return _dispatch("sum_distinct", [_column_or_name(col)])
+
+
+def sumDistinct(col):
+    """sumDistinct(col): deprecated since 3.4, use sum_distinct."""
+    import warnings
+
+    warnings.warn("Deprecated in 3.4, use sum_distinct instead.", 
FutureWarning)
+    return sum_distinct(col)
+
+
 def count_distinct(col, *cols):
     """count_distinct(col, *cols): the number of distinct combinations of the 
columns.
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index fd9bfd9..af59898 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -7001,6 +7001,33 @@ def test_function_calls_render_as_pyspark(column, 
expected):
     assert repr(column(F)) == f"Column<'{expected}'>"
 
 
+def test_count_and_sum_distinct_take_their_arguments_as_pyspark():
+    # As PySpark Connect's: a Column or a column name, else NOT_EXPECTED_TYPE; 
count of a star
+    # Column counts the rows, as count(1), and sumDistinct warns that it is 
deprecated.
+    import warnings
+
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    df = _offline_session().range(1)
+    assert repr(F.count(F.col("*"))) == "Column<'count(1)'>"
+    assert repr(F.count(df["*"])) == "Column<'count(1)'>"
+    assert repr(F.count(F.col("a.*"))) == "Column<'count(1)'>"
+    assert repr(F.count("a")) == repr(F.count(F.col("a"))) == 
"Column<'count(a)'>"
+    # A name is the column of it: count("*") is the count of a star, as in 
PySpark.
+    assert repr(F.count("*")) == "Column<'count(unresolvedstar())'>"
+    assert repr(F.sum_distinct("a")) == "Column<'sum(distinct a)'>"
+    for call in (lambda: F.count(1), lambda: F.sum_distinct(1), lambda: 
F.sum_distinct(None)):
+        with pytest.raises(PySparkTypeError) as error:
+            call()
+        assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+        assert error.value.getMessageParameters()["expected_type"] == "Column 
or str"
+    with warnings.catch_warnings(record=True) as record:
+        warnings.simplefilter("always")
+        assert repr(F.sumDistinct("a")) == "Column<'sum(distinct a)'>"
+    assert [str(w.message) for w in record] == ["Deprecated in 3.4, use 
sum_distinct instead."]
+
+
 @pytest.mark.parametrize("build,expected", [
     # As PySpark's lit: a list is the array of the literals of its elements.
     (lambda F: F.lit([1, 2]), "array(1, 2)"),


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to