This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new ce6ba5e  [SPARK-61037][CONNECT][PYTHON] Group by the grouping columns 
of groupingSets in the Python client as PySpark does
ce6ba5e is described below

commit ce6ba5ebe092015f4c9ecb7805d098f07e79c8f1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:02:39 2026 +0900

    [SPARK-61037][CONNECT][PYTHON] Group by the grouping columns of 
groupingSets in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61037](https://issues.apache.org/jira/browse/SPARK-61037): 
`groupingSets` now groups by its grouping columns `*cols`, as PySpark Connect 
does.
    
    `df.groupingSets(groupingSets, *cols)` groups by the grouping sets with the 
grouping columns `cols`, which are the columns of the result besides the 
aggregations. A grouping column that a set does not hold is null in the rows of 
that set, and without `cols` the result has only the aggregations. As PySpark's 
`_to_cols` of ordinals:
    - a set is a sequence of Columns, column names and 1-based ordinals, or one 
of them alone
    - `cols` are such columns themselves, not a list of them
    - anything else raises `NOT_EXPECTED_TYPE`
    
    The core takes the grouping columns in the new 
`DataFrame::grouping_sets_with_columns`.
    
    ### Why are the changes needed?
    
    The client ignored `cols` and grouped by the union of the columns of the 
sets. `df.groupingSets([["a"]], "a", "b").agg(sum("c"))` had the columns `a` 
and the sum, not `a`, `b` and the sum. `df.groupingSets([["a"], 
["b"]]).agg(sum("c"))` had `a`, `b` and the sum, not the sum only.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `groupingSets` returns the columns and rows of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the columns and rows of `groupingSets` with and 
without grouping columns, and of ordinals. It passes with official PySpark 
4.3.0 and fails without this change.
    - A new offline test checks the `Aggregate` that `groupingSets` sends to a 
mock Connect server, and its errors.
    - On a local server, 14 cases of `groupingSets` behave as in official 
PySpark 4.3.0.
    - The lib tests of both crates (571 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the 
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #239 from HyukjinKwon/SPARK-61037-grouping-sets-cols.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs    | 51 ++++++++++++++++++++++++++---------
 crates/spark-connect/src/dataframe.rs | 12 +++++++++
 crates/spark-connect/src/group.rs     | 11 ++++++++
 python/tests/test_dropin_e2e.py       | 17 ++++++++++++
 python/tests/test_dropin_offline.py   | 37 +++++++++++++++++++++++++
 5 files changed, 116 insertions(+), 12 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 0888ba2..c5a7aa1 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -181,10 +181,20 @@ impl PyDataFrame {
         py: Python<'_>,
         cols: Vec<Bound<'_, PyAny>>,
         arg_name: &str,
+    ) -> PyResult<Vec<spark_connect::column::Column>> {
+        self.grouping_column_list(py, unwrap_sequence(cols)?, arg_name)
+    }
+
+    /// The grouping columns of `cols`, each a Column, a column name or the 
1-based ordinal of
+    /// a column, as PySpark's `_to_cols` of ordinals takes the items of a 
sequence.
+    fn grouping_column_list(
+        &self,
+        py: Python<'_>,
+        cols: Vec<Bound<'_, PyAny>>,
+        arg_name: &str,
     ) -> PyResult<Vec<spark_connect::column::Column>> {
         use pyo3::types::{PyBool, PyString};
-        unwrap_sequence(cols)?
-            .iter()
+        cols.iter()
             .map(|c| {
                 if let Ok(column) = c.extract::<PyRef<'_, PyColumn>>() {
                     return Ok(column.column.clone());
@@ -964,23 +974,40 @@ impl PyDataFrame {
 
     /// Grouping sets aggregation: `groupingSets([[a, b], [a], []], "a", "b")`.
     ///
-    /// `*cols` (the overall grouping columns) is accepted for signature 
parity with
-    /// reference pyspark; the connect core derives the grouping from the 
sets, so the
-    /// trailing columns are currently informational.
+    /// As PySpark's: `*cols` are the grouping columns, the columns of the 
result besides the
+    /// aggregations, of Columns, names and ordinals, as are those of the sets.
     #[pyo3(name = "groupingSets", signature = (groupingSets, *cols))]
-    #[allow(unused_variables)]
     fn grouping_sets(
         &self,
         py: Python<'_>,
-        groupingSets: Vec<Vec<Bound<'_, PyAny>>>,
+        groupingSets: &Bound<'_, PyAny>,
         cols: Vec<Bound<'_, PyAny>>,
     ) -> PyResult<PyGroupedData> {
-        let mut out: Vec<Vec<spark_connect::column::Column>> =
-            Vec::with_capacity(groupingSets.len());
-        for s in groupingSets {
-            out.push(self.grouping_columns(py, s, "groupingSets")?);
+        use pyo3::types::{PyInt, PyString};
+        // As PySpark's `_to_cols(grouping_set)`: a set is a sequence of 
columns, or one column.
+        let mut sets = Vec::new();
+        for set in groupingSets.try_iter()? {
+            let set = set?;
+            let one = set.is_instance_of::<PyString>()
+                || set.is_instance_of::<PyInt>()
+                || set.is_instance_of::<PyColumn>();
+            let sequence = !one
+                && py
+                    .import("collections.abc")?
+                    .getattr("Sequence")
+                    .and_then(|sequence| set.is_instance(&sequence))?;
+            let items = if sequence {
+                set.try_iter()?.collect::<PyResult<Vec<_>>>()?
+            } else {
+                vec![set]
+            };
+            sets.push(self.grouping_column_list(py, items, "groupingSets")?);
         }
-        Ok(PyGroupedData::new(self.dataframe.grouping_sets(out)))
+        // As PySpark's `_to_cols(cols)`, of the columns themselves, not a 
list of them.
+        let columns = self.grouping_column_list(py, cols, "cols")?;
+        Ok(PyGroupedData::new(
+            self.dataframe.grouping_sets_with_columns(sets, columns),
+        ))
     }
 
     /// Drop duplicates within the event-time watermark.
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 29b56b5..7d32d57 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1667,6 +1667,18 @@ impl DataFrame {
         crate::group::GroupedData::new_grouping_sets(self.clone(), group_cols)
     }
 
+    /// Group by the grouping sets `grouping_sets` with the grouping columns 
`group_cols`,
+    /// which are the columns of the result besides the aggregations, 
mirroring PySpark's
+    /// `DataFrame.groupingSets(groupingSets, *cols)`: a column of the 
grouping columns that
+    /// a set does not hold is null in the rows of that set.
+    pub fn grouping_sets_with_columns(
+        &self,
+        grouping_sets: Vec<Vec<Column>>,
+        group_cols: Vec<Column>,
+    ) -> crate::group::GroupedData {
+        crate::group::GroupedData::new_grouping_sets_of(self.clone(), 
grouping_sets, group_cols)
+    }
+
     /// Sort within partitions (local sort).
     pub fn sort_within_partitions(&self, columns: Vec<Expression>) -> 
DataFrame {
         let plan = LogicalPlan::Sort {
diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index 3c976f4..a380704 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -83,6 +83,17 @@ impl GroupedData {
                 }
             }
         }
+        Self::new_grouping_sets_of(dataframe, grouping_sets, group_cols)
+    }
+
+    /// Grouped data of the grouping sets `grouping_sets` and the grouping 
columns
+    /// `group_cols`, the columns of the result besides the aggregations, as 
PySpark's
+    /// `groupingSets(groupingSets, *cols)`.
+    pub(crate) fn new_grouping_sets_of(
+        dataframe: DataFrame,
+        grouping_sets: Vec<Vec<Column>>,
+        group_cols: Vec<Column>,
+    ) -> Self {
         GroupedData {
             dataframe,
             group_cols,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index c42d4ec..d4dcf83 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -647,6 +647,23 @@ def test_create_dataframe_from_empty_arrow_table(session):
 _Pair = namedtuple("_Pair", ["x", "y"])
 
 
+def test_grouping_sets_return_the_rows_of_pyspark(session):
+    # As PySpark's: the grouping columns are those of `cols`, null in the rows 
of a set that
+    # does not hold them, and none without `cols`.
+    df = session.createDataFrame(
+        [(1, "x", 10), (1, "y", 20), (2, "x", 30)], "a int, b string, c int")
+    total = F.sum("c").alias("s")
+    with_columns = df.groupingSets([["a"], []], "a", "b").agg(total)
+    assert with_columns.columns == ["a", "b", "s"]
+    assert sorted(with_columns.collect(), key=lambda r: (r.a is None, r.a)) == 
[
+        Row(a=1, b=None, s=30), Row(a=2, b=None, s=30), Row(a=None, b=None, 
s=60)]
+    without_columns = df.groupingSets([["a"], ["b"]]).agg(total)
+    assert without_columns.columns == ["s"]
+    assert sorted(r.s for r in without_columns.collect()) == [20, 30, 30, 40]
+    # Of ordinals too.
+    assert df.groupingSets([[2]], 2).agg(total).columns == ["b", "s"]
+
+
 def test_grouped_aggregations_name_and_take_columns_as_pyspark(session):
     # As PySpark's: count is named count, the numeric aggregations of no 
columns aggregate
     # every numeric column, and df.agg takes a dict.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 4baa50a..bb243cb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5124,6 +5124,43 @@ if __name__ == "__main__":
     assert result.stdout.strip().splitlines()[-1] == str(["col"] * 6)
 
 
+def 
test_grouping_sets_group_by_their_grouping_columns_as_pyspark(mock_connect):
+    # As PySpark's, groupingSets(groupingSets, *cols) groups by the sets with 
the grouping
+    # columns `cols`: Relation.aggregate=9, Aggregate.group_type=2 
(GROUPING_SETS=5),
+    # grouping_expressions=3, grouping_sets=6 (GroupingSets.grouping_set=1).
+    from pyspark.errors import PySparkTypeError
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    df = spark.range(1)
+
+    def column_names(expressions):
+        # Expression.unresolved_attribute=2, 
UnresolvedAttribute.unparsed_identifier=1.
+        return [parse_message(parse_message(e)[2][0])[1][0].decode() for e in 
expressions]
+
+    def sent(grouped):
+        grouped.agg(F.count("*")).collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        aggregate = parse_message(parse_message(plan)[9][0])
+        sets = [column_names(parse_message(s).get(1, [])) for s in 
aggregate.get(6, [])]
+        return aggregate[2], column_names(aggregate.get(3, [])), sets
+
+    assert sent(df.groupingSets([["id"], []], "id", F.col("x"))) == (
+        [5], ["id", "x"], [["id"], []])
+    # No grouping columns, and a set of one column given alone.
+    assert sent(df.groupingSets(["id", ("x", 1)])) == ([5], [], [["id"], ["x", 
"id"]])
+    for call, arg_name, arg_type in (
+        (lambda: df.groupingSets([["id"]], ["id"]), "cols", "list"),
+        (lambda: df.groupingSets([[["id"]]]), "groupingSets", "list"),
+        (lambda: df.groupingSets([1.5]), "groupingSets", "float"),
+    ):
+        with pytest.raises(PySparkTypeError) as error:
+            call()
+        assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+        assert error.value.getMessageParameters() == {
+            "expected_type": "Column, int or str", "arg_name": arg_name, 
"arg_type": arg_type}
+
+
 def test_create_dataframe_checks_every_row_against_the_merged_schema():
     from pyspark.errors import PySparkValueError
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to