This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 64271e9  [SPARK-61096][CONNECT][PYTHON] Represent GroupedData by its 
grouping expressions, the types of its DataFrame and its type in the Python 
client as PySpark does
64271e9 is described below

commit 64271e9654f1d654a30e4811b318688aa5d1f413
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:57:27 2026 +0900

    [SPARK-61096][CONNECT][PYTHON] Represent GroupedData by its grouping 
expressions, the types of its DataFrame and its type in the Python client as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61096](https://issues.apache.org/jira/browse/SPARK-61096): the repr 
of a `GroupedData` is now PySpark Connect's.
    
    As PySpark Connect's, it names the grouping expressions, the columns of the 
DataFrame with their types, and the type of the grouping (`GroupBy`, `RollUp`, 
`Cube`, `GroupingSets` or `Pivot`):
    
    ```
    GroupedData[grouping expressions: [+(k, 1), s], value: [k: int, s: string, 
v: double], type: GroupBy]
    ```
    
    The core's `GroupedData` gains `grouping_columns`, `group_type` and 
`dataframe`.
    
    ### Why are the changes needed?
    
    The client gave the default repr of an object: 
`<pyspark.sql.group.GroupedData object at 0x...>`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The repr of a `GroupedData` is that of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the repr of a grouping by an expression and a 
column, of a rollup and of a pivot. It passes with official PySpark 4.3.0 and 
fails without this change.
    - A new offline test checks the repr of each type of grouping against a 
mock Connect server.
    - On a local server, the reprs of 8 groupings, by columns, expressions and 
aliases, of each type and of none, are the same as official PySpark 4.3.0's.
    - The lib tests of both crates (595 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2047 passed, 12 skipped), and the 
live drop-in and Arrow suites (151 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #301 from HyukjinKwon/SPARK-61096-grouped-data-repr.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/group.rs      | 34 ++++++++++++++++++++++++++++++++++
 crates/spark-connect/src/group.rs   | 15 +++++++++++++++
 python/tests/test_dropin_e2e.py     | 15 +++++++++++++++
 python/tests/test_dropin_offline.py | 19 +++++++++++++++++++
 4 files changed, 83 insertions(+)

diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index fefe83c..1f148dc 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -103,6 +103,40 @@ impl PyGroupedData {
 
 #[pymethods]
 impl PyGroupedData {
+    /// As PySpark Connect's: the grouping expressions, the columns of the 
DataFrame and their
+    /// types, and the type of the grouping, such as
+    /// `GroupedData[grouping expressions: [k], value: [k: int, v: double], 
type: GroupBy]`.
+    fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
+        use spark_connect::plan::AggregateGroupType;
+
+        let grouping: Vec<String> = self
+            .grouped_data
+            .grouping_columns()
+            .iter()
+            .map(|column| column.expression().render())
+            .collect();
+        let dataframe = self.grouped_data.dataframe().clone();
+        let dtypes = py
+            .detach(|| dataframe.dtypes().map_err(Box::new))
+            .to_pyerr()?;
+        let values: Vec<String> = dtypes
+            .iter()
+            .map(|(name, data_type)| format!("{name}: {data_type}"))
+            .collect();
+        let kind = match self.grouped_data.group_type() {
+            AggregateGroupType::GroupBy => "GroupBy",
+            AggregateGroupType::Rollup => "RollUp",
+            AggregateGroupType::Cube => "Cube",
+            AggregateGroupType::GroupingSets => "GroupingSets",
+            AggregateGroupType::Pivot => "Pivot",
+        };
+        Ok(format!(
+            "GroupedData[grouping expressions: [{}], value: [{}], type: 
{kind}]",
+            grouping.join(", "),
+            values.join(", ")
+        ))
+    }
+
     /// Aggregate with expressions (accepts Column objects), or the dict form
     /// `agg({"col": "aggfunc"})` (PySpark parity), e.g. `agg({"age": "max"})`.
     #[pyo3(signature = (*exprs))]
diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index 2b14768..4ae9828 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -50,6 +50,21 @@ pub struct GroupedData {
 }
 
 impl GroupedData {
+    /// The grouping columns, as PySpark Connect's 
`GroupedData._grouping_cols`.
+    pub fn grouping_columns(&self) -> &[Column] {
+        &self.group_cols
+    }
+
+    /// The type of the grouping, as PySpark Connect's 
`GroupedData._group_type`.
+    pub fn group_type(&self) -> AggregateGroupType {
+        self.group_type
+    }
+
+    /// The DataFrame that is grouped, as PySpark Connect's `GroupedData._df`.
+    pub fn dataframe(&self) -> &DataFrame {
+        &self.dataframe
+    }
+
     /// Create a new GroupedData.
     pub(crate) fn new(
         dataframe: DataFrame,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 3ec0338..3f09677 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1655,6 +1655,21 @@ def test_collected_rows_behave_as_pyspark_rows(session):
     assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
 
 
+def test_grouped_data_are_represented_as_pyspark(session):
+    # As PySpark Connect's: the repr of a GroupedData names its grouping 
expressions, the
+    # columns of its DataFrame and their types, and the type of the grouping.
+    from pyspark.sql import functions as F
+
+    df = session.createDataFrame([(1, "a", 2.0)], "k int, s string, v double")
+    assert repr(df.groupBy(F.col("k") + 1, "s")) == (
+        "GroupedData[grouping expressions: [+(k, 1), s], "
+        "value: [k: int, s: string, v: double], type: GroupBy]")
+    assert repr(df.rollup("k")).endswith("type: RollUp]")
+    assert repr(df.groupBy("k").pivot("s")).endswith("type: Pivot]")
+
+
+
+
 def test_pivot_pivots_only_a_group_by_as_pyspark(session):
     # As PySpark's: the pivot of a rollup, a cube, grouping sets or a pivot 
raises
     # UNSUPPORTED_OPERATION, rather than pivoting as a groupBy, which pivots.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index abe705c..dd533eb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3655,6 +3655,25 @@ def test_functions_substring():
     assert type(F.substr(str_col, F.lit(1), F.lit(5))).__name__ == "Column"
 
 
+def test_grouped_data_are_represented_as_pyspark(mock_connect):
+    # As PySpark Connect's: the repr of a GroupedData names its grouping 
expressions, the
+    # columns of its DataFrame and their types, and the type of the grouping.
+    from pyspark.sql import functions as F
+
+    spark, _ = mock_connect
+    df = spark.range(3)
+    value = ", ".join("%s: %s" % column for column in df.dtypes)
+    for grouped, expressions, kind in (
+            (df.groupBy("id"), "id", "GroupBy"), (df.rollup("id"), "id", 
"RollUp"),
+            (df.cube(F.col("id") + 1), "+(id, 1)", "Cube"),
+            (df.groupingSets([["id"]], "id"), "id", "GroupingSets"),
+            (df.groupBy("id").pivot("id"), "id", "Pivot"), (df.groupBy(), "", 
"GroupBy")):
+        assert repr(grouped) == (
+            f"GroupedData[grouping expressions: [{expressions}], value: 
[{value}], type: {kind}]")
+
+
+
+
 def test_pivot_pivots_only_a_group_by_and_checks_its_arguments_as_pyspark():
     # As PySpark's GroupedData.pivot: only the grouping of groupBy pivots, 
which it checks
     # first, and then the column is a str and the values a list of bools, 
floats, ints and


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to