This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 64271e9 [SPARK-61096][CONNECT][PYTHON] Represent GroupedData by its
grouping expressions, the types of its DataFrame and its type in the Python
client as PySpark does
64271e9 is described below
commit 64271e9654f1d654a30e4811b318688aa5d1f413
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:57:27 2026 +0900
[SPARK-61096][CONNECT][PYTHON] Represent GroupedData by its grouping
expressions, the types of its DataFrame and its type in the Python client as
PySpark does
### What changes were proposed in this pull request?
[SPARK-61096](https://issues.apache.org/jira/browse/SPARK-61096): the repr
of a `GroupedData` is now PySpark Connect's.
As PySpark Connect's, it names the grouping expressions, the columns of the
DataFrame with their types, and the type of the grouping (`GroupBy`, `RollUp`,
`Cube`, `GroupingSets` or `Pivot`):
```
GroupedData[grouping expressions: [+(k, 1), s], value: [k: int, s: string,
v: double], type: GroupBy]
```
The core's `GroupedData` gains `grouping_columns`, `group_type` and
`dataframe`.
### Why are the changes needed?
The client gave the default repr of an object:
`<pyspark.sql.group.GroupedData object at 0x...>`.
### Does this PR introduce _any_ user-facing change?
Yes. The repr of a `GroupedData` is that of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks the repr of a grouping by an expression and a
column, of a rollup and of a pivot. It passes with official PySpark 4.3.0 and
fails without this change.
- A new offline test checks the repr of each type of grouping against a
mock Connect server.
- On a local server, the reprs of 8 groupings, by columns, expressions and
aliases, of each type and of none, are the same as official PySpark 4.3.0's.
- The lib tests of both crates (595 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2047 passed, 12 skipped), and the
live drop-in and Arrow suites (151 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #301 from HyukjinKwon/SPARK-61096-grouped-data-repr.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/group.rs | 34 ++++++++++++++++++++++++++++++++++
crates/spark-connect/src/group.rs | 15 +++++++++++++++
python/tests/test_dropin_e2e.py | 15 +++++++++++++++
python/tests/test_dropin_offline.py | 19 +++++++++++++++++++
4 files changed, 83 insertions(+)
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index fefe83c..1f148dc 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -103,6 +103,40 @@ impl PyGroupedData {
#[pymethods]
impl PyGroupedData {
+ /// As PySpark Connect's: the grouping expressions, the columns of the
DataFrame and their
+ /// types, and the type of the grouping, such as
+ /// `GroupedData[grouping expressions: [k], value: [k: int, v: double],
type: GroupBy]`.
+ fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
+ use spark_connect::plan::AggregateGroupType;
+
+ let grouping: Vec<String> = self
+ .grouped_data
+ .grouping_columns()
+ .iter()
+ .map(|column| column.expression().render())
+ .collect();
+ let dataframe = self.grouped_data.dataframe().clone();
+ let dtypes = py
+ .detach(|| dataframe.dtypes().map_err(Box::new))
+ .to_pyerr()?;
+ let values: Vec<String> = dtypes
+ .iter()
+ .map(|(name, data_type)| format!("{name}: {data_type}"))
+ .collect();
+ let kind = match self.grouped_data.group_type() {
+ AggregateGroupType::GroupBy => "GroupBy",
+ AggregateGroupType::Rollup => "RollUp",
+ AggregateGroupType::Cube => "Cube",
+ AggregateGroupType::GroupingSets => "GroupingSets",
+ AggregateGroupType::Pivot => "Pivot",
+ };
+ Ok(format!(
+ "GroupedData[grouping expressions: [{}], value: [{}], type:
{kind}]",
+ grouping.join(", "),
+ values.join(", ")
+ ))
+ }
+
/// Aggregate with expressions (accepts Column objects), or the dict form
/// `agg({"col": "aggfunc"})` (PySpark parity), e.g. `agg({"age": "max"})`.
#[pyo3(signature = (*exprs))]
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index 2b14768..4ae9828 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -50,6 +50,21 @@ pub struct GroupedData {
}
impl GroupedData {
+ /// The grouping columns, as PySpark Connect's
`GroupedData._grouping_cols`.
+ pub fn grouping_columns(&self) -> &[Column] {
+ &self.group_cols
+ }
+
+ /// The type of the grouping, as PySpark Connect's
`GroupedData._group_type`.
+ pub fn group_type(&self) -> AggregateGroupType {
+ self.group_type
+ }
+
+ /// The DataFrame that is grouped, as PySpark Connect's `GroupedData._df`.
+ pub fn dataframe(&self) -> &DataFrame {
+ &self.dataframe
+ }
+
/// Create a new GroupedData.
pub(crate) fn new(
dataframe: DataFrame,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 3ec0338..3f09677 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1655,6 +1655,21 @@ def test_collected_rows_behave_as_pyspark_rows(session):
assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
+def test_grouped_data_are_represented_as_pyspark(session):
+ # As PySpark Connect's: the repr of a GroupedData names its grouping
expressions, the
+ # columns of its DataFrame and their types, and the type of the grouping.
+ from pyspark.sql import functions as F
+
+ df = session.createDataFrame([(1, "a", 2.0)], "k int, s string, v double")
+ assert repr(df.groupBy(F.col("k") + 1, "s")) == (
+ "GroupedData[grouping expressions: [+(k, 1), s], "
+ "value: [k: int, s: string, v: double], type: GroupBy]")
+ assert repr(df.rollup("k")).endswith("type: RollUp]")
+ assert repr(df.groupBy("k").pivot("s")).endswith("type: Pivot]")
+
+
+
+
def test_pivot_pivots_only_a_group_by_as_pyspark(session):
# As PySpark's: the pivot of a rollup, a cube, grouping sets or a pivot
raises
# UNSUPPORTED_OPERATION, rather than pivoting as a groupBy, which pivots.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index abe705c..dd533eb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3655,6 +3655,25 @@ def test_functions_substring():
assert type(F.substr(str_col, F.lit(1), F.lit(5))).__name__ == "Column"
+def test_grouped_data_are_represented_as_pyspark(mock_connect):
+ # As PySpark Connect's: the repr of a GroupedData names its grouping
expressions, the
+ # columns of its DataFrame and their types, and the type of the grouping.
+ from pyspark.sql import functions as F
+
+ spark, _ = mock_connect
+ df = spark.range(3)
+ value = ", ".join("%s: %s" % column for column in df.dtypes)
+ for grouped, expressions, kind in (
+ (df.groupBy("id"), "id", "GroupBy"), (df.rollup("id"), "id",
"RollUp"),
+ (df.cube(F.col("id") + 1), "+(id, 1)", "Cube"),
+ (df.groupingSets([["id"]], "id"), "id", "GroupingSets"),
+ (df.groupBy("id").pivot("id"), "id", "Pivot"), (df.groupBy(), "",
"GroupBy")):
+ assert repr(grouped) == (
+ f"GroupedData[grouping expressions: [{expressions}], value:
[{value}], type: {kind}]")
+
+
+
+
def test_pivot_pivots_only_a_group_by_and_checks_its_arguments_as_pyspark():
# As PySpark's GroupedData.pivot: only the grouping of groupBy pivots,
which it checks
# first, and then the column is a str and the values a list of bools,
floats, ints and
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]