This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new ce6ba5e [SPARK-61037][CONNECT][PYTHON] Group by the grouping columns
of groupingSets in the Python client as PySpark does
ce6ba5e is described below
commit ce6ba5ebe092015f4c9ecb7805d098f07e79c8f1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:02:39 2026 +0900
[SPARK-61037][CONNECT][PYTHON] Group by the grouping columns of
groupingSets in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61037](https://issues.apache.org/jira/browse/SPARK-61037):
`groupingSets` now groups by its grouping columns `*cols`, as PySpark Connect
does.
`df.groupingSets(groupingSets, *cols)` groups by the grouping sets with the
grouping columns `cols`, which are the columns of the result besides the
aggregations. A grouping column that a set does not hold is null in the rows of
that set, and without `cols` the result has only the aggregations. As PySpark's
`_to_cols` of ordinals:
- a set is a sequence of Columns, column names and 1-based ordinals, or one
of them alone
- `cols` are such columns themselves, not a list of them
- anything else raises `NOT_EXPECTED_TYPE`
The core takes the grouping columns in the new
`DataFrame::grouping_sets_with_columns`.
### Why are the changes needed?
The client ignored `cols` and grouped by the union of the columns of the
sets. `df.groupingSets([["a"]], "a", "b").agg(sum("c"))` had the columns `a`
and the sum, not `a`, `b` and the sum. `df.groupingSets([["a"],
["b"]]).agg(sum("c"))` had `a`, `b` and the sum, not the sum only.
### Does this PR introduce _any_ user-facing change?
Yes. `groupingSets` returns the columns and rows of PySpark 4.3.0.
### How was this patch tested?
- A new live test checks the columns and rows of `groupingSets` with and
without grouping columns, and of ordinals. It passes with official PySpark
4.3.0 and fails without this change.
- A new offline test checks the `Aggregate` that `groupingSets` sends to a
mock Connect server, and its errors.
- On a local server, 14 cases of `groupingSets` behave as in official
PySpark 4.3.0.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #239 from HyukjinKwon/SPARK-61037-grouping-sets-cols.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 51 ++++++++++++++++++++++++++---------
crates/spark-connect/src/dataframe.rs | 12 +++++++++
crates/spark-connect/src/group.rs | 11 ++++++++
python/tests/test_dropin_e2e.py | 17 ++++++++++++
python/tests/test_dropin_offline.py | 37 +++++++++++++++++++++++++
5 files changed, 116 insertions(+), 12 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 0888ba2..c5a7aa1 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -181,10 +181,20 @@ impl PyDataFrame {
py: Python<'_>,
cols: Vec<Bound<'_, PyAny>>,
arg_name: &str,
+ ) -> PyResult<Vec<spark_connect::column::Column>> {
+ self.grouping_column_list(py, unwrap_sequence(cols)?, arg_name)
+ }
+
+ /// The grouping columns of `cols`, each a Column, a column name or the
1-based ordinal of
+ /// a column, as PySpark's `_to_cols` of ordinals takes the items of a
sequence.
+ fn grouping_column_list(
+ &self,
+ py: Python<'_>,
+ cols: Vec<Bound<'_, PyAny>>,
+ arg_name: &str,
) -> PyResult<Vec<spark_connect::column::Column>> {
use pyo3::types::{PyBool, PyString};
- unwrap_sequence(cols)?
- .iter()
+ cols.iter()
.map(|c| {
if let Ok(column) = c.extract::<PyRef<'_, PyColumn>>() {
return Ok(column.column.clone());
@@ -964,23 +974,40 @@ impl PyDataFrame {
/// Grouping sets aggregation: `groupingSets([[a, b], [a], []], "a", "b")`.
///
- /// `*cols` (the overall grouping columns) is accepted for signature
parity with
- /// reference pyspark; the connect core derives the grouping from the
sets, so the
- /// trailing columns are currently informational.
+ /// As PySpark's: `*cols` are the grouping columns, the columns of the
result besides the
+ /// aggregations, of Columns, names and ordinals, as are those of the sets.
#[pyo3(name = "groupingSets", signature = (groupingSets, *cols))]
- #[allow(unused_variables)]
fn grouping_sets(
&self,
py: Python<'_>,
- groupingSets: Vec<Vec<Bound<'_, PyAny>>>,
+ groupingSets: &Bound<'_, PyAny>,
cols: Vec<Bound<'_, PyAny>>,
) -> PyResult<PyGroupedData> {
- let mut out: Vec<Vec<spark_connect::column::Column>> =
- Vec::with_capacity(groupingSets.len());
- for s in groupingSets {
- out.push(self.grouping_columns(py, s, "groupingSets")?);
+ use pyo3::types::{PyInt, PyString};
+ // As PySpark's `_to_cols(grouping_set)`: a set is a sequence of
columns, or one column.
+ let mut sets = Vec::new();
+ for set in groupingSets.try_iter()? {
+ let set = set?;
+ let one = set.is_instance_of::<PyString>()
+ || set.is_instance_of::<PyInt>()
+ || set.is_instance_of::<PyColumn>();
+ let sequence = !one
+ && py
+ .import("collections.abc")?
+ .getattr("Sequence")
+ .and_then(|sequence| set.is_instance(&sequence))?;
+ let items = if sequence {
+ set.try_iter()?.collect::<PyResult<Vec<_>>>()?
+ } else {
+ vec![set]
+ };
+ sets.push(self.grouping_column_list(py, items, "groupingSets")?);
}
- Ok(PyGroupedData::new(self.dataframe.grouping_sets(out)))
+ // As PySpark's `_to_cols(cols)`, of the columns themselves, not a
list of them.
+ let columns = self.grouping_column_list(py, cols, "cols")?;
+ Ok(PyGroupedData::new(
+ self.dataframe.grouping_sets_with_columns(sets, columns),
+ ))
}
/// Drop duplicates within the event-time watermark.
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index 29b56b5..7d32d57 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1667,6 +1667,18 @@ impl DataFrame {
crate::group::GroupedData::new_grouping_sets(self.clone(), group_cols)
}
+ /// Group by the grouping sets `grouping_sets` with the grouping columns
`group_cols`,
+ /// which are the columns of the result besides the aggregations,
mirroring PySpark's
+ /// `DataFrame.groupingSets(groupingSets, *cols)`: a column of the
grouping columns that
+ /// a set does not hold is null in the rows of that set.
+ pub fn grouping_sets_with_columns(
+ &self,
+ grouping_sets: Vec<Vec<Column>>,
+ group_cols: Vec<Column>,
+ ) -> crate::group::GroupedData {
+ crate::group::GroupedData::new_grouping_sets_of(self.clone(),
grouping_sets, group_cols)
+ }
+
/// Sort within partitions (local sort).
pub fn sort_within_partitions(&self, columns: Vec<Expression>) ->
DataFrame {
let plan = LogicalPlan::Sort {
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index 3c976f4..a380704 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -83,6 +83,17 @@ impl GroupedData {
}
}
}
+ Self::new_grouping_sets_of(dataframe, grouping_sets, group_cols)
+ }
+
+ /// Grouped data of the grouping sets `grouping_sets` and the grouping
columns
+ /// `group_cols`, the columns of the result besides the aggregations, as
PySpark's
+ /// `groupingSets(groupingSets, *cols)`.
+ pub(crate) fn new_grouping_sets_of(
+ dataframe: DataFrame,
+ grouping_sets: Vec<Vec<Column>>,
+ group_cols: Vec<Column>,
+ ) -> Self {
GroupedData {
dataframe,
group_cols,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index c42d4ec..d4dcf83 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -647,6 +647,23 @@ def test_create_dataframe_from_empty_arrow_table(session):
_Pair = namedtuple("_Pair", ["x", "y"])
+def test_grouping_sets_return_the_rows_of_pyspark(session):
+ # As PySpark's: the grouping columns are those of `cols`, null in the rows
of a set that
+ # does not hold them, and none without `cols`.
+ df = session.createDataFrame(
+ [(1, "x", 10), (1, "y", 20), (2, "x", 30)], "a int, b string, c int")
+ total = F.sum("c").alias("s")
+ with_columns = df.groupingSets([["a"], []], "a", "b").agg(total)
+ assert with_columns.columns == ["a", "b", "s"]
+ assert sorted(with_columns.collect(), key=lambda r: (r.a is None, r.a)) ==
[
+ Row(a=1, b=None, s=30), Row(a=2, b=None, s=30), Row(a=None, b=None,
s=60)]
+ without_columns = df.groupingSets([["a"], ["b"]]).agg(total)
+ assert without_columns.columns == ["s"]
+ assert sorted(r.s for r in without_columns.collect()) == [20, 30, 30, 40]
+ # Of ordinals too.
+ assert df.groupingSets([[2]], 2).agg(total).columns == ["b", "s"]
+
+
def test_grouped_aggregations_name_and_take_columns_as_pyspark(session):
# As PySpark's: count is named count, the numeric aggregations of no
columns aggregate
# every numeric column, and df.agg takes a dict.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 4baa50a..bb243cb 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5124,6 +5124,43 @@ if __name__ == "__main__":
assert result.stdout.strip().splitlines()[-1] == str(["col"] * 6)
+def
test_grouping_sets_group_by_their_grouping_columns_as_pyspark(mock_connect):
+ # As PySpark's, groupingSets(groupingSets, *cols) groups by the sets with
the grouping
+ # columns `cols`: Relation.aggregate=9, Aggregate.group_type=2
(GROUPING_SETS=5),
+ # grouping_expressions=3, grouping_sets=6 (GroupingSets.grouping_set=1).
+ from pyspark.errors import PySparkTypeError
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+ df = spark.range(1)
+
+ def column_names(expressions):
+ # Expression.unresolved_attribute=2,
UnresolvedAttribute.unparsed_identifier=1.
+ return [parse_message(parse_message(e)[2][0])[1][0].decode() for e in
expressions]
+
+ def sent(grouped):
+ grouped.agg(F.count("*")).collect()
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ aggregate = parse_message(parse_message(plan)[9][0])
+ sets = [column_names(parse_message(s).get(1, [])) for s in
aggregate.get(6, [])]
+ return aggregate[2], column_names(aggregate.get(3, [])), sets
+
+ assert sent(df.groupingSets([["id"], []], "id", F.col("x"))) == (
+ [5], ["id", "x"], [["id"], []])
+ # No grouping columns, and a set of one column given alone.
+ assert sent(df.groupingSets(["id", ("x", 1)])) == ([5], [], [["id"], ["x",
"id"]])
+ for call, arg_name, arg_type in (
+ (lambda: df.groupingSets([["id"]], ["id"]), "cols", "list"),
+ (lambda: df.groupingSets([[["id"]]]), "groupingSets", "list"),
+ (lambda: df.groupingSets([1.5]), "groupingSets", "float"),
+ ):
+ with pytest.raises(PySparkTypeError) as error:
+ call()
+ assert error.value.getCondition() == "NOT_EXPECTED_TYPE"
+ assert error.value.getMessageParameters() == {
+ "expected_type": "Column, int or str", "arg_name": arg_name,
"arg_type": arg_type}
+
+
def test_create_dataframe_checks_every_row_against_the_merged_schema():
from pyspark.errors import PySparkValueError
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]