This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 0cc15b9 [SPARK-61046][CONNECT][PYTHON] Pivot only a groupBy, and
raise the errors of the arguments of pivot, in the Python client as PySpark does
0cc15b9 is described below
commit 0cc15b9f2ab67357955434006fc71e6435c8c2e4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:26:37 2026 +0900
[SPARK-61046][CONNECT][PYTHON] Pivot only a groupBy, and raise the errors
of the arguments of pivot, in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61046](https://issues.apache.org/jira/browse/SPARK-61046):
`GroupedData.pivot` now pivots only a `groupBy`, and checks its arguments, as
PySpark Connect does.
- The pivot of a `rollup`, a `cube` or `groupingSets` raises
`UNSUPPORTED_OPERATION` of "PIVOT after ROLLUP", "PIVOT after CUBE" or "PIVOT
after GROUPING_SETS". The pivot of a pivot raises it of "Repeated PIVOT
operation". As in PySpark, the grouping is checked before the arguments.
- It then raises `NOT_EXPECTED_TYPE`:
- of `pivot_col`, for a pivot column that is not a str
- of `values`, for values that are not a list, such as a tuple
- of `value`, for a value that is not a bool, a float, an int or a str
The core `GroupedData::pivot` now returns a `Result`, with the error of the
new `GroupedData::check_pivot`.
### Why are the changes needed?
The client pivoted any grouping as a `groupBy`. So
`df.rollup("k").pivot("s").sum("v")` returned the rows of
`df.groupBy("k").pivot("s").sum("v")`, without the rows of the rollup, and a
second pivot replaced the first. These wrong results are the main fix. It also
raised `TypeError`s for the arguments, and took a tuple of values.
### Does this PR introduce _any_ user-facing change?
Yes. `pivot` raises the errors of PySpark 4.3.0, rather than returning the
rows of another grouping.
### How was this patch tested?
- A new live test checks the `UNSUPPORTED_OPERATION` of the pivots of a
rollup, a cube, grouping sets and a pivot, the rows of a pivot of a `groupBy`,
and the `NOT_EXPECTED_TYPE` of a tuple of values. It passes with official
PySpark 4.3.0, and fails without this change.
- A new offline test checks these errors and their parameters, and that the
grouping is checked first.
- A new lib test checks the errors of the core `pivot`.
- On a local server, 70 cases of `pivot` give the rows and errors of
official PySpark 4.3.0 for `groupBy`, `rollup`, `cube` and `groupingSets`.
- The lib tests of both crates (575 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #245 from HyukjinKwon/SPARK-61046-pivot.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/group.rs | 42 ++++++++++++++----
crates/spark-connect/src/group.rs | 64 ++++++++++++++++++++++++---
crates/spark-connect/tests/e2e_coverage.rs | 1 +
crates/spark-connect/tests/e2e_ops_breadth.rs | 1 +
python/tests/test_dropin_e2e.py | 24 ++++++++++
python/tests/test_dropin_offline.py | 39 ++++++++++++++++
6 files changed, 157 insertions(+), 14 deletions(-)
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 2666823..971282a 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -101,25 +101,49 @@ impl PyGroupedData {
Ok(PyDataFrame::new(df))
}
- /// Pivot on a column; `values=None` auto-discovers the distinct pivot
values.
+ /// Pivot on a column; `values=None` auto-discovers the distinct pivot
values. As PySpark's,
+ /// only the grouping of `groupBy` pivots, else `UNSUPPORTED_OPERATION`,
and then the column
+ /// is a str and the values a list of bools, floats, ints and strs, else
`NOT_EXPECTED_TYPE`.
#[pyo3(signature = (pivot_col, values=None))]
fn pivot(
&self,
- pivot_col: &str,
- values: Option<Vec<Bound<'_, PyAny>>>,
+ pivot_col: &Bound<'_, PyAny>,
+ values: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyGroupedData> {
- let pcol = spark_connect::functions::col(pivot_col);
+ use crate::stat_arguments::not_expected_type;
+ use pyo3::types::{PyBool, PyFloat, PyInt, PyList, PyString};
+ self.grouped_data.check_pivot().to_pyerr()?;
+ let Ok(name) = pivot_col.cast::<PyString>() else {
+ return Err(not_expected_type(pivot_col, "pivot_col", "str"));
+ };
let vals = match values {
None => None,
- Some(vs) => {
- let mut out = Vec::with_capacity(vs.len());
- for v in &vs {
- out.push(crate::session::py_to_value(v)?);
+ Some(values) => {
+ let Ok(values) = values.cast::<PyList>() else {
+ return Err(not_expected_type(values, "values", "list"));
+ };
+ let mut out = Vec::with_capacity(values.len());
+ for value in values.iter() {
+ if !(value.is_instance_of::<PyBool>()
+ || value.is_instance_of::<PyFloat>()
+ || value.is_instance_of::<PyInt>()
+ || value.is_instance_of::<PyString>())
+ {
+ return Err(not_expected_type(
+ &value,
+ "value",
+ "bool, float, int or str",
+ ));
+ }
+ out.push(crate::session::py_to_value(&value)?);
}
Some(out)
}
};
- Ok(PyGroupedData::new(self.grouped_data.pivot(pcol, vals)))
+ let pcol = spark_connect::functions::col(&name.to_cow()?);
+ Ok(PyGroupedData::new(
+ self.grouped_data.pivot(pcol, vals).to_pyerr()?,
+ ))
}
/// Sum values in each group.
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index a380704..2b14768 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -108,21 +108,46 @@ impl GroupedData {
///
/// Mirrors `df.groupBy(...).pivot(col, values)`. When `values` is `None`
the
/// server computes the distinct values; when supplied they are serialized
as
- /// pivot-value literals (previously they were dropped entirely).
- pub fn pivot(&self, pivot_col: Column, values:
Option<Vec<crate::row::Value>>) -> GroupedData {
+ /// pivot-value literals. As PySpark's, only the grouping of `group_by`
pivots: that of
+ /// a rollup, a cube, grouping sets or a pivot is the error of
[`GroupedData::check_pivot`].
+ pub fn pivot(
+ &self,
+ pivot_col: Column,
+ values: Option<Vec<crate::row::Value>>,
+ ) -> Result<GroupedData> {
+ self.check_pivot()?;
let pivot_values = values
.unwrap_or_default()
.into_iter()
.map(value_to_lit_expr)
.collect();
- GroupedData {
+ Ok(GroupedData {
dataframe: self.dataframe.clone(),
group_cols: self.group_cols.clone(),
group_type: AggregateGroupType::Pivot,
pivot_col: Some(pivot_col.expression().clone()),
pivot_values,
grouping_sets: vec![],
- }
+ })
+ }
+
+ /// Whether this grouping pivots, as PySpark's `GroupedData.pivot` checks
before its
+ /// arguments: only that of `group_by` does. That of a rollup, a cube or
grouping sets is
+ /// an `UNSUPPORTED_OPERATION` error of "PIVOT after ROLLUP", "PIVOT after
CUBE" or
+ /// "PIVOT after GROUPING_SETS", and that of a pivot of "Repeated PIVOT
operation".
+ pub fn check_pivot(&self) -> Result<()> {
+ let operation = match self.group_type {
+ AggregateGroupType::GroupBy => return Ok(()),
+ AggregateGroupType::Pivot => "Repeated PIVOT operation",
+ AggregateGroupType::Rollup => "PIVOT after ROLLUP",
+ AggregateGroupType::Cube => "PIVOT after CUBE",
+ AggregateGroupType::GroupingSets => "PIVOT after GROUPING_SETS",
+ };
+ Err(SparkError::classed(
+ SparkErrorKind::NotImplementedError,
+ "UNSUPPORTED_OPERATION",
+ &[("operation", operation)],
+ ))
}
/// Column names of the underlying (pre-grouping) DataFrame. These are
passed as the
@@ -986,11 +1011,39 @@ mod tests {
let spark = session();
let df = spark.range(10).unwrap();
let grouped = df.group_by(vec![crate::column::col("id")]);
- let pivot_grouped = grouped.pivot(crate::column::col("category"),
None);
+ let pivot_grouped = grouped.pivot(crate::column::col("category"),
None).unwrap();
assert_eq!(pivot_grouped.group_type, AggregateGroupType::Pivot);
assert!(pivot_grouped.pivot_col.is_some());
}
+ #[test]
+ fn only_the_grouping_of_group_by_pivots() {
+ let df = session().range(1).unwrap();
+ let id = || vec![crate::column::col("id")];
+ let pivot = |grouped: GroupedData|
grouped.pivot(crate::column::col("id"), None);
+ assert!(pivot(df.group_by(id())).is_ok());
+ for (grouped, operation) in [
+ (df.rollup(id()), "PIVOT after ROLLUP"),
+ (df.cube(id()), "PIVOT after CUBE"),
+ (df.grouping_sets(vec![id()]), "PIVOT after GROUPING_SETS"),
+ (
+ pivot(df.group_by(id())).unwrap(),
+ "Repeated PIVOT operation",
+ ),
+ ] {
+ let error = pivot(grouped).err().expect("an error");
+ assert_eq!(error.kind, SparkErrorKind::NotImplementedError);
+ assert_eq!(
+ error.get_condition().as_deref(),
+ Some("UNSUPPORTED_OPERATION")
+ );
+ assert_eq!(
+ error.get_message_parameters().unwrap()["operation"],
+ operation
+ );
+ }
+ }
+
#[test]
fn pivot_day_time_intervals_are_exact_literals() {
use spark_connect_proto::{expression::literal::LiteralType,
relation::RelType};
@@ -1007,6 +1060,7 @@ mod tests {
.collect(),
),
)
+ .unwrap()
.count();
let Some(RelType::Aggregate(aggregate)) =
pivot.plan.to_proto().rel_type else {
panic!("expected Aggregate");
diff --git a/crates/spark-connect/tests/e2e_coverage.rs
b/crates/spark-connect/tests/e2e_coverage.rs
index 1edaf59..8e13236 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -394,6 +394,7 @@ fn agg_na_stat_reshape_surface() {
let _ = df
.group_by(vec![col("id")])
.pivot(col("name"), None)
+ .unwrap()
.count()
.collect();
diff --git a/crates/spark-connect/tests/e2e_ops_breadth.rs
b/crates/spark-connect/tests/e2e_ops_breadth.rs
index d95453b..c66f124 100644
--- a/crates/spark-connect/tests/e2e_ops_breadth.rs
+++ b/crates/spark-connect/tests/e2e_ops_breadth.rs
@@ -188,6 +188,7 @@ fn aggregates_execute() {
let _ = df
.group_by(vec![col("id")])
.pivot(col("name"), None)
+ .unwrap()
.agg(vec![expr(f::sum(col("v")))])
.collect();
let _ = df.agg(vec![expr(f::sum(col("v")))]).collect().unwrap();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 6261cd2..85c6dd7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1118,6 +1118,30 @@ def test_collected_rows_behave_as_pyspark_rows(session):
assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
+def test_pivot_pivots_only_a_group_by_as_pyspark(session):
+ # As PySpark's: the pivot of a rollup, a cube, grouping sets or a pivot
raises
+ # UNSUPPORTED_OPERATION, rather than pivoting as a groupBy, which pivots.
+ from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
+
+ df = session.createDataFrame(
+ [(1, 10, "a"), (1, 20, "b"), (2, 30, "a")], "k int, v int, s string"
+ )
+ for grouped in [
+ df.rollup("k"),
+ df.cube("k"),
+ df.groupingSets([["k"]], "k"),
+ df.groupBy("k").pivot("s"),
+ ]:
+ with pytest.raises(PySparkNotImplementedError) as e:
+ grouped.pivot("s")
+ assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+ rows = df.groupBy("k").pivot("s", ["a",
"b"]).sum("v").orderBy("k").collect()
+ assert [tuple(row) for row in rows] == [(1, 10, 20), (2, 30, None)]
+ with pytest.raises(PySparkTypeError) as e:
+ df.groupBy("k").pivot("s", ("a",))
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+
+
def test_sample_sort_group_by_drop_and_select_expr_as_pyspark(session):
# As PySpark's: sample takes a fraction and a seed positionally, sort and
groupBy take the
# 1-based ordinals of columns, drop takes Columns, and selectExpr a list
of expressions.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index a24f545..73cf3e8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3001,6 +3001,45 @@ def test_functions_substring():
assert type(F.substr(str_col, F.lit(1), F.lit(5))).__name__ == "Column"
+def test_pivot_pivots_only_a_group_by_and_checks_its_arguments_as_pyspark():
+ # As PySpark's GroupedData.pivot: only the grouping of groupBy pivots,
which it checks
+ # first, and then the column is a str and the values a list of bools,
floats, ints and
+ # strs.
+ from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
+ from pyspark.sql import functions as F
+
+ df = _offline_session().range(1)
+ for grouped, operation in [
+ (df.rollup("id"), "PIVOT after ROLLUP"),
+ (df.cube("id"), "PIVOT after CUBE"),
+ (df.groupingSets([["id"]], "id"), "PIVOT after GROUPING_SETS"),
+ (df.groupBy("id").pivot("id"), "Repeated PIVOT operation"),
+ ]:
+ with pytest.raises(PySparkNotImplementedError) as e:
+ grouped.pivot(1)
+ assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+ assert e.value.getMessageParameters() == {"operation": operation}
+ grouped = df.groupBy("id")
+ for args, arg_name, expected_type, arg_type in [
+ ((1,), "pivot_col", "str", "int"),
+ ((F.col("id"),), "pivot_col", "str", "Column"),
+ (("id", "a"), "values", "list", "str"),
+ (("id", ("a",)), "values", "list", "tuple"),
+ (("id", [F.lit(1)]), "value", "bool, float, int or str", "Column"),
+ (("id", [None]), "value", "bool, float, int or str", "NoneType"),
+ ]:
+ with pytest.raises(PySparkTypeError) as e:
+ grouped.pivot(*args)
+ assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+ assert e.value.getMessageParameters() == {
+ "arg_name": arg_name,
+ "expected_type": expected_type,
+ "arg_type": arg_type,
+ }
+ for values in [None, [], [True, 1.5, 2, "a"]]:
+ grouped.pivot("id", values)
+
+
def test_replace_and_unpivot_check_their_arguments_as_pyspark():
# As PySpark's DataFrame.replace and unpivot, which check their arguments
on the client.
import warnings
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]