This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 0cc15b9  [SPARK-61046][CONNECT][PYTHON] Pivot only a groupBy, and 
raise the errors of the arguments of pivot, in the Python client as PySpark does
0cc15b9 is described below

commit 0cc15b9f2ab67357955434006fc71e6435c8c2e4
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:26:37 2026 +0900

    [SPARK-61046][CONNECT][PYTHON] Pivot only a groupBy, and raise the errors 
of the arguments of pivot, in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61046](https://issues.apache.org/jira/browse/SPARK-61046): 
`GroupedData.pivot` now pivots only a `groupBy`, and checks its arguments, as 
PySpark Connect does.
    
    - The pivot of a `rollup`, a `cube` or `groupingSets` raises 
`UNSUPPORTED_OPERATION` of "PIVOT after ROLLUP", "PIVOT after CUBE" or "PIVOT 
after GROUPING_SETS". The pivot of a pivot raises it of "Repeated PIVOT 
operation". As in PySpark, the grouping is checked before the arguments.
    - It then raises `NOT_EXPECTED_TYPE`:
      - of `pivot_col`, for a pivot column that is not a str
      - of `values`, for values that are not a list, such as a tuple
      - of `value`, for a value that is not a bool, a float, an int or a str
    
    The core `GroupedData::pivot` now returns a `Result`, with the error of the 
new `GroupedData::check_pivot`.
    
    ### Why are the changes needed?
    
    The client pivoted any grouping as a `groupBy`. So 
`df.rollup("k").pivot("s").sum("v")` returned the rows of 
`df.groupBy("k").pivot("s").sum("v")`, without the rows of the rollup, and a 
second pivot replaced the first. These wrong results are the main fix. It also 
raised `TypeError`s for the arguments, and took a tuple of values.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `pivot` raises the errors of PySpark 4.3.0, rather than returning the 
rows of another grouping.
    
    ### How was this patch tested?
    
    - A new live test checks the `UNSUPPORTED_OPERATION` of the pivots of a 
rollup, a cube, grouping sets and a pivot, the rows of a pivot of a `groupBy`, 
and the `NOT_EXPECTED_TYPE` of a tuple of values. It passes with official 
PySpark 4.3.0, and fails without this change.
    - A new offline test checks these errors and their parameters, and that the 
grouping is checked first.
    - A new lib test checks the errors of the core `pivot`.
    - On a local server, 70 cases of `pivot` give the rows and errors of 
official PySpark 4.3.0 for `groupBy`, `rollup`, `cube` and `groupingSets`.
    - The lib tests of both crates (575 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1811 passed, 8 skipped), and the 
live drop-in and Arrow suites (110 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #245 from HyukjinKwon/SPARK-61046-pivot.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/group.rs                | 42 ++++++++++++++----
 crates/spark-connect/src/group.rs             | 64 ++++++++++++++++++++++++---
 crates/spark-connect/tests/e2e_coverage.rs    |  1 +
 crates/spark-connect/tests/e2e_ops_breadth.rs |  1 +
 python/tests/test_dropin_e2e.py               | 24 ++++++++++
 python/tests/test_dropin_offline.py           | 39 ++++++++++++++++
 6 files changed, 157 insertions(+), 14 deletions(-)

diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 2666823..971282a 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -101,25 +101,49 @@ impl PyGroupedData {
         Ok(PyDataFrame::new(df))
     }
 
-    /// Pivot on a column; `values=None` auto-discovers the distinct pivot 
values.
+    /// Pivot on a column; `values=None` auto-discovers the distinct pivot 
values. As PySpark's,
+    /// only the grouping of `groupBy` pivots, else `UNSUPPORTED_OPERATION`, 
and then the column
+    /// is a str and the values a list of bools, floats, ints and strs, else 
`NOT_EXPECTED_TYPE`.
     #[pyo3(signature = (pivot_col, values=None))]
     fn pivot(
         &self,
-        pivot_col: &str,
-        values: Option<Vec<Bound<'_, PyAny>>>,
+        pivot_col: &Bound<'_, PyAny>,
+        values: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyGroupedData> {
-        let pcol = spark_connect::functions::col(pivot_col);
+        use crate::stat_arguments::not_expected_type;
+        use pyo3::types::{PyBool, PyFloat, PyInt, PyList, PyString};
+        self.grouped_data.check_pivot().to_pyerr()?;
+        let Ok(name) = pivot_col.cast::<PyString>() else {
+            return Err(not_expected_type(pivot_col, "pivot_col", "str"));
+        };
         let vals = match values {
             None => None,
-            Some(vs) => {
-                let mut out = Vec::with_capacity(vs.len());
-                for v in &vs {
-                    out.push(crate::session::py_to_value(v)?);
+            Some(values) => {
+                let Ok(values) = values.cast::<PyList>() else {
+                    return Err(not_expected_type(values, "values", "list"));
+                };
+                let mut out = Vec::with_capacity(values.len());
+                for value in values.iter() {
+                    if !(value.is_instance_of::<PyBool>()
+                        || value.is_instance_of::<PyFloat>()
+                        || value.is_instance_of::<PyInt>()
+                        || value.is_instance_of::<PyString>())
+                    {
+                        return Err(not_expected_type(
+                            &value,
+                            "value",
+                            "bool, float, int or str",
+                        ));
+                    }
+                    out.push(crate::session::py_to_value(&value)?);
                 }
                 Some(out)
             }
         };
-        Ok(PyGroupedData::new(self.grouped_data.pivot(pcol, vals)))
+        let pcol = spark_connect::functions::col(&name.to_cow()?);
+        Ok(PyGroupedData::new(
+            self.grouped_data.pivot(pcol, vals).to_pyerr()?,
+        ))
     }
 
     /// Sum values in each group.
diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index a380704..2b14768 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -108,21 +108,46 @@ impl GroupedData {
     ///
     /// Mirrors `df.groupBy(...).pivot(col, values)`. When `values` is `None` 
the
     /// server computes the distinct values; when supplied they are serialized 
as
-    /// pivot-value literals (previously they were dropped entirely).
-    pub fn pivot(&self, pivot_col: Column, values: 
Option<Vec<crate::row::Value>>) -> GroupedData {
+    /// pivot-value literals. As PySpark's, only the grouping of `group_by` 
pivots: that of
+    /// a rollup, a cube, grouping sets or a pivot is the error of 
[`GroupedData::check_pivot`].
+    pub fn pivot(
+        &self,
+        pivot_col: Column,
+        values: Option<Vec<crate::row::Value>>,
+    ) -> Result<GroupedData> {
+        self.check_pivot()?;
         let pivot_values = values
             .unwrap_or_default()
             .into_iter()
             .map(value_to_lit_expr)
             .collect();
-        GroupedData {
+        Ok(GroupedData {
             dataframe: self.dataframe.clone(),
             group_cols: self.group_cols.clone(),
             group_type: AggregateGroupType::Pivot,
             pivot_col: Some(pivot_col.expression().clone()),
             pivot_values,
             grouping_sets: vec![],
-        }
+        })
+    }
+
+    /// Whether this grouping pivots, as PySpark's `GroupedData.pivot` checks 
before its
+    /// arguments: only that of `group_by` does. That of a rollup, a cube or 
grouping sets is
+    /// an `UNSUPPORTED_OPERATION` error of "PIVOT after ROLLUP", "PIVOT after 
CUBE" or
+    /// "PIVOT after GROUPING_SETS", and that of a pivot of "Repeated PIVOT 
operation".
+    pub fn check_pivot(&self) -> Result<()> {
+        let operation = match self.group_type {
+            AggregateGroupType::GroupBy => return Ok(()),
+            AggregateGroupType::Pivot => "Repeated PIVOT operation",
+            AggregateGroupType::Rollup => "PIVOT after ROLLUP",
+            AggregateGroupType::Cube => "PIVOT after CUBE",
+            AggregateGroupType::GroupingSets => "PIVOT after GROUPING_SETS",
+        };
+        Err(SparkError::classed(
+            SparkErrorKind::NotImplementedError,
+            "UNSUPPORTED_OPERATION",
+            &[("operation", operation)],
+        ))
     }
 
     /// Column names of the underlying (pre-grouping) DataFrame. These are 
passed as the
@@ -986,11 +1011,39 @@ mod tests {
         let spark = session();
         let df = spark.range(10).unwrap();
         let grouped = df.group_by(vec![crate::column::col("id")]);
-        let pivot_grouped = grouped.pivot(crate::column::col("category"), 
None);
+        let pivot_grouped = grouped.pivot(crate::column::col("category"), 
None).unwrap();
         assert_eq!(pivot_grouped.group_type, AggregateGroupType::Pivot);
         assert!(pivot_grouped.pivot_col.is_some());
     }
 
+    #[test]
+    fn only_the_grouping_of_group_by_pivots() {
+        let df = session().range(1).unwrap();
+        let id = || vec![crate::column::col("id")];
+        let pivot = |grouped: GroupedData| 
grouped.pivot(crate::column::col("id"), None);
+        assert!(pivot(df.group_by(id())).is_ok());
+        for (grouped, operation) in [
+            (df.rollup(id()), "PIVOT after ROLLUP"),
+            (df.cube(id()), "PIVOT after CUBE"),
+            (df.grouping_sets(vec![id()]), "PIVOT after GROUPING_SETS"),
+            (
+                pivot(df.group_by(id())).unwrap(),
+                "Repeated PIVOT operation",
+            ),
+        ] {
+            let error = pivot(grouped).err().expect("an error");
+            assert_eq!(error.kind, SparkErrorKind::NotImplementedError);
+            assert_eq!(
+                error.get_condition().as_deref(),
+                Some("UNSUPPORTED_OPERATION")
+            );
+            assert_eq!(
+                error.get_message_parameters().unwrap()["operation"],
+                operation
+            );
+        }
+    }
+
     #[test]
     fn pivot_day_time_intervals_are_exact_literals() {
         use spark_connect_proto::{expression::literal::LiteralType, 
relation::RelType};
@@ -1007,6 +1060,7 @@ mod tests {
                         .collect(),
                 ),
             )
+            .unwrap()
             .count();
         let Some(RelType::Aggregate(aggregate)) = 
pivot.plan.to_proto().rel_type else {
             panic!("expected Aggregate");
diff --git a/crates/spark-connect/tests/e2e_coverage.rs 
b/crates/spark-connect/tests/e2e_coverage.rs
index 1edaf59..8e13236 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -394,6 +394,7 @@ fn agg_na_stat_reshape_surface() {
     let _ = df
         .group_by(vec![col("id")])
         .pivot(col("name"), None)
+        .unwrap()
         .count()
         .collect();
 
diff --git a/crates/spark-connect/tests/e2e_ops_breadth.rs 
b/crates/spark-connect/tests/e2e_ops_breadth.rs
index d95453b..c66f124 100644
--- a/crates/spark-connect/tests/e2e_ops_breadth.rs
+++ b/crates/spark-connect/tests/e2e_ops_breadth.rs
@@ -188,6 +188,7 @@ fn aggregates_execute() {
     let _ = df
         .group_by(vec![col("id")])
         .pivot(col("name"), None)
+        .unwrap()
         .agg(vec![expr(f::sum(col("v")))])
         .collect();
     let _ = df.agg(vec![expr(f::sum(col("v")))]).collect().unwrap();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 6261cd2..85c6dd7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1118,6 +1118,30 @@ def test_collected_rows_behave_as_pyspark_rows(session):
     assert repr(pickle.loads(pickle.dumps(duplicate))) == "Row(a=1, a=2)"
 
 
+def test_pivot_pivots_only_a_group_by_as_pyspark(session):
+    # As PySpark's: the pivot of a rollup, a cube, grouping sets or a pivot 
raises
+    # UNSUPPORTED_OPERATION, rather than pivoting as a groupBy, which pivots.
+    from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
+
+    df = session.createDataFrame(
+        [(1, 10, "a"), (1, 20, "b"), (2, 30, "a")], "k int, v int, s string"
+    )
+    for grouped in [
+        df.rollup("k"),
+        df.cube("k"),
+        df.groupingSets([["k"]], "k"),
+        df.groupBy("k").pivot("s"),
+    ]:
+        with pytest.raises(PySparkNotImplementedError) as e:
+            grouped.pivot("s")
+        assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+    rows = df.groupBy("k").pivot("s", ["a", 
"b"]).sum("v").orderBy("k").collect()
+    assert [tuple(row) for row in rows] == [(1, 10, 20), (2, 30, None)]
+    with pytest.raises(PySparkTypeError) as e:
+        df.groupBy("k").pivot("s", ("a",))
+    assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+
+
 def test_sample_sort_group_by_drop_and_select_expr_as_pyspark(session):
     # As PySpark's: sample takes a fraction and a seed positionally, sort and 
groupBy take the
     # 1-based ordinals of columns, drop takes Columns, and selectExpr a list 
of expressions.
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index a24f545..73cf3e8 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -3001,6 +3001,45 @@ def test_functions_substring():
     assert type(F.substr(str_col, F.lit(1), F.lit(5))).__name__ == "Column"
 
 
+def test_pivot_pivots_only_a_group_by_and_checks_its_arguments_as_pyspark():
+    # As PySpark's GroupedData.pivot: only the grouping of groupBy pivots, 
which it checks
+    # first, and then the column is a str and the values a list of bools, 
floats, ints and
+    # strs.
+    from pyspark.errors import PySparkNotImplementedError, PySparkTypeError
+    from pyspark.sql import functions as F
+
+    df = _offline_session().range(1)
+    for grouped, operation in [
+        (df.rollup("id"), "PIVOT after ROLLUP"),
+        (df.cube("id"), "PIVOT after CUBE"),
+        (df.groupingSets([["id"]], "id"), "PIVOT after GROUPING_SETS"),
+        (df.groupBy("id").pivot("id"), "Repeated PIVOT operation"),
+    ]:
+        with pytest.raises(PySparkNotImplementedError) as e:
+            grouped.pivot(1)
+        assert e.value.getCondition() == "UNSUPPORTED_OPERATION"
+        assert e.value.getMessageParameters() == {"operation": operation}
+    grouped = df.groupBy("id")
+    for args, arg_name, expected_type, arg_type in [
+        ((1,), "pivot_col", "str", "int"),
+        ((F.col("id"),), "pivot_col", "str", "Column"),
+        (("id", "a"), "values", "list", "str"),
+        (("id", ("a",)), "values", "list", "tuple"),
+        (("id", [F.lit(1)]), "value", "bool, float, int or str", "Column"),
+        (("id", [None]), "value", "bool, float, int or str", "NoneType"),
+    ]:
+        with pytest.raises(PySparkTypeError) as e:
+            grouped.pivot(*args)
+        assert e.value.getCondition() == "NOT_EXPECTED_TYPE"
+        assert e.value.getMessageParameters() == {
+            "arg_name": arg_name,
+            "expected_type": expected_type,
+            "arg_type": arg_type,
+        }
+    for values in [None, [], [True, 1.5, 2, "a"]]:
+        grouped.pivot("id", values)
+
+
 def test_replace_and_unpivot_check_their_arguments_as_pyspark():
     # As PySpark's DataFrame.replace and unpivot, which check their arguments 
on the client.
     import warnings


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to