This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new fd9d661  [SPARK-61028][CONNECT][PYTHON] Name the column of 
GroupedData.count, aggregate every numeric column by default, and take a dict 
in DataFrame.agg, as PySpark does
fd9d661 is described below

commit fd9d6613e1085af879871f3c71e1202cee383ad1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 04:09:31 2026 +0900

    [SPARK-61028][CONNECT][PYTHON] Name the column of GroupedData.count, 
aggregate every numeric column by default, and take a dict in DataFrame.agg, as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61028](https://issues.apache.org/jira/browse/SPARK-61028): the 
aggregations of `GroupedData` and `DataFrame.agg` now name and take their 
columns as PySpark's do.
    
    - `df.groupBy(...).count()` names its column `count`, as it aggregates 
`count(lit(1)).alias("count")`.
    - `sum`, `avg`, `mean`, `max` and `min` of a GroupedData aggregate every 
numeric column of the DataFrame when no column is given. For a column that is 
not a numeric one of the schema, they raise `NOT_NUMERIC_COLUMNS`, as PySpark's 
`verify_numeric_col_name` finds it from the parts of the name. They assert that 
the columns are strings.
    - `GroupedData.agg` asserts that it is given expressions, which are Columns 
unless they are one dict.
    - `df.agg` raises `CANNOT_BE_EMPTY` without expressions, and aggregates a 
dict of column names and functions. It asserts that other expressions are 
Columns.
    
    In the Rust API, `GroupedData::numeric_agg` aggregates numeric columns as 
PySpark's `_numeric_agg`, and `GroupedData::count` names its column `count`.
    
    ### Why are the changes needed?
    
    - The column of `count()` was named `count(1)`, so 
`df.groupBy("k").count().orderBy("count")` failed.
    - `sum()` and the others aggregated no column when none was given.
    - `df.groupBy("k").sum("k")` failed on the server.
    - `df.agg({"x": "max"})` raised a `TypeError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These aggregations name their columns, and take their arguments, as 
PySpark 4.3.0 does.
    
    ### How was this patch tested?
    
    - A new live test checks the names and the results of these aggregations, 
and `NOT_NUMERIC_COLUMNS`. It passes with official PySpark 4.3.0, and fails 
without this change.
    - A new offline test checks the aggregations sent to a mock Connect server 
and the errors. It fails without this change.
    - A new lib test checks the parsing of attribute names and the numeric 
check of nested fields, against PySpark's `parse_attr_name`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #228 from HyukjinKwon/SPARK-61028-grouped.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs     |  36 +++++-
 crates/pyspark-rs/src/group.rs         |  91 ++++++++-------
 crates/spark-connect/src/expression.rs |   2 +-
 crates/spark-connect/src/group.rs      | 207 ++++++++++++++++++++++++++++++++-
 python/tests/test_dropin_e2e.py        |  18 +++
 python/tests/test_dropin_offline.py    |  46 ++++++++
 6 files changed, 351 insertions(+), 49 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 63bfc59..3750db0 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1452,9 +1452,39 @@ impl PyDataFrame {
     ///
     /// Mirrors `pyspark.sql.DataFrame.agg(*exprs)`.
     #[pyo3(signature = (*exprs))]
-    fn agg(&self, _py: Python<'_>, exprs: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let columns = to_column_list(exprs)?;
-        let expressions = columns.iter().map(|c| 
c.expression().clone()).collect();
+    fn agg(&self, py: Python<'_>, exprs: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        if exprs.is_empty() {
+            return Err(crate::column::raise_pyspark(
+                py,
+                "PySparkValueError",
+                "CANNOT_BE_EMPTY",
+                &[("item", "exprs")],
+            ));
+        }
+        // As PySpark's: a dict of column names and functions, the aggregation 
of each, as of
+        // `groupBy().agg`, else Columns, which it asserts.
+        if let [only] = exprs.as_slice() {
+            if let Ok(dict) = only.cast::<PyDict>() {
+                let mut expressions = Vec::with_capacity(dict.len());
+                for (column, function) in dict.iter() {
+                    let column = 
spark_connect::functions::col(&column.extract::<String>()?);
+                    
expressions.push(spark_connect::expression::Expression::function(
+                        function.extract::<String>()?,
+                        vec![column.expression().clone()],
+                    ));
+                }
+                return Ok(PyDataFrame::new(self.dataframe.agg(expressions)));
+            }
+        }
+        let mut expressions = Vec::with_capacity(exprs.len());
+        for expr in &exprs {
+            let Ok(column) = expr.extract::<PyRef<'_, PyColumn>>() else {
+                return Err(pyo3::exceptions::PyAssertionError::new_err(
+                    "all exprs should be Expression",
+                ));
+            };
+            expressions.push(column.column.expression().clone());
+        }
         Ok(PyDataFrame::new(self.dataframe.agg(expressions)))
     }
 
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 8b7d4ad..a7463f2 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -8,7 +8,6 @@ use crate::dataframe::{
     build_map_udf, infer_group_map_eval_type, validate_map_function, 
PyDataFrame,
 };
 use crate::errors::ResultExt;
-use crate::functions::to_column;
 
 /// Python wrapper for Spark GroupedData.
 #[pyclass(name = "GroupedData", module = "pyspark.sql.group")]
@@ -22,12 +21,43 @@ impl PyGroupedData {
     }
 }
 
+impl PyGroupedData {
+    /// `sum`, `avg`, `mean`, `min` and `max`, as PySpark's `_numeric_agg`: of 
column names,
+    /// which it asserts, every numeric column without any, and 
`NOT_NUMERIC_COLUMNS` for a
+    /// column that is not numeric.
+    fn numeric_agg(
+        &self,
+        py: Python<'_>,
+        function: &str,
+        cols: Vec<Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        let mut names = Vec::with_capacity(cols.len());
+        for col in &cols {
+            if !col.is_instance_of::<pyo3::types::PyString>() {
+                return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+            }
+            names.push(col.extract::<String>()?);
+        }
+        let refs: Vec<&str> = names.iter().map(String::as_str).collect();
+        let grouped = &self.grouped_data;
+        py.detach(|| grouped.numeric_agg(function, refs).map_err(Box::new))
+            .to_pyerr()
+            .map(PyDataFrame::new)
+    }
+}
+
 #[pymethods]
 impl PyGroupedData {
     /// Aggregate with expressions (accepts Column objects), or the dict form
     /// `agg({"col": "aggfunc"})` (PySpark parity), e.g. `agg({"age": "max"})`.
     #[pyo3(signature = (*exprs))]
     fn agg(&self, _py: Python<'_>, exprs: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        // As PySpark's, which asserts that it has expressions.
+        if exprs.is_empty() {
+            return Err(pyo3::exceptions::PyAssertionError::new_err(
+                "exprs should not be empty",
+            ));
+        }
         // Dict form: {column_name: aggregate_function_name} -> 
func(col(name)) per entry.
         if exprs.len() == 1 {
             if let Ok(dict) = exprs[0].cast::<pyo3::types::PyDict>() {
@@ -51,10 +81,15 @@ impl PyGroupedData {
                 return Ok(PyDataFrame::new(df));
             }
         }
+        // As PySpark's, which asserts that the expressions are Columns.
         let mut agg_exprs = vec![];
         for col in exprs {
-            let rust_col = to_column(&col)?;
-            agg_exprs.push(rust_col.expression().clone());
+            let Ok(col) = col.extract::<PyRef<'_, crate::column::PyColumn>>() 
else {
+                return Err(pyo3::exceptions::PyAssertionError::new_err(
+                    "all exprs should be Column",
+                ));
+            };
+            agg_exprs.push(col.column.expression().clone());
         }
         let df = self.grouped_data.agg(agg_exprs);
         Ok(PyDataFrame::new(df))
@@ -89,62 +124,32 @@ impl PyGroupedData {
 
     /// Sum values in each group.
     #[pyo3(signature = (*cols))]
-    fn sum(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let col_names: Vec<String> = cols
-            .iter()
-            .map(|c| c.extract::<String>())
-            .collect::<PyResult<Vec<_>>>()?;
-        let col_refs: Vec<&str> = col_names.iter().map(|s| 
s.as_str()).collect();
-        let df = self.grouped_data.sum(col_refs);
-        Ok(PyDataFrame::new(df))
+    fn sum(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        self.numeric_agg(py, "sum", cols)
     }
 
     /// Average values in each group.
     #[pyo3(signature = (*cols))]
-    fn avg(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let col_names: Vec<String> = cols
-            .iter()
-            .map(|c| c.extract::<String>())
-            .collect::<PyResult<Vec<_>>>()?;
-        let col_refs: Vec<&str> = col_names.iter().map(|s| 
s.as_str()).collect();
-        let df = self.grouped_data.avg(col_refs);
-        Ok(PyDataFrame::new(df))
+    fn avg(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        self.numeric_agg(py, "avg", cols)
     }
 
     /// Minimum values in each group.
     #[pyo3(signature = (*cols))]
-    fn min(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let col_names: Vec<String> = cols
-            .iter()
-            .map(|c| c.extract::<String>())
-            .collect::<PyResult<Vec<_>>>()?;
-        let col_refs: Vec<&str> = col_names.iter().map(|s| 
s.as_str()).collect();
-        let df = self.grouped_data.min(col_refs);
-        Ok(PyDataFrame::new(df))
+    fn min(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        self.numeric_agg(py, "min", cols)
     }
 
     /// Maximum values in each group.
     #[pyo3(signature = (*cols))]
-    fn max(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let col_names: Vec<String> = cols
-            .iter()
-            .map(|c| c.extract::<String>())
-            .collect::<PyResult<Vec<_>>>()?;
-        let col_refs: Vec<&str> = col_names.iter().map(|s| 
s.as_str()).collect();
-        let df = self.grouped_data.max(col_refs);
-        Ok(PyDataFrame::new(df))
+    fn max(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        self.numeric_agg(py, "max", cols)
     }
 
     /// Mean (alias for avg).
     #[pyo3(signature = (*cols))]
-    fn mean(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
-        let col_names: Vec<String> = cols
-            .iter()
-            .map(|c| c.extract::<String>())
-            .collect::<PyResult<Vec<_>>>()?;
-        let col_refs: Vec<&str> = col_names.iter().map(|s| 
s.as_str()).collect();
-        let df = self.grouped_data.mean(col_refs);
-        Ok(PyDataFrame::new(df))
+    fn mean(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        self.numeric_agg(py, "avg", cols)
     }
 
     /// `GroupedData.applyInPandas`.
diff --git a/crates/spark-connect/src/expression.rs 
b/crates/spark-connect/src/expression.rs
index c96787b..d2774da 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -1431,7 +1431,7 @@ fn python_bytes_repr(bytes: &[u8]) -> String {
 }
 
 /// Python's `repr` of a string, as a list shows its strings.
-fn python_str_repr(value: &str) -> String {
+pub(crate) fn python_str_repr(value: &str) -> String {
     let quote = python_quote(value.contains('\''), value.contains('"'));
     let mut out = String::new();
     out.push(quote);
diff --git a/crates/spark-connect/src/group.rs 
b/crates/spark-connect/src/group.rs
index 2840eb2..2332c37 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -4,7 +4,7 @@
 
 use std::sync::Arc;
 
-use spark_connect_core::error::Result;
+use spark_connect_core::error::{Result, SparkError, SparkErrorKind};
 
 use crate::column::Column;
 use crate::dataframe::DataFrame;
@@ -299,18 +299,63 @@ impl GroupedData {
         DataFrame::new(self.dataframe.session.clone(), plan)
     }
 
-    /// Count rows in each group.
+    /// Count rows in each group, in a column `count`, mirroring 
`GroupedData.count`, which
+    /// aggregates `count(lit(1)).alias("count")`.
     pub fn count(&self) -> DataFrame {
         use crate::functions;
         let count_expr = functions::count(Column::new(Expression::literal(
             crate::expression::LiteralExpression::int(1),
         )))
+        .alias("count")
         .expression()
         .clone();
 
         self.agg(vec![count_expr])
     }
 
+    /// Aggregate the numeric `columns` by `function` ("min", "max", "avg" or 
"sum") in each
+    /// group, mirroring `GroupedData._numeric_agg` of `min`, `max`, `avg`, 
`mean` and `sum`:
+    /// every numeric column of the DataFrame without columns, and 
`NOT_NUMERIC_COLUMNS` for a
+    /// column that is not a numeric one of its schema.
+    pub fn numeric_agg(&self, function: &str, columns: Vec<&str>) -> 
Result<DataFrame> {
+        let schema = self.dataframe.schema()?;
+        let names: Vec<String> = if columns.is_empty() {
+            match &schema {
+                DataType::Struct { fields } => fields
+                    .iter()
+                    .filter(|field| is_numeric(&field.data_type))
+                    .map(|field| field.name.clone())
+                    .collect(),
+                _ => Vec::new(),
+            }
+        } else {
+            let mut invalid = Vec::new();
+            for column in &columns {
+                if !verify_numeric_col_name(column, &schema)? {
+                    invalid.push(crate::expression::python_str_repr(column));
+                }
+            }
+            if !invalid.is_empty() {
+                return Err(SparkError::classed(
+                    SparkErrorKind::TypeError,
+                    "NOT_NUMERIC_COLUMNS",
+                    &[("invalid_columns", &format!("[{}]", invalid.join(", 
")))],
+                ));
+            }
+            columns.iter().map(|column| column.to_string()).collect()
+        };
+        let expressions = names
+            .iter()
+            .map(|name| {
+                Expression::function(
+                    function,
+                    vec![crate::column::col(name).expression().clone()],
+                )
+            })
+            .collect();
+        Ok(self.agg(expressions))
+    }
+
     /// Sum values in each group.
     pub fn sum(&self, columns: Vec<&str>) -> DataFrame {
         use crate::functions;
@@ -361,6 +406,98 @@ impl GroupedData {
     }
 }
 
+/// Whether `data_type` is a `NumericType` of PySpark.
+fn is_numeric(data_type: &DataType) -> bool {
+    matches!(
+        data_type,
+        DataType::Byte
+            | DataType::Short
+            | DataType::Integer
+            | DataType::Long
+            | DataType::Float
+            | DataType::Double
+            | DataType::Decimal { .. }
+    )
+}
+
+/// The parts of an attribute name, as PySpark's `parse_attr_name`: split by 
dots, of
+/// backtick-quoted parts too; None for a name that it does not parse.
+fn parse_attr_name(name: &str) -> Option<Vec<String>> {
+    let chars: Vec<char> = name.chars().collect();
+    let n = chars.len();
+    let (mut parts, mut part, mut in_backtick, mut i) = (Vec::new(), 
String::new(), false, 0);
+    while i < n {
+        let c = chars[i];
+        if in_backtick {
+            if c == '`' {
+                if i + 1 < n && chars[i + 1] == '`' {
+                    part.push('`');
+                    i += 1;
+                } else {
+                    in_backtick = false;
+                    if i + 1 < n && chars[i + 1] != '.' {
+                        return None;
+                    }
+                }
+            } else {
+                part.push(c);
+            }
+        } else if c == '`' {
+            if !part.is_empty() {
+                return None;
+            }
+            in_backtick = true;
+        } else if c == '.' {
+            // As Python's `name[i - 1]`, the last character for the first.
+            let previous = chars[(i + n - 1) % n];
+            if previous == '.' || i == n - 1 {
+                return None;
+            }
+            parts.push(std::mem::take(&mut part));
+        } else {
+            part.push(c);
+        }
+        i += 1;
+    }
+    if in_backtick {
+        return None;
+    }
+    parts.push(part);
+    Some(parts)
+}
+
+/// Whether `name` is of a numeric column of `schema`, as PySpark's
+/// `verify_numeric_col_name`: by its parts, of the fields of structs and of 
the structs of
+/// arrays. As PySpark's, the name of a struct is an `IndexError`.
+fn verify_numeric_col_name(name: &str, schema: &DataType) -> Result<bool> {
+    fn verify(parts: &[String], data_type: &DataType) -> Result<bool> {
+        if parts.is_empty() && is_numeric(data_type) {
+            return Ok(true);
+        }
+        let fields = match data_type {
+            DataType::Struct { fields } => fields,
+            DataType::Array { element_type, .. } => match 
element_type.as_ref() {
+                DataType::Struct { fields } => fields,
+                _ => return Ok(false),
+            },
+            _ => return Ok(false),
+        };
+        let Some((part, rest)) = parts.split_first() else {
+            let mut err = SparkError::value_msg("list index out of range");
+            err.kind = SparkErrorKind::IndexError;
+            return Err(err);
+        };
+        match fields.iter().find(|field| field.name == *part) {
+            Some(field) => verify(rest, &field.data_type),
+            None => Ok(false),
+        }
+    }
+    match parse_attr_name(name) {
+        Some(parts) if !parts.is_empty() => verify(&parts, schema),
+        _ => Ok(false),
+    }
+}
+
 /// Statistical functions for DataFrames.
 ///
 /// Mirrors `pyspark.sql.DataFrameStatFunctions`.
@@ -567,6 +704,72 @@ mod tests {
             .expect("failed to build session")
     }
 
+    #[test]
+    fn numeric_column_names_are_verified_as_pyspark() {
+        use crate::types::StructField;
+        let field = |name: &str, data_type: DataType| StructField {
+            name: name.to_string(),
+            data_type,
+            nullable: true,
+            metadata: Default::default(),
+        };
+        let schema = DataType::Struct {
+            fields: vec![
+                field("x", DataType::Integer),
+                field(
+                    "k",
+                    DataType::String {
+                        collation: "UTF8_BINARY".to_string(),
+                    },
+                ),
+                field(
+                    "st",
+                    DataType::Struct {
+                        fields: vec![field("a", DataType::Double)],
+                    },
+                ),
+                field(
+                    "arr",
+                    DataType::Array {
+                        element_type: Box::new(DataType::Struct {
+                            fields: vec![field("c", DataType::Long)],
+                        }),
+                        contains_null: true,
+                    },
+                ),
+            ],
+        };
+        // As PySpark's parse_attr_name of these.
+        assert_eq!(
+            parse_attr_name("a.`b.c`.`d``e`").unwrap(),
+            vec!["a", "b.c", "d`e"]
+        );
+        assert_eq!(parse_attr_name(".a").unwrap(), vec!["", "a"]);
+        assert_eq!(parse_attr_name("`a.b`").unwrap(), vec!["a.b"]);
+        for unparsed in ["a..b", "a.", "`a`b", "a`b`", "`a", "a.`b.c`.``d``"] {
+            assert_eq!(parse_attr_name(unparsed), None, "{unparsed}");
+        }
+        for (name, numeric) in [
+            ("x", true),
+            ("k", false),
+            ("st.a", true),
+            ("`st`.`a`", true),
+            ("arr.c", true),
+            ("nope", false),
+            ("st..a", false),
+        ] {
+            assert_eq!(
+                verify_numeric_col_name(name, &schema).unwrap(),
+                numeric,
+                "{name}"
+            );
+        }
+        // As PySpark's, of which a struct is an IndexError.
+        let err = verify_numeric_col_name("st", &schema).unwrap_err();
+        assert_eq!(err.kind, SparkErrorKind::IndexError);
+        assert_eq!(err.message(), "list index out of range");
+    }
+
     #[test]
     fn grouped_and_stat_builders_share_the_source_plan() {
         let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e11b270..143995b 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -538,6 +538,24 @@ def test_create_dataframe_from_empty_arrow_table(session):
 _Pair = namedtuple("_Pair", ["x", "y"])
 
 
+def test_grouped_aggregations_name_and_take_columns_as_pyspark(session):
+    # As PySpark's: count is named count, the numeric aggregations of no 
columns aggregate
+    # every numeric column, and df.agg takes a dict.
+    from pyspark.errors import PySparkTypeError
+
+    df = session.createDataFrame([(1, "b", 2.0), (2, "b", 3.0)], "x int, k 
string, v double")
+    grouped = df.groupBy("k")
+    assert grouped.count().columns == ["k", "count"]
+    assert grouped.count().orderBy("count").collect() == [Row(k="b", count=2)]
+    assert grouped.sum().collect() == [Row(k="b", **{"sum(x)": 3, "sum(v)": 
5.0})]
+    assert grouped.min().columns == ["k", "min(x)", "min(v)"]
+    assert df.groupBy().count().collect() == [Row(count=2)]
+    assert df.agg({"x": "max"}).collect() == [Row(**{"max(x)": 2})]
+    with pytest.raises(PySparkTypeError) as error:
+        grouped.avg("k")
+    assert error.value.getMessageParameters() == {"invalid_columns": "['k']"}
+
+
 @pytest.mark.parametrize(
     "data, schema, columns, rows",
     [
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 75cd9f9..b7d675c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5719,6 +5719,52 @@ def 
test_schema_cache_derived_and_new_plans_resolve_independently(mock_connect):
     assert df.columns == ["id"]
 
 
+def test_grouped_aggregations_as_pyspark(mock_connect):
+    # As PySpark's GroupedData: count is named count, sum, avg, mean, min and 
max aggregate
+    # every numeric column without columns, and the aggregations check their 
arguments.
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+    from pyspark.sql import functions as F
+
+    spark, service = mock_connect
+    df = spark.range(1)
+
+    def aggregations(dataframe):
+        # ExecutePlanRequest.plan=3, Plan.root=1, Relation.aggregate=9,
+        # Aggregate.aggregate_expressions=4.
+        dataframe.collect()
+        plan = 
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+        return parse_message(parse_message(plan)[9][0])[4]
+
+    # Expression.alias=6: Alias.expr=1, name=2; 
Expression.unresolved_function=3:
+    # UnresolvedFunction.function_name=1, arguments=2.
+    (count,) = aggregations(df.groupBy("id").count())
+    alias = parse_message(parse_message(count)[6][0])
+    assert alias[2] == [b"count"]
+    assert parse_message(parse_message(alias[1][0])[3][0])[1] == [b"count"]
+    for aggregate, function in ((lambda g: g.sum(), b"sum"), (lambda g: 
g.mean(), b"avg"),
+                                (lambda g: g.max("id"), b"max")):
+        (expression,) = aggregations(aggregate(df.groupBy("id")))
+        assert parse_message(parse_message(expression)[3][0])[1] == [function]
+    (expression,) = aggregations(df.agg({"id": "min"}))
+    assert parse_message(parse_message(expression)[3][0])[1] == [b"min"]
+
+    with pytest.raises(PySparkTypeError) as error:
+        df.groupBy("id").sum("nope")
+    assert error.value.getCondition() == "NOT_NUMERIC_COLUMNS"
+    assert error.value.getMessageParameters() == {"invalid_columns": 
"['nope']"}
+    with pytest.raises(PySparkValueError) as error:
+        df.agg()
+    assert error.value.getCondition() == "CANNOT_BE_EMPTY"
+    for call, message in ((lambda: df.groupBy("id").agg(), "exprs should not 
be empty"),
+                          (lambda: df.groupBy("id").agg("id"), "all exprs 
should be Column"),
+                          (lambda: df.agg("id"), "all exprs should be 
Expression"),
+                          (lambda: df.groupBy("id").sum(1), "")):
+        with pytest.raises(AssertionError) as error:
+            call()
+        assert str(error.value) == message
+    assert df.groupBy("id").agg(F.count("id")) is not None
+
+
 @pytest.mark.parametrize("transform", [
     pytest.param(lambda df: df.filter("id > 0"), id="filter"),
     pytest.param(lambda df: df.limit(2), id="limit"),


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to