This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new fd9d661 [SPARK-61028][CONNECT][PYTHON] Name the column of
GroupedData.count, aggregate every numeric column by default, and take a dict
in DataFrame.agg, as PySpark does
fd9d661 is described below
commit fd9d6613e1085af879871f3c71e1202cee383ad1
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 04:09:31 2026 +0900
[SPARK-61028][CONNECT][PYTHON] Name the column of GroupedData.count,
aggregate every numeric column by default, and take a dict in DataFrame.agg, as
PySpark does
### What changes were proposed in this pull request?
[SPARK-61028](https://issues.apache.org/jira/browse/SPARK-61028): the
aggregations of `GroupedData` and `DataFrame.agg` now name and take their
columns as PySpark's do.
- `df.groupBy(...).count()` names its column `count`, as it aggregates
`count(lit(1)).alias("count")`.
- `sum`, `avg`, `mean`, `max` and `min` of a GroupedData aggregate every
numeric column of the DataFrame when no column is given. For a column that is
not a numeric one of the schema, they raise `NOT_NUMERIC_COLUMNS`, as PySpark's
`verify_numeric_col_name` finds it from the parts of the name. They assert that
the columns are strings.
- `GroupedData.agg` asserts that it is given expressions, which are Columns
unless they are one dict.
- `df.agg` raises `CANNOT_BE_EMPTY` without expressions, and aggregates a
dict of column names and functions. It asserts that other expressions are
Columns.
In the Rust API, `GroupedData::numeric_agg` aggregates numeric columns as
PySpark's `_numeric_agg`, and `GroupedData::count` names its column `count`.
### Why are the changes needed?
- The column of `count()` was named `count(1)`, so
`df.groupBy("k").count().orderBy("count")` failed.
- `sum()` and the others aggregated no column when none was given.
- `df.groupBy("k").sum("k")` failed on the server.
- `df.agg({"x": "max"})` raised a `TypeError`.
### Does this PR introduce _any_ user-facing change?
Yes. These aggregations name their columns, and take their arguments, as
PySpark 4.3.0 does.
### How was this patch tested?
- A new live test checks the names and the results of these aggregations,
and `NOT_NUMERIC_COLUMNS`. It passes with official PySpark 4.3.0, and fails
without this change.
- A new offline test checks the aggregations sent to a mock Connect server
and the errors. It fails without this change.
- A new lib test checks the parsing of attribute names and the numeric
check of nested fields, against PySpark's `parse_attr_name`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #228 from HyukjinKwon/SPARK-61028-grouped.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 36 +++++-
crates/pyspark-rs/src/group.rs | 91 ++++++++-------
crates/spark-connect/src/expression.rs | 2 +-
crates/spark-connect/src/group.rs | 207 ++++++++++++++++++++++++++++++++-
python/tests/test_dropin_e2e.py | 18 +++
python/tests/test_dropin_offline.py | 46 ++++++++
6 files changed, 351 insertions(+), 49 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 63bfc59..3750db0 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1452,9 +1452,39 @@ impl PyDataFrame {
///
/// Mirrors `pyspark.sql.DataFrame.agg(*exprs)`.
#[pyo3(signature = (*exprs))]
- fn agg(&self, _py: Python<'_>, exprs: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let columns = to_column_list(exprs)?;
- let expressions = columns.iter().map(|c|
c.expression().clone()).collect();
+ fn agg(&self, py: Python<'_>, exprs: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ if exprs.is_empty() {
+ return Err(crate::column::raise_pyspark(
+ py,
+ "PySparkValueError",
+ "CANNOT_BE_EMPTY",
+ &[("item", "exprs")],
+ ));
+ }
+ // As PySpark's: a dict of column names and functions, the aggregation
of each, as of
+ // `groupBy().agg`, else Columns, which it asserts.
+ if let [only] = exprs.as_slice() {
+ if let Ok(dict) = only.cast::<PyDict>() {
+ let mut expressions = Vec::with_capacity(dict.len());
+ for (column, function) in dict.iter() {
+ let column =
spark_connect::functions::col(&column.extract::<String>()?);
+
expressions.push(spark_connect::expression::Expression::function(
+ function.extract::<String>()?,
+ vec![column.expression().clone()],
+ ));
+ }
+ return Ok(PyDataFrame::new(self.dataframe.agg(expressions)));
+ }
+ }
+ let mut expressions = Vec::with_capacity(exprs.len());
+ for expr in &exprs {
+ let Ok(column) = expr.extract::<PyRef<'_, PyColumn>>() else {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(
+ "all exprs should be Expression",
+ ));
+ };
+ expressions.push(column.column.expression().clone());
+ }
Ok(PyDataFrame::new(self.dataframe.agg(expressions)))
}
diff --git a/crates/pyspark-rs/src/group.rs b/crates/pyspark-rs/src/group.rs
index 8b7d4ad..a7463f2 100644
--- a/crates/pyspark-rs/src/group.rs
+++ b/crates/pyspark-rs/src/group.rs
@@ -8,7 +8,6 @@ use crate::dataframe::{
build_map_udf, infer_group_map_eval_type, validate_map_function,
PyDataFrame,
};
use crate::errors::ResultExt;
-use crate::functions::to_column;
/// Python wrapper for Spark GroupedData.
#[pyclass(name = "GroupedData", module = "pyspark.sql.group")]
@@ -22,12 +21,43 @@ impl PyGroupedData {
}
}
+impl PyGroupedData {
+ /// `sum`, `avg`, `mean`, `min` and `max`, as PySpark's `_numeric_agg`: of
column names,
+ /// which it asserts, every numeric column without any, and
`NOT_NUMERIC_COLUMNS` for a
+ /// column that is not numeric.
+ fn numeric_agg(
+ &self,
+ py: Python<'_>,
+ function: &str,
+ cols: Vec<Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ let mut names = Vec::with_capacity(cols.len());
+ for col in &cols {
+ if !col.is_instance_of::<pyo3::types::PyString>() {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(()));
+ }
+ names.push(col.extract::<String>()?);
+ }
+ let refs: Vec<&str> = names.iter().map(String::as_str).collect();
+ let grouped = &self.grouped_data;
+ py.detach(|| grouped.numeric_agg(function, refs).map_err(Box::new))
+ .to_pyerr()
+ .map(PyDataFrame::new)
+ }
+}
+
#[pymethods]
impl PyGroupedData {
/// Aggregate with expressions (accepts Column objects), or the dict form
/// `agg({"col": "aggfunc"})` (PySpark parity), e.g. `agg({"age": "max"})`.
#[pyo3(signature = (*exprs))]
fn agg(&self, _py: Python<'_>, exprs: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ // As PySpark's, which asserts that it has expressions.
+ if exprs.is_empty() {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(
+ "exprs should not be empty",
+ ));
+ }
// Dict form: {column_name: aggregate_function_name} ->
func(col(name)) per entry.
if exprs.len() == 1 {
if let Ok(dict) = exprs[0].cast::<pyo3::types::PyDict>() {
@@ -51,10 +81,15 @@ impl PyGroupedData {
return Ok(PyDataFrame::new(df));
}
}
+ // As PySpark's, which asserts that the expressions are Columns.
let mut agg_exprs = vec![];
for col in exprs {
- let rust_col = to_column(&col)?;
- agg_exprs.push(rust_col.expression().clone());
+ let Ok(col) = col.extract::<PyRef<'_, crate::column::PyColumn>>()
else {
+ return Err(pyo3::exceptions::PyAssertionError::new_err(
+ "all exprs should be Column",
+ ));
+ };
+ agg_exprs.push(col.column.expression().clone());
}
let df = self.grouped_data.agg(agg_exprs);
Ok(PyDataFrame::new(df))
@@ -89,62 +124,32 @@ impl PyGroupedData {
/// Sum values in each group.
#[pyo3(signature = (*cols))]
- fn sum(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let col_names: Vec<String> = cols
- .iter()
- .map(|c| c.extract::<String>())
- .collect::<PyResult<Vec<_>>>()?;
- let col_refs: Vec<&str> = col_names.iter().map(|s|
s.as_str()).collect();
- let df = self.grouped_data.sum(col_refs);
- Ok(PyDataFrame::new(df))
+ fn sum(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ self.numeric_agg(py, "sum", cols)
}
/// Average values in each group.
#[pyo3(signature = (*cols))]
- fn avg(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let col_names: Vec<String> = cols
- .iter()
- .map(|c| c.extract::<String>())
- .collect::<PyResult<Vec<_>>>()?;
- let col_refs: Vec<&str> = col_names.iter().map(|s|
s.as_str()).collect();
- let df = self.grouped_data.avg(col_refs);
- Ok(PyDataFrame::new(df))
+ fn avg(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ self.numeric_agg(py, "avg", cols)
}
/// Minimum values in each group.
#[pyo3(signature = (*cols))]
- fn min(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let col_names: Vec<String> = cols
- .iter()
- .map(|c| c.extract::<String>())
- .collect::<PyResult<Vec<_>>>()?;
- let col_refs: Vec<&str> = col_names.iter().map(|s|
s.as_str()).collect();
- let df = self.grouped_data.min(col_refs);
- Ok(PyDataFrame::new(df))
+ fn min(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ self.numeric_agg(py, "min", cols)
}
/// Maximum values in each group.
#[pyo3(signature = (*cols))]
- fn max(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let col_names: Vec<String> = cols
- .iter()
- .map(|c| c.extract::<String>())
- .collect::<PyResult<Vec<_>>>()?;
- let col_refs: Vec<&str> = col_names.iter().map(|s|
s.as_str()).collect();
- let df = self.grouped_data.max(col_refs);
- Ok(PyDataFrame::new(df))
+ fn max(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ self.numeric_agg(py, "max", cols)
}
/// Mean (alias for avg).
#[pyo3(signature = (*cols))]
- fn mean(&self, _py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
- let col_names: Vec<String> = cols
- .iter()
- .map(|c| c.extract::<String>())
- .collect::<PyResult<Vec<_>>>()?;
- let col_refs: Vec<&str> = col_names.iter().map(|s|
s.as_str()).collect();
- let df = self.grouped_data.mean(col_refs);
- Ok(PyDataFrame::new(df))
+ fn mean(&self, py: Python<'_>, cols: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ self.numeric_agg(py, "avg", cols)
}
/// `GroupedData.applyInPandas`.
diff --git a/crates/spark-connect/src/expression.rs
b/crates/spark-connect/src/expression.rs
index c96787b..d2774da 100644
--- a/crates/spark-connect/src/expression.rs
+++ b/crates/spark-connect/src/expression.rs
@@ -1431,7 +1431,7 @@ fn python_bytes_repr(bytes: &[u8]) -> String {
}
/// Python's `repr` of a string, as a list shows its strings.
-fn python_str_repr(value: &str) -> String {
+pub(crate) fn python_str_repr(value: &str) -> String {
let quote = python_quote(value.contains('\''), value.contains('"'));
let mut out = String::new();
out.push(quote);
diff --git a/crates/spark-connect/src/group.rs
b/crates/spark-connect/src/group.rs
index 2840eb2..2332c37 100644
--- a/crates/spark-connect/src/group.rs
+++ b/crates/spark-connect/src/group.rs
@@ -4,7 +4,7 @@
use std::sync::Arc;
-use spark_connect_core::error::Result;
+use spark_connect_core::error::{Result, SparkError, SparkErrorKind};
use crate::column::Column;
use crate::dataframe::DataFrame;
@@ -299,18 +299,63 @@ impl GroupedData {
DataFrame::new(self.dataframe.session.clone(), plan)
}
- /// Count rows in each group.
+ /// Count rows in each group, in a column `count`, mirroring
`GroupedData.count`, which
+ /// aggregates `count(lit(1)).alias("count")`.
pub fn count(&self) -> DataFrame {
use crate::functions;
let count_expr = functions::count(Column::new(Expression::literal(
crate::expression::LiteralExpression::int(1),
)))
+ .alias("count")
.expression()
.clone();
self.agg(vec![count_expr])
}
+ /// Aggregate the numeric `columns` by `function` ("min", "max", "avg" or
"sum") in each
+ /// group, mirroring `GroupedData._numeric_agg` of `min`, `max`, `avg`,
`mean` and `sum`:
+ /// every numeric column of the DataFrame without columns, and
`NOT_NUMERIC_COLUMNS` for a
+ /// column that is not a numeric one of its schema.
+ pub fn numeric_agg(&self, function: &str, columns: Vec<&str>) ->
Result<DataFrame> {
+ let schema = self.dataframe.schema()?;
+ let names: Vec<String> = if columns.is_empty() {
+ match &schema {
+ DataType::Struct { fields } => fields
+ .iter()
+ .filter(|field| is_numeric(&field.data_type))
+ .map(|field| field.name.clone())
+ .collect(),
+ _ => Vec::new(),
+ }
+ } else {
+ let mut invalid = Vec::new();
+ for column in &columns {
+ if !verify_numeric_col_name(column, &schema)? {
+ invalid.push(crate::expression::python_str_repr(column));
+ }
+ }
+ if !invalid.is_empty() {
+ return Err(SparkError::classed(
+ SparkErrorKind::TypeError,
+ "NOT_NUMERIC_COLUMNS",
+ &[("invalid_columns", &format!("[{}]", invalid.join(",
")))],
+ ));
+ }
+ columns.iter().map(|column| column.to_string()).collect()
+ };
+ let expressions = names
+ .iter()
+ .map(|name| {
+ Expression::function(
+ function,
+ vec![crate::column::col(name).expression().clone()],
+ )
+ })
+ .collect();
+ Ok(self.agg(expressions))
+ }
+
/// Sum values in each group.
pub fn sum(&self, columns: Vec<&str>) -> DataFrame {
use crate::functions;
@@ -361,6 +406,98 @@ impl GroupedData {
}
}
+/// Whether `data_type` is a `NumericType` of PySpark.
+fn is_numeric(data_type: &DataType) -> bool {
+ matches!(
+ data_type,
+ DataType::Byte
+ | DataType::Short
+ | DataType::Integer
+ | DataType::Long
+ | DataType::Float
+ | DataType::Double
+ | DataType::Decimal { .. }
+ )
+}
+
+/// The parts of an attribute name, as PySpark's `parse_attr_name`: split by
dots, of
+/// backtick-quoted parts too; None for a name that it does not parse.
+fn parse_attr_name(name: &str) -> Option<Vec<String>> {
+ let chars: Vec<char> = name.chars().collect();
+ let n = chars.len();
+ let (mut parts, mut part, mut in_backtick, mut i) = (Vec::new(),
String::new(), false, 0);
+ while i < n {
+ let c = chars[i];
+ if in_backtick {
+ if c == '`' {
+ if i + 1 < n && chars[i + 1] == '`' {
+ part.push('`');
+ i += 1;
+ } else {
+ in_backtick = false;
+ if i + 1 < n && chars[i + 1] != '.' {
+ return None;
+ }
+ }
+ } else {
+ part.push(c);
+ }
+ } else if c == '`' {
+ if !part.is_empty() {
+ return None;
+ }
+ in_backtick = true;
+ } else if c == '.' {
+ // As Python's `name[i - 1]`, the last character for the first.
+ let previous = chars[(i + n - 1) % n];
+ if previous == '.' || i == n - 1 {
+ return None;
+ }
+ parts.push(std::mem::take(&mut part));
+ } else {
+ part.push(c);
+ }
+ i += 1;
+ }
+ if in_backtick {
+ return None;
+ }
+ parts.push(part);
+ Some(parts)
+}
+
+/// Whether `name` is of a numeric column of `schema`, as PySpark's
+/// `verify_numeric_col_name`: by its parts, of the fields of structs and of
the structs of
+/// arrays. As PySpark's, the name of a struct is an `IndexError`.
+fn verify_numeric_col_name(name: &str, schema: &DataType) -> Result<bool> {
+ fn verify(parts: &[String], data_type: &DataType) -> Result<bool> {
+ if parts.is_empty() && is_numeric(data_type) {
+ return Ok(true);
+ }
+ let fields = match data_type {
+ DataType::Struct { fields } => fields,
+ DataType::Array { element_type, .. } => match
element_type.as_ref() {
+ DataType::Struct { fields } => fields,
+ _ => return Ok(false),
+ },
+ _ => return Ok(false),
+ };
+ let Some((part, rest)) = parts.split_first() else {
+ let mut err = SparkError::value_msg("list index out of range");
+ err.kind = SparkErrorKind::IndexError;
+ return Err(err);
+ };
+ match fields.iter().find(|field| field.name == *part) {
+ Some(field) => verify(rest, &field.data_type),
+ None => Ok(false),
+ }
+ }
+ match parse_attr_name(name) {
+ Some(parts) if !parts.is_empty() => verify(&parts, schema),
+ _ => Ok(false),
+ }
+}
+
/// Statistical functions for DataFrames.
///
/// Mirrors `pyspark.sql.DataFrameStatFunctions`.
@@ -567,6 +704,72 @@ mod tests {
.expect("failed to build session")
}
+ #[test]
+ fn numeric_column_names_are_verified_as_pyspark() {
+ use crate::types::StructField;
+ let field = |name: &str, data_type: DataType| StructField {
+ name: name.to_string(),
+ data_type,
+ nullable: true,
+ metadata: Default::default(),
+ };
+ let schema = DataType::Struct {
+ fields: vec![
+ field("x", DataType::Integer),
+ field(
+ "k",
+ DataType::String {
+ collation: "UTF8_BINARY".to_string(),
+ },
+ ),
+ field(
+ "st",
+ DataType::Struct {
+ fields: vec![field("a", DataType::Double)],
+ },
+ ),
+ field(
+ "arr",
+ DataType::Array {
+ element_type: Box::new(DataType::Struct {
+ fields: vec![field("c", DataType::Long)],
+ }),
+ contains_null: true,
+ },
+ ),
+ ],
+ };
+ // As PySpark's parse_attr_name of these.
+ assert_eq!(
+ parse_attr_name("a.`b.c`.`d``e`").unwrap(),
+ vec!["a", "b.c", "d`e"]
+ );
+ assert_eq!(parse_attr_name(".a").unwrap(), vec!["", "a"]);
+ assert_eq!(parse_attr_name("`a.b`").unwrap(), vec!["a.b"]);
+ for unparsed in ["a..b", "a.", "`a`b", "a`b`", "`a", "a.`b.c`.``d``"] {
+ assert_eq!(parse_attr_name(unparsed), None, "{unparsed}");
+ }
+ for (name, numeric) in [
+ ("x", true),
+ ("k", false),
+ ("st.a", true),
+ ("`st`.`a`", true),
+ ("arr.c", true),
+ ("nope", false),
+ ("st..a", false),
+ ] {
+ assert_eq!(
+ verify_numeric_col_name(name, &schema).unwrap(),
+ numeric,
+ "{name}"
+ );
+ }
+ // As PySpark's, of which a struct is an IndexError.
+ let err = verify_numeric_col_name("st", &schema).unwrap_err();
+ assert_eq!(err.kind, SparkErrorKind::IndexError);
+ assert_eq!(err.message(), "list index out of range");
+ }
+
#[test]
fn grouped_and_stat_builders_share_the_source_plan() {
let spark = session();
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index e11b270..143995b 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -538,6 +538,24 @@ def test_create_dataframe_from_empty_arrow_table(session):
_Pair = namedtuple("_Pair", ["x", "y"])
+def test_grouped_aggregations_name_and_take_columns_as_pyspark(session):
+ # As PySpark's: count is named count, the numeric aggregations of no
columns aggregate
+ # every numeric column, and df.agg takes a dict.
+ from pyspark.errors import PySparkTypeError
+
+ df = session.createDataFrame([(1, "b", 2.0), (2, "b", 3.0)], "x int, k
string, v double")
+ grouped = df.groupBy("k")
+ assert grouped.count().columns == ["k", "count"]
+ assert grouped.count().orderBy("count").collect() == [Row(k="b", count=2)]
+ assert grouped.sum().collect() == [Row(k="b", **{"sum(x)": 3, "sum(v)":
5.0})]
+ assert grouped.min().columns == ["k", "min(x)", "min(v)"]
+ assert df.groupBy().count().collect() == [Row(count=2)]
+ assert df.agg({"x": "max"}).collect() == [Row(**{"max(x)": 2})]
+ with pytest.raises(PySparkTypeError) as error:
+ grouped.avg("k")
+ assert error.value.getMessageParameters() == {"invalid_columns": "['k']"}
+
+
@pytest.mark.parametrize(
"data, schema, columns, rows",
[
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 75cd9f9..b7d675c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5719,6 +5719,52 @@ def
test_schema_cache_derived_and_new_plans_resolve_independently(mock_connect):
assert df.columns == ["id"]
+def test_grouped_aggregations_as_pyspark(mock_connect):
+ # As PySpark's GroupedData: count is named count, sum, avg, mean, min and
max aggregate
+ # every numeric column without columns, and the aggregations check their
arguments.
+ from pyspark.errors import PySparkTypeError, PySparkValueError
+ from pyspark.sql import functions as F
+
+ spark, service = mock_connect
+ df = spark.range(1)
+
+ def aggregations(dataframe):
+ # ExecutePlanRequest.plan=3, Plan.root=1, Relation.aggregate=9,
+ # Aggregate.aggregate_expressions=4.
+ dataframe.collect()
+ plan =
parse_message(parse_message(service.execute_requests[-1])[3][0])[1][0]
+ return parse_message(parse_message(plan)[9][0])[4]
+
+ # Expression.alias=6: Alias.expr=1, name=2;
Expression.unresolved_function=3:
+ # UnresolvedFunction.function_name=1, arguments=2.
+ (count,) = aggregations(df.groupBy("id").count())
+ alias = parse_message(parse_message(count)[6][0])
+ assert alias[2] == [b"count"]
+ assert parse_message(parse_message(alias[1][0])[3][0])[1] == [b"count"]
+ for aggregate, function in ((lambda g: g.sum(), b"sum"), (lambda g:
g.mean(), b"avg"),
+ (lambda g: g.max("id"), b"max")):
+ (expression,) = aggregations(aggregate(df.groupBy("id")))
+ assert parse_message(parse_message(expression)[3][0])[1] == [function]
+ (expression,) = aggregations(df.agg({"id": "min"}))
+ assert parse_message(parse_message(expression)[3][0])[1] == [b"min"]
+
+ with pytest.raises(PySparkTypeError) as error:
+ df.groupBy("id").sum("nope")
+ assert error.value.getCondition() == "NOT_NUMERIC_COLUMNS"
+ assert error.value.getMessageParameters() == {"invalid_columns":
"['nope']"}
+ with pytest.raises(PySparkValueError) as error:
+ df.agg()
+ assert error.value.getCondition() == "CANNOT_BE_EMPTY"
+ for call, message in ((lambda: df.groupBy("id").agg(), "exprs should not
be empty"),
+ (lambda: df.groupBy("id").agg("id"), "all exprs
should be Column"),
+ (lambda: df.agg("id"), "all exprs should be
Expression"),
+ (lambda: df.groupBy("id").sum(1), "")):
+ with pytest.raises(AssertionError) as error:
+ call()
+ assert str(error.value) == message
+ assert df.groupBy("id").agg(F.count("id")) is not None
+
+
@pytest.mark.parametrize("transform", [
pytest.param(lambda df: df.filter("id > 0"), id="filter"),
pytest.param(lambda df: df.limit(2), id="limit"),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]