This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new b6d1c1e [SPARK-61023][CONNECT][PYTHON] Take the arguments of dropna,
na.drop, fillna, describe, corr and approxQuantile as PySpark does
b6d1c1e is described below
commit b6d1c1ec29066afd9dacfd024bc092135770878b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 02:46:45 2026 +0900
[SPARK-61023][CONNECT][PYTHON] Take the arguments of dropna, na.drop,
fillna, describe, corr and approxQuantile as PySpark does
### What changes were proposed in this pull request?
[SPARK-61023](https://issues.apache.org/jira/browse/SPARK-61023): the NA
and statistic functions of a DataFrame now take and check their arguments as
PySpark's do.
- `dropna` and `na.drop` take a `subset` that is a column name, or a list
or tuple of them, and a `how` of "any" or "all" (`VALUE_NOT_ALLOWED`).
- `fillna` and `na.fill` take a value that is a bool, an int, a float, a
string, or a non-empty dict of them by column names (`NOT_EXPECTED_TYPE`,
`CANNOT_BE_EMPTY`).
- `describe` takes column names, or a list of them.
- `corr` takes only the method "pearson", the default (`VALUE_NOT_ALLOWED`).
- `approxQuantile` takes a column, or a list or tuple of them, and returns
the quantiles as a list of those of each. It raises `NEGATIVE_VALUE` for a
negative `relativeError`.
- `cov`, `corr`, `crosstab`, `freqItems`, `approxQuantile` and `summary`
raise `NOT_EXPECTED_TYPE` for arguments of other types. The support of
`freqItems` is 0.01 unless one is given.
The checks of the DataFrame methods and of `df.stat` are shared, in
`stat_arguments`.
### Why are the changes needed?
- `df.dropna(subset="x")` and `df.na.drop(subset="x")` raised a `TypeError`.
- `df.dropna(how="bogus")` was accepted.
- `df.fillna([1])` filled the string `'List([Long(1)])'`.
- `df.describe(["x", "v"])` and `df.approxQuantile(["x", "v"], [0.5], 0.0)`
raised a `TypeError`.
- `df.corr("x", "v", "kendall")` returned the Pearson correlation.
- `df.approxQuantile("v", [0.5], -1.0)` failed on the server.
### Does this PR introduce _any_ user-facing change?
Yes. These functions take the arguments of PySpark 4.3.0 and raise its
errors.
### How was this patch tested?
- A new live test drops by a column name and a tuple, describes a list of
columns, and gets the quantiles of a list of columns, among others. It passes
with official PySpark 4.3.0, and fails without this change.
- A new offline test checks the errors of 22 calls. It passes with official
PySpark 4.3.0, and fails without this change.
- Probes of 69 calls match official PySpark 4.3.0. The exceptions are the
key order of printed message parameters, and one `TypeError` message from
protobuf.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (672 passed, and 129 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1788 passed, 8 skipped),
and the live drop-in and Arrow suites (96 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #224 from HyukjinKwon/SPARK-61023-na-stat.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 183 +++++++---------
crates/pyspark-rs/src/lib.rs | 1 +
crates/pyspark-rs/src/stat.rs | 85 +++-----
crates/pyspark-rs/src/stat_arguments.rs | 376 ++++++++++++++++++++++++++++++++
python/tests/test_dropin_e2e.py | 20 ++
python/tests/test_dropin_offline.py | 48 ++++
6 files changed, 556 insertions(+), 157 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 99ec54b..52e8516 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -734,18 +734,21 @@ impl PyDataFrame {
PyColumn::new(self.dataframe.col_regex(colName))
}
- /// Basic statistics for the given columns (count/mean/stddev/min/max).
+ /// Basic statistics for the given columns (count/mean/stddev/min/max), of
column names or
+ /// a list of them, as PySpark's `DataFrame.describe`.
#[pyo3(signature = (*cols))]
- fn describe(&self, _py: Python<'_>, cols: Vec<String>) -> PyDataFrame {
+ fn describe(&self, cols: Vec<Bound<'_, PyAny>>) -> PyResult<PyDataFrame> {
+ let cols = crate::stat_arguments::describe_columns(&cols)?;
let refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
- PyDataFrame::new(self.dataframe.describe(refs))
+ Ok(PyDataFrame::new(self.dataframe.describe(refs)))
}
- /// Summary statistics (percentiles etc.).
+ /// Summary statistics (percentiles etc.), of strings, as PySpark's
`DataFrame.summary`.
#[pyo3(signature = (*statistics))]
- fn summary(&self, _py: Python<'_>, statistics: Vec<String>) -> PyDataFrame
{
+ fn summary(&self, statistics: Vec<Bound<'_, PyAny>>) ->
PyResult<PyDataFrame> {
+ let statistics =
crate::stat_arguments::summary_statistics(&statistics)?;
let refs: Vec<&str> = statistics.iter().map(|s| s.as_str()).collect();
- PyDataFrame::new(self.dataframe.summary(refs))
+ Ok(PyDataFrame::new(self.dataframe.summary(refs)))
}
/// Persist with the default MEMORY_AND_DISK storage level.
@@ -833,37 +836,29 @@ impl PyDataFrame {
.to_pyerr()
}
- /// Drop rows containing nulls.
- #[pyo3(signature = (how="any", thresh=None, subset=None))]
- fn dropna(&self, how: &str, thresh: Option<i32>, subset:
Option<Vec<String>>) -> PyDataFrame {
- let owned = subset;
- let refs = owned
- .as_ref()
- .map(|v| v.iter().map(|s| s.as_str()).collect());
- PyDataFrame::new(self.dataframe.dropna(Some(how), thresh, refs))
+ /// Drop rows containing nulls, as PySpark's `DataFrame.dropna`.
+ #[pyo3(
+ signature = (how=None, thresh=None, subset=None),
+ text_signature = "($self, how='any', thresh=None, subset=None)"
+ )]
+ fn dropna(
+ &self,
+ how: Option<&Bound<'_, PyAny>>,
+ thresh: Option<&Bound<'_, PyAny>>,
+ subset: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ crate::stat_arguments::dropna(&self.dataframe, how, thresh, subset)
}
- /// Fill null values. `value` may be a scalar (optionally over `subset`)
or a
- /// {column: value} mapping, mirroring `pyspark.sql.DataFrame.fillna`.
+ /// Fill null values, as PySpark's `DataFrame.fillna`: `value` is a
scalar, over
+ /// `subset`, or a {column: value} mapping.
#[pyo3(signature = (value, subset=None))]
fn fillna(
&self,
value: &Bound<'_, PyAny>,
subset: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- if let Ok(d) = value.cast::<PyDict>() {
- let mut pairs = Vec::with_capacity(d.len());
- for (k, v) in d.iter() {
- pairs.push((k.extract::<String>()?,
crate::session::py_to_value(&v)?));
- }
- return Ok(PyDataFrame::new(self.dataframe.fillna_map(pairs)));
- }
- let val = crate::session::py_to_value(value)?;
- let owned = to_subset(subset)?;
- let refs: Option<Vec<&str>> = owned
- .as_ref()
- .map(|v| v.iter().map(|s| s.as_str()).collect());
- Ok(PyDataFrame::new(self.dataframe.fillna_value(val, refs)))
+ crate::stat_arguments::fillna(&self.dataframe, value, subset)
}
/// Statistical functions (`df.stat`).
@@ -875,42 +870,37 @@ impl PyDataFrame {
/// `df.crosstab(col1, col2)` - pyspark exposes it on DataFrame as well as
/// `df.stat.crosstab`.
- fn crosstab(&self, col1: &str, col2: &str) -> PyDataFrame {
- PyDataFrame::new(self.dataframe.stat().crosstab(col1, col2))
+ fn crosstab(&self, col1: &Bound<'_, PyAny>, col2: &Bound<'_, PyAny>) ->
PyResult<PyDataFrame> {
+ crate::stat_arguments::crosstab(&self.dataframe.stat(), col1, col2)
}
- /// `df.freqItems(cols, support=0.01)` (also on df.stat).
- #[pyo3(name = "freqItems", signature = (cols, support=0.01))]
- fn freq_items(&self, cols: Vec<String>, support: f64) -> PyDataFrame {
- let refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
- PyDataFrame::new(self.dataframe.stat().freq_items(refs, support))
+ /// `df.freqItems(cols, support=None)` (also on df.stat).
+ #[pyo3(name = "freqItems", signature = (cols, support=None))]
+ fn freq_items(
+ &self,
+ cols: &Bound<'_, PyAny>,
+ support: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ crate::stat_arguments::freq_items(&self.dataframe.stat(), cols,
support)
}
- /// `df.approxQuantile(col, probabilities, relativeError)` -> list[float]
- /// (also on df.stat). The server returns one row whose column is the
- /// array(-of-arrays) of quantiles; for a single column we return the
inner list.
+ /// `df.approxQuantile(col, probabilities, relativeError)` (also on
df.stat): the
+ /// quantiles of a column, or a list of them of each of a list or tuple of
columns.
#[pyo3(name = "approxQuantile")]
fn approx_quantile(
&self,
py: Python<'_>,
- col: &str,
- probabilities: Vec<f64>,
- relativeError: f64,
- ) -> PyResult<Vec<f64>> {
- use spark_connect::row::Value;
- let df = self
- .dataframe
- .stat()
- .approx_quantile(vec![col], probabilities, relativeError);
- let rows = py.detach(|| df.collect()).to_pyerr()?;
- let quantiles = match rows.first().and_then(|r| r.get(0)) {
- Some(Value::List(outer)) => match outer.first() {
- Some(Value::List(inner)) => inner.iter().filter_map(|v|
v.as_f64()).collect(),
- _ => outer.iter().filter_map(|v| v.as_f64()).collect(),
- },
- _ => Vec::new(),
- };
- Ok(quantiles)
+ col: &Bound<'_, PyAny>,
+ probabilities: &Bound<'_, PyAny>,
+ relativeError: &Bound<'_, PyAny>,
+ ) -> PyResult<Py<PyAny>> {
+ crate::stat_arguments::approx_quantile(
+ py,
+ &self.dataframe.stat(),
+ col,
+ probabilities,
+ relativeError,
+ )
}
/// Observe named metrics on the stream/df. `observation` is an Observation
@@ -986,18 +976,26 @@ impl PyDataFrame {
))
}
- /// Pearson correlation (or `method`) between two columns.
+ /// Pearson correlation between two columns; PySpark's `method` is only
"pearson".
#[pyo3(signature = (col1, col2, method=None))]
- #[allow(unused_variables)]
- fn corr(&self, py: Python<'_>, col1: &str, col2: &str, method:
Option<&str>) -> PyResult<f64> {
- py.detach(|| self.dataframe.stat().corr(col1, col2))
- .to_pyerr()
+ fn corr(
+ &self,
+ py: Python<'_>,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+ method: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<f64> {
+ crate::stat_arguments::corr(py, &self.dataframe.stat(), col1, col2,
method)
}
/// Sample covariance between two columns.
- fn cov(&self, py: Python<'_>, col1: &str, col2: &str) -> PyResult<f64> {
- py.detach(|| self.dataframe.stat().cov(col1, col2))
- .to_pyerr()
+ fn cov(
+ &self,
+ py: Python<'_>,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+ ) -> PyResult<f64> {
+ crate::stat_arguments::cov(py, &self.dataframe.stat(), col1, col2)
}
/// Stratified sample without replacement per key.
@@ -1584,7 +1582,6 @@ impl PyDataFrame {
#[getter]
fn na(&self) -> PyDataFrameNaFunctions {
PyDataFrameNaFunctions {
- inner: self.dataframe.na(),
df: self.dataframe.clone(),
}
}
@@ -1774,47 +1771,34 @@ impl PyDataFrame {
/// Python wrapper for `DataFrameNaFunctions` (`df.na`).
#[pyclass(name = "DataFrameNaFunctions", module = "pyspark.sql.dataframe")]
pub struct PyDataFrameNaFunctions {
- inner: spark_connect::group::NaFunctions,
df: spark_connect::dataframe::DataFrame,
}
#[pymethods]
impl PyDataFrameNaFunctions {
- #[pyo3(signature = (how=None, thresh=None, subset=None))]
+ /// Drop rows containing nulls, as PySpark's `DataFrameNaFunctions.drop`,
which is
+ /// `DataFrame.dropna`.
+ #[pyo3(
+ signature = (how=None, thresh=None, subset=None),
+ text_signature = "($self, how='any', thresh=None, subset=None)"
+ )]
fn drop(
&self,
- how: Option<String>,
- thresh: Option<i32>,
- subset: Option<Vec<String>>,
- ) -> PyDataFrame {
- let subset_refs: Option<Vec<&str>> = subset
- .as_ref()
- .map(|v| v.iter().map(String::as_str).collect());
- PyDataFrame::new(self.inner.drop(how.as_deref(), thresh, subset_refs))
+ how: Option<&Bound<'_, PyAny>>,
+ thresh: Option<&Bound<'_, PyAny>>,
+ subset: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ crate::stat_arguments::dropna(&self.df, how, thresh, subset)
}
- /// Fill nulls; `value` may be a scalar (float / int / str / bool,
optionally over
- /// `subset`) or a {column: value} mapping. Alias of `DataFrame.fillna`.
PySpark fills
- /// only columns whose type matches the value's type (server-side).
+ /// Fill nulls, as PySpark's `DataFrameNaFunctions.fill`, which is
`DataFrame.fillna`.
#[pyo3(signature = (value, subset=None))]
fn fill(
&self,
value: &Bound<'_, PyAny>,
subset: Option<&Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
- if let Ok(d) = value.cast::<PyDict>() {
- let mut pairs = Vec::with_capacity(d.len());
- for (k, v) in d.iter() {
- pairs.push((k.extract::<String>()?,
crate::session::py_to_value(&v)?));
- }
- return Ok(PyDataFrame::new(self.df.fillna_map(pairs)));
- }
- let val = crate::session::py_to_value(value)?;
- let owned = to_subset(subset)?;
- let refs: Option<Vec<&str>> = owned
- .as_ref()
- .map(|v| v.iter().map(|s| s.as_str()).collect());
- Ok(PyDataFrame::new(self.df.fillna_value(val, refs)))
+ crate::stat_arguments::fillna(&self.df, value, subset)
}
/// Replace values, as PySpark's `DataFrameNaFunctions.replace`, which is
@@ -1830,21 +1814,6 @@ impl PyDataFrameNaFunctions {
}
}
-/// Coerce a `subset` argument (a str column name, a list of names, or None) to
-/// an owned Vec<String>, mirroring reference pyspark which accepts either.
-fn to_subset(subset: Option<&Bound<'_, PyAny>>) ->
PyResult<Option<Vec<String>>> {
- match subset {
- None => Ok(None),
- Some(s) => {
- if let Ok(name) = s.extract::<String>() {
- Ok(Some(vec![name]))
- } else {
- Ok(Some(s.extract::<Vec<String>>()?))
- }
- }
- }
-}
-
/// The `value` argument of `DataFrame.replace`, where PySpark tells an
omitted value apart
/// from None: `replace(to_replace)` requires a dict `to_replace`, while
/// `replace(to_replace, None)` replaces with nulls.
diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index 549565d..96c9f43 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -46,6 +46,7 @@ mod resource;
mod row;
mod session;
mod stat;
+mod stat_arguments;
mod storagelevel;
mod streaming;
mod tablearg;
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index ca26489..bb45d83 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -4,7 +4,6 @@ use pyo3::prelude::*;
use spark_connect::group::StatFunctions;
use crate::dataframe::PyDataFrame;
-use crate::errors::ResultExt;
/// Python wrapper for DataFrame statistical functions.
#[pyclass(name = "DataFrameStatFunctions", module = "pyspark.sql.dataframe")]
@@ -20,41 +19,46 @@ impl PyStatFunctions {
#[pymethods]
impl PyStatFunctions {
- /// Pearson correlation between two numeric columns.
+ /// Pearson correlation between two numeric columns, as PySpark's, whose
`method` is only
+ /// "pearson".
///
- /// `corr`/`cov` execute a plan (server round-trip via `scalar()`), so
release
+ /// `corr`/`cov` execute a plan (server round-trip via `scalar()`), so
they release
/// the GIL across the call - otherwise a Python thread is blocked for the
whole
- /// RPC. (`crosstab`/`freq_items`/`approx_quantile` return lazy DataFrames
and do
- /// no I/O here, so they need no detach.)
- /// Pearson correlation. `method` is accepted for signature parity with
reference
- /// pyspark, which currently supports only "pearson".
+ /// RPC.
#[pyo3(signature = (col1, col2, method=None))]
- fn corr(&self, py: Python<'_>, col1: &str, col2: &str, method:
Option<&str>) -> PyResult<f64> {
- if let Some(m) = method {
- if m != "pearson" {
- return Err(PyErr::new::<pyo3::exceptions::PyValueError,
_>(format!(
- "Currently only the calculation of the Pearson Correlation
coefficient is supported, got {m}"
- )));
- }
- }
- py.detach(|| self.stat.corr(col1, col2)).to_pyerr()
+ fn corr(
+ &self,
+ py: Python<'_>,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+ method: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<f64> {
+ crate::stat_arguments::corr(py, &self.stat, col1, col2, method)
}
/// Sample covariance between two numeric columns.
- fn cov(&self, py: Python<'_>, col1: &str, col2: &str) -> PyResult<f64> {
- py.detach(|| self.stat.cov(col1, col2)).to_pyerr()
+ fn cov(
+ &self,
+ py: Python<'_>,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+ ) -> PyResult<f64> {
+ crate::stat_arguments::cov(py, &self.stat, col1, col2)
}
/// Contingency table (cross-tabulation) of two columns.
- fn crosstab(&self, col1: &str, col2: &str) -> PyDataFrame {
- PyDataFrame::new(self.stat.crosstab(col1, col2))
+ fn crosstab(&self, col1: &Bound<'_, PyAny>, col2: &Bound<'_, PyAny>) ->
PyResult<PyDataFrame> {
+ crate::stat_arguments::crosstab(&self.stat, col1, col2)
}
/// Frequent items for the given columns.
- #[pyo3(name = "freqItems", signature = (cols, support=0.01))]
- fn freq_items(&self, cols: Vec<String>, support: f64) -> PyDataFrame {
- let refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
- PyDataFrame::new(self.stat.freq_items(refs, support))
+ #[pyo3(name = "freqItems", signature = (cols, support=None))]
+ fn freq_items(
+ &self,
+ cols: &Bound<'_, PyAny>,
+ support: Option<&Bound<'_, PyAny>>,
+ ) -> PyResult<PyDataFrame> {
+ crate::stat_arguments::freq_items(&self.stat, cols, support)
}
/// Stratified sample without replacement. Mirrors
`DataFrameStatFunctions.sampleBy(
@@ -70,36 +74,17 @@ impl PyStatFunctions {
Ok(PyDataFrame::new(self.stat.sample_by(col, fractions, seed)))
}
- /// Approximate quantiles of a column at the given probabilities.
- ///
- /// Mirrors reference `DataFrameStatFunctions.approxQuantile(col,
probabilities,
- /// relativeError)`, which returns a `list[float]` (the quantiles) - not a
- /// DataFrame. The server returns a single row whose one column is the
- /// `array<double>` of quantiles; collect it (releasing the GIL for the
RPC) and
- /// return that list.
+ /// Approximate quantiles of a column, or of each of a list or tuple of
columns, at the
+ /// given probabilities, as PySpark's
`DataFrameStatFunctions.approxQuantile`.
#[pyo3(name = "approxQuantile")]
fn approx_quantile(
&self,
py: Python<'_>,
- col: &str,
- probabilities: Vec<f64>,
- relativeError: f64,
- ) -> PyResult<Vec<f64>> {
- use spark_connect::row::Value;
- let df = self
- .stat
- .approx_quantile(vec![col], probabilities, relativeError);
- let rows = py.detach(|| df.collect()).to_pyerr()?;
- // The server returns a single row, col 0 = array-of-arrays (one inner
array of
- // quantiles per input column); for a single column we return that
inner array.
- let quantiles = match rows.first().and_then(|r| r.get(0)) {
- Some(Value::List(outer)) => match outer.first() {
- Some(Value::List(inner)) => inner.iter().filter_map(|v|
v.as_f64()).collect(),
- _ => outer.iter().filter_map(|v| v.as_f64()).collect(),
- },
- _ => Vec::new(),
- };
- Ok(quantiles)
+ col: &Bound<'_, PyAny>,
+ probabilities: &Bound<'_, PyAny>,
+ relativeError: &Bound<'_, PyAny>,
+ ) -> PyResult<Py<PyAny>> {
+ crate::stat_arguments::approx_quantile(py, &self.stat, col,
probabilities, relativeError)
}
}
diff --git a/crates/pyspark-rs/src/stat_arguments.rs
b/crates/pyspark-rs/src/stat_arguments.rs
new file mode 100644
index 0000000..099069f
--- /dev/null
+++ b/crates/pyspark-rs/src/stat_arguments.rs
@@ -0,0 +1,376 @@
+//! The NA and statistic functions of a DataFrame (`df.dropna`, `df.na.fill`,
`df.stat.corr`,
+//! ...), which check their arguments on the client as PySpark Connect's
DataFrame does.
+
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyFloat, PyInt, PyList, PyString, PyTuple};
+use spark_connect::dataframe::DataFrame;
+use spark_connect::group::StatFunctions;
+
+use crate::column::raise_pyspark;
+use crate::dataframe::PyDataFrame;
+use crate::errors::ResultExt;
+
+/// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which
should be of
+/// the type `expected`.
+pub(crate) fn not_expected_type(value: &Bound<'_, PyAny>, name: &str,
expected: &str) -> PyErr {
+ let type_name = value
+ .get_type()
+ .name()
+ .map(|name| name.to_string())
+ .unwrap_or_default();
+ raise_pyspark(
+ value.py(),
+ "PySparkTypeError",
+ "NOT_EXPECTED_TYPE",
+ &[
+ ("arg_name", name),
+ ("expected_type", expected),
+ ("arg_type", &type_name),
+ ],
+ )
+}
+
+fn is_sequence(value: &Bound<'_, PyAny>) -> bool {
+ value.is_instance_of::<PyList>() || value.is_instance_of::<PyTuple>()
+}
+
+/// A bool, an int, a float or a str: a bool is an int.
+fn is_primitive(value: &Bound<'_, PyAny>) -> bool {
+ value.is_instance_of::<PyInt>()
+ || value.is_instance_of::<PyFloat>()
+ || value.is_instance_of::<PyString>()
+}
+
+/// The argument `name`, a str, as PySpark's statistic functions take a column.
+fn string_argument(value: &Bound<'_, PyAny>, name: &str) -> PyResult<String> {
+ if value.is_instance_of::<PyString>() {
+ value.extract()
+ } else {
+ Err(not_expected_type(value, name, "str"))
+ }
+}
+
+/// The columns of the `subset` of `dropna` and `fillna`, as PySpark's: a
column name, or a
+/// list or tuple of them; `expected` is the type that the error of another
type names.
+fn subset_columns(subset: Option<&Bound<'_, PyAny>>, expected: &str) ->
PyResult<Vec<String>> {
+ let Some(subset) = subset else {
+ return Ok(Vec::new());
+ };
+ if subset.is_instance_of::<PyString>() {
+ return Ok(vec![subset.extract()?]);
+ }
+ if !is_sequence(subset) {
+ return Err(not_expected_type(subset, "subset", expected));
+ }
+ subset
+ .try_iter()?
+ .map(|column| {
+ let column = column?;
+ if column.is_instance_of::<PyString>() {
+ column.extract()
+ } else {
+ Err(not_expected_type(&column, "cols", "list, str or tuple"))
+ }
+ })
+ .collect()
+}
+
+fn column_refs(columns: &[String]) -> Option<Vec<&str>> {
+ (!columns.is_empty()).then(|| columns.iter().map(String::as_str).collect())
+}
+
+/// `df.dropna(how, thresh, subset)` and `df.na.drop`, as PySpark's: `how` is
"any" or "all",
+/// else `VALUE_NOT_ALLOWED`, `thresh`, an int, overrides it, and `subset` is
a column name,
+/// or a list or tuple of them.
+pub(crate) fn dropna(
+ df: &DataFrame,
+ how: Option<&Bound<'_, PyAny>>,
+ thresh: Option<&Bound<'_, PyAny>>,
+ subset: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+ let mut how_name = "any";
+ if let Some(how) = how {
+ match string_argument(how, "how")?.as_str() {
+ "all" => how_name = "all",
+ "any" => {}
+ _ => {
+ return Err(raise_pyspark(
+ how.py(),
+ "PySparkValueError",
+ "VALUE_NOT_ALLOWED",
+ &[("arg_name", "how"), ("allowed_values", "['any',
'all']")],
+ ))
+ }
+ }
+ }
+ let thresh = match thresh {
+ None => None,
+ Some(thresh) if thresh.is_instance_of::<PyInt>() =>
Some(thresh.extract::<i32>()?),
+ Some(thresh) => return Err(not_expected_type(thresh, "thresh", "int")),
+ };
+ let columns = subset_columns(subset, "list, str or tuple")?;
+ Ok(PyDataFrame::new(df.dropna(
+ Some(how_name),
+ thresh,
+ column_refs(&columns),
+ )))
+}
+
+/// `df.fillna(value, subset)` and `df.na.fill`, as PySpark's: `value` is a
bool, an int, a
+/// float or a str, over the columns of `subset`, or a non-empty dict of them
by column names.
+pub(crate) fn fillna(
+ df: &DataFrame,
+ value: &Bound<'_, PyAny>,
+ subset: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+ let dict = value.cast::<PyDict>().ok();
+ if dict.is_none() && !is_primitive(value) {
+ return Err(not_expected_type(
+ value,
+ "value",
+ "bool, dict, float, int or str",
+ ));
+ }
+ if let Some(dict) = &dict {
+ if dict.is_empty() {
+ return Err(raise_pyspark(
+ value.py(),
+ "PySparkValueError",
+ "CANNOT_BE_EMPTY",
+ &[("item", "value")],
+ ));
+ }
+ for (column, fill) in dict.iter() {
+ if !column.is_instance_of::<PyString>() {
+ return Err(not_expected_type(&column, "key type of dict",
"str"));
+ }
+ if !is_primitive(&fill) {
+ return Err(not_expected_type(
+ &fill,
+ "value type of dict",
+ "bool, float, int or str",
+ ));
+ }
+ }
+ }
+ let columns = subset_columns(subset, "list or tuple")?;
+ if let Some(dict) = dict {
+ let mut pairs = Vec::with_capacity(dict.len());
+ for (column, fill) in dict.iter() {
+ pairs.push((column.extract()?,
crate::session::py_to_value(&fill)?));
+ }
+ return Ok(PyDataFrame::new(df.fillna_map(pairs)));
+ }
+ Ok(PyDataFrame::new(df.fillna_value(
+ crate::session::py_to_value(value)?,
+ column_refs(&columns),
+ )))
+}
+
+/// The columns of `df.describe(*cols)`, as PySpark's: column names, or a list
of them.
+pub(crate) fn describe_columns(cols: &[Bound<'_, PyAny>]) ->
PyResult<Vec<String>> {
+ let cols: Vec<Bound<'_, PyAny>> = match cols {
+ [only] if only.is_instance_of::<PyList>() =>
only.try_iter()?.collect::<PyResult<_>>()?,
+ _ => cols.to_vec(),
+ };
+ let mut names = Vec::new();
+ for column in cols {
+ if column.is_instance_of::<PyString>() {
+ names.push(column.extract()?);
+ } else {
+ for name in column.try_iter()? {
+ names.push(name?.extract()?);
+ }
+ }
+ }
+ Ok(names)
+}
+
+/// The statistics of `df.summary(*statistics)`, as PySpark's: strings.
+pub(crate) fn summary_statistics(statistics: &[Bound<'_, PyAny>]) ->
PyResult<Vec<String>> {
+ statistics
+ .iter()
+ .map(|statistic| {
+ if statistic.is_instance_of::<PyString>() {
+ statistic.extract()
+ } else {
+ Err(not_expected_type(statistic, "statistics", "list[str]"))
+ }
+ })
+ .collect()
+}
+
+/// `df.cov(col1, col2)`, of two column names, as PySpark's.
+pub(crate) fn cov(
+ py: Python<'_>,
+ stat: &StatFunctions,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+) -> PyResult<f64> {
+ let (col1, col2) = (
+ string_argument(col1, "col1")?,
+ string_argument(col2, "col2")?,
+ );
+ py.detach(|| stat.cov(&col1, &col2).map_err(Box::new))
+ .to_pyerr()
+}
+
+/// `df.corr(col1, col2, method)`, of two column names and the method
"pearson", the default,
+/// else `VALUE_NOT_ALLOWED`, as PySpark's.
+pub(crate) fn corr(
+ py: Python<'_>,
+ stat: &StatFunctions,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+ method: Option<&Bound<'_, PyAny>>,
+) -> PyResult<f64> {
+ let (col1, col2) = (
+ string_argument(col1, "col1")?,
+ string_argument(col2, "col2")?,
+ );
+ if let Some(method) = method {
+ if method.is_truthy()? && !method.eq("pearson")? {
+ return Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "VALUE_NOT_ALLOWED",
+ &[("arg_name", "method"), ("allowed_values", "['pearson']")],
+ ));
+ }
+ }
+ py.detach(|| stat.corr(&col1, &col2).map_err(Box::new))
+ .to_pyerr()
+}
+
+/// `df.crosstab(col1, col2)`, of two column names, as PySpark's.
+pub(crate) fn crosstab(
+ stat: &StatFunctions,
+ col1: &Bound<'_, PyAny>,
+ col2: &Bound<'_, PyAny>,
+) -> PyResult<PyDataFrame> {
+ let (col1, col2) = (
+ string_argument(col1, "col1")?,
+ string_argument(col2, "col2")?,
+ );
+ Ok(PyDataFrame::new(stat.crosstab(&col1, &col2)))
+}
+
+/// `df.freqItems(cols, support)`, as PySpark's: `cols` is a list or tuple,
and `support` is
+/// 0.01 unless one is given.
+pub(crate) fn freq_items(
+ stat: &StatFunctions,
+ cols: &Bound<'_, PyAny>,
+ support: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+ if !is_sequence(cols) {
+ return Err(not_expected_type(cols, "cols", "list or tuple"));
+ }
+ let cols: Vec<String> = cols.extract()?;
+ let support = match support {
+ Some(support) if support.is_truthy()? => support.extract()?,
+ _ => 0.01,
+ };
+ let refs: Vec<&str> = cols.iter().map(String::as_str).collect();
+ Ok(PyDataFrame::new(stat.freq_items(refs, support)))
+}
+
+/// `df.approxQuantile(col, probabilities, relativeError)`, as PySpark's: the
quantiles of a
+/// column, or a list of those of each of a list or tuple of columns, at
`probabilities`, a
+/// list or tuple of numbers from 0 to 1, with a non-negative `relativeError`.
+pub(crate) fn approx_quantile(
+ py: Python<'_>,
+ stat: &StatFunctions,
+ col: &Bound<'_, PyAny>,
+ probabilities: &Bound<'_, PyAny>,
+ relative_error: &Bound<'_, PyAny>,
+) -> PyResult<Py<PyAny>> {
+ use pyo3::IntoPyObjectExt;
+ use spark_connect::row::Value;
+
+ let one_column = col.is_instance_of::<PyString>();
+ if !one_column && !is_sequence(col) {
+ return Err(not_expected_type(col, "col", "list, str or tuple"));
+ }
+ let columns: Vec<Bound<'_, PyAny>> = if one_column {
+ vec![col.clone()]
+ } else {
+ col.try_iter()?.collect::<PyResult<_>>()?
+ };
+ let columns: Vec<String> = columns
+ .iter()
+ .map(|column| {
+ if column.is_instance_of::<PyString>() {
+ column.extract()
+ } else {
+ Err(not_expected_type(column, "columns", "list[str]"))
+ }
+ })
+ .collect::<PyResult<_>>()?;
+ if !is_sequence(probabilities) {
+ return Err(not_expected_type(
+ probabilities,
+ "probabilities",
+ "list or tuple",
+ ));
+ }
+ let mut quantiles = Vec::new();
+ for probability in probabilities.try_iter()? {
+ let probability = probability?;
+ let number =
+ probability.is_instance_of::<PyInt>() ||
probability.is_instance_of::<PyFloat>();
+ let value: f64 = if number { probability.extract()? } else { 0.0 };
+ // As PySpark's `p < 0 or p > 1`, which a NaN passes, unlike a range's
`contains`.
+ #[allow(clippy::manual_range_contains)]
+ let out_of_range = value < 0.0 || value > 1.0;
+ if !number || out_of_range {
+ return Err(not_expected_type(
+ &probability,
+ "probabilities",
+ "list[float, int]",
+ ));
+ }
+ quantiles.push(value);
+ }
+ if !(relative_error.is_instance_of::<PyInt>() ||
relative_error.is_instance_of::<PyFloat>()) {
+ return Err(not_expected_type(
+ relative_error,
+ "relativeError",
+ "float or int",
+ ));
+ }
+ let error: f64 = relative_error.extract()?;
+ if error < 0.0 {
+ return Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "NEGATIVE_VALUE",
+ &[
+ ("arg_name", "relativeError"),
+ ("arg_value", &relative_error.str()?.to_string()),
+ ],
+ ));
+ }
+ let refs: Vec<&str> = columns.iter().map(String::as_str).collect();
+ let df = stat.approx_quantile(refs, quantiles, error);
+ let rows = py.detach(|| df.collect().map_err(Box::new)).to_pyerr()?;
+ // One row, of the array of the quantiles of each column.
+ let per_column: Vec<Vec<f64>> = match rows.first().and_then(|row|
row.get(0)) {
+ Some(Value::List(columns)) => columns
+ .iter()
+ .map(|quantiles| match quantiles {
+ Value::List(values) => values.iter().filter_map(|v|
v.as_f64()).collect(),
+ _ => Vec::new(),
+ })
+ .collect(),
+ _ => Vec::new(),
+ };
+ if one_column {
+ per_column
+ .into_iter()
+ .next()
+ .unwrap_or_default()
+ .into_py_any(py)
+ } else {
+ per_column.into_py_any(py)
+ }
+}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 3d2066a..61042fa 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1449,6 +1449,26 @@ def
test_create_dataframe_infers_by_the_session_configurations_as_pyspark(sessio
session.conf.unset(key)
+def test_na_and_stat_functions_take_their_arguments_as_pyspark(session):
+ # As PySpark's: the subset of dropna, a column name or a list or tuple of
them, the columns
+ # of describe, names or a list of them, and the quantiles of
approxQuantile of a list or
+ # tuple of columns, a list of those of each.
+ df = session.createDataFrame(
+ [(1, "a", None), (None, "b", 2.0), (3, None, 3.0)], "x int, k string,
v double"
+ )
+ kept = [Row(x=1, k="a", v=None), Row(x=3, k=None, v=3.0)]
+ assert df.dropna(subset="x").collect() == kept
+ assert df.na.drop(subset=("x",)).collect() == kept
+ assert df.dropna(how="all", thresh=3).collect() == []
+ assert df.describe(["x", "v"]).columns == ["summary", "x", "v"]
+ assert df.describe(("x", "v")).columns == ["summary", "x", "v"]
+ assert df.approxQuantile(["x", "v"], [0.0, 1.0], 0.0) == [[1.0, 3.0],
[2.0, 3.0]]
+ assert df.stat.approxQuantile("x", (0.0, 1), 0) == [1.0, 3.0]
+ linear = session.createDataFrame([(1, 2.0), (2, 4.0), (3, 6.0)], "a int, b
double")
+ assert linear.corr("a", "b", None) == linear.stat.corr("a", "b",
"pearson") == 1.0
+ assert df.freqItems(["k"]).columns == ["k_freqItems"]
+
+
def test_functions_apply_their_optional_arguments_as_pyspark(session):
df = session.sql(
"SELECT 'key' AS k, 'msg' AS m, parse_json('{\"a\": 1}') AS v, "
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 47a4ac4..404731a 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1585,6 +1585,54 @@ def test_row_count_and_index():
r.index(99)
+def test_na_and_stat_functions_check_their_arguments_as_pyspark():
+ # As PySpark's NA and statistic functions of a DataFrame, which check
their arguments on
+ # the client.
+ from pyspark.errors import PySparkTypeError, PySparkValueError
+
+ df = _offline_session().range(1)
+
+ def not_expected(arg_name, expected_type, arg_type):
+ return (PySparkTypeError, "NOT_EXPECTED_TYPE",
+ {"arg_name": arg_name, "expected_type": expected_type,
"arg_type": arg_type})
+
+ for call, (error_type, condition, parameters) in (
+ (lambda: df.dropna(how="bogus"), (PySparkValueError,
"VALUE_NOT_ALLOWED",
+ {"arg_name": "how",
"allowed_values": "['any', 'all']"})),
+ (lambda: df.na.drop(how=1), not_expected("how", "str", "int")),
+ (lambda: df.dropna(thresh=1.5), not_expected("thresh", "int",
"float")),
+ (lambda: df.dropna(subset=1), not_expected("subset", "list, str or
tuple", "int")),
+ (lambda: df.na.drop(subset=["id", 1]), not_expected("cols", "list, str
or tuple", "int")),
+ (lambda: df.fillna([1]), not_expected("value", "bool, dict, float, int
or str", "list")),
+ (lambda: df.na.fill({}), (PySparkValueError, "CANNOT_BE_EMPTY",
{"item": "value"})),
+ (lambda: df.fillna({1: 0}), not_expected("key type of dict", "str",
"int")),
+ (lambda: df.fillna({"id": [1]}),
+ not_expected("value type of dict", "bool, float, int or str",
"list")),
+ (lambda: df.fillna(0, subset=1), not_expected("subset", "list or
tuple", "int")),
+ (lambda: df.summary(1), not_expected("statistics", "list[str]",
"int")),
+ (lambda: df.cov("id", 1), not_expected("col2", "str", "int")),
+ (lambda: df.stat.corr(1, "id"), not_expected("col1", "str", "int")),
+ (lambda: df.corr("id", "id", "kendall"), (PySparkValueError,
"VALUE_NOT_ALLOWED",
+ {"arg_name": "method",
+ "allowed_values":
"['pearson']"})),
+ (lambda: df.stat.crosstab("id", 1), not_expected("col2", "str",
"int")),
+ (lambda: df.freqItems("id"), not_expected("cols", "list or tuple",
"str")),
+ (lambda: df.approxQuantile(1, [0.5], 0.0), not_expected("col", "list,
str or tuple", "int")),
+ (lambda: df.approxQuantile(["id", 1], [0.5], 0.0),
not_expected("columns", "list[str]", "int")),
+ (lambda: df.stat.approxQuantile("id", 0.5, 0.0),
+ not_expected("probabilities", "list or tuple", "float")),
+ (lambda: df.approxQuantile("id", [1.5], 0.0),
+ not_expected("probabilities", "list[float, int]", "float")),
+ (lambda: df.approxQuantile("id", [0.5], "0"),
not_expected("relativeError", "float or int", "str")),
+ (lambda: df.approxQuantile("id", [0.5], -1), (PySparkValueError,
"NEGATIVE_VALUE",
+ {"arg_name":
"relativeError", "arg_value": "-1"})),
+ ):
+ with pytest.raises(error_type) as error:
+ call()
+ assert error.value.getCondition() == condition
+ assert error.value.getMessageParameters() == parameters
+
+
def test_avro_protobuf_partitioning_submodules():
# The official import paths resolve (the implementations live in
# pyspark.sql.functions and are re-exported here).
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]