This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new b6d1c1e  [SPARK-61023][CONNECT][PYTHON] Take the arguments of dropna, 
na.drop, fillna, describe, corr and approxQuantile as PySpark does
b6d1c1e is described below

commit b6d1c1ec29066afd9dacfd024bc092135770878b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 02:46:45 2026 +0900

    [SPARK-61023][CONNECT][PYTHON] Take the arguments of dropna, na.drop, 
fillna, describe, corr and approxQuantile as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61023](https://issues.apache.org/jira/browse/SPARK-61023): the NA 
and statistic functions of a DataFrame now take and check their arguments as 
PySpark's do.
    
    - `dropna` and `na.drop` take a `subset` that is a column name, or a list 
or tuple of them, and a `how` of "any" or "all" (`VALUE_NOT_ALLOWED`).
    - `fillna` and `na.fill` take a value that is a bool, an int, a float, a 
string, or a non-empty dict of them by column names (`NOT_EXPECTED_TYPE`, 
`CANNOT_BE_EMPTY`).
    - `describe` takes column names, or a list of them.
    - `corr` takes only the method "pearson", the default (`VALUE_NOT_ALLOWED`).
    - `approxQuantile` takes a column, or a list or tuple of them, and returns 
the quantiles as a list of those of each. It raises `NEGATIVE_VALUE` for a 
negative `relativeError`.
    - `cov`, `corr`, `crosstab`, `freqItems`, `approxQuantile` and `summary` 
raise `NOT_EXPECTED_TYPE` for arguments of other types. The support of 
`freqItems` is 0.01 unless one is given.
    
    The checks of the DataFrame methods and of `df.stat` are shared, in 
`stat_arguments`.
    
    ### Why are the changes needed?
    
    - `df.dropna(subset="x")` and `df.na.drop(subset="x")` raised a `TypeError`.
    - `df.dropna(how="bogus")` was accepted.
    - `df.fillna([1])` filled the string `'List([Long(1)])'`.
    - `df.describe(["x", "v"])` and `df.approxQuantile(["x", "v"], [0.5], 0.0)` 
raised a `TypeError`.
    - `df.corr("x", "v", "kendall")` returned the Pearson correlation.
    - `df.approxQuantile("v", [0.5], -1.0)` failed on the server.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These functions take the arguments of PySpark 4.3.0 and raise its 
errors.
    
    ### How was this patch tested?
    
    - A new live test drops by a column name and a tuple, describes a list of 
columns, and gets the quantiles of a list of columns, among others. It passes 
with official PySpark 4.3.0, and fails without this change.
    - A new offline test checks the errors of 22 calls. It passes with official 
PySpark 4.3.0, and fails without this change.
    - Probes of 69 calls match official PySpark 4.3.0. The exceptions are the 
key order of printed message parameters, and one `TypeError` message from 
protobuf.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (672 passed, and 129 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (1788 passed, 8 skipped), 
and the live drop-in and Arrow suites (96 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #224 from HyukjinKwon/SPARK-61023-na-stat.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs      | 183 +++++++---------
 crates/pyspark-rs/src/lib.rs            |   1 +
 crates/pyspark-rs/src/stat.rs           |  85 +++-----
 crates/pyspark-rs/src/stat_arguments.rs | 376 ++++++++++++++++++++++++++++++++
 python/tests/test_dropin_e2e.py         |  20 ++
 python/tests/test_dropin_offline.py     |  48 ++++
 6 files changed, 556 insertions(+), 157 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 99ec54b..52e8516 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -734,18 +734,21 @@ impl PyDataFrame {
         PyColumn::new(self.dataframe.col_regex(colName))
     }
 
-    /// Basic statistics for the given columns (count/mean/stddev/min/max).
+    /// Basic statistics for the given columns (count/mean/stddev/min/max), of 
column names or
+    /// a list of them, as PySpark's `DataFrame.describe`.
     #[pyo3(signature = (*cols))]
-    fn describe(&self, _py: Python<'_>, cols: Vec<String>) -> PyDataFrame {
+    fn describe(&self, cols: Vec<Bound<'_, PyAny>>) -> PyResult<PyDataFrame> {
+        let cols = crate::stat_arguments::describe_columns(&cols)?;
         let refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
-        PyDataFrame::new(self.dataframe.describe(refs))
+        Ok(PyDataFrame::new(self.dataframe.describe(refs)))
     }
 
-    /// Summary statistics (percentiles etc.).
+    /// Summary statistics (percentiles etc.), of strings, as PySpark's 
`DataFrame.summary`.
     #[pyo3(signature = (*statistics))]
-    fn summary(&self, _py: Python<'_>, statistics: Vec<String>) -> PyDataFrame 
{
+    fn summary(&self, statistics: Vec<Bound<'_, PyAny>>) -> 
PyResult<PyDataFrame> {
+        let statistics = 
crate::stat_arguments::summary_statistics(&statistics)?;
         let refs: Vec<&str> = statistics.iter().map(|s| s.as_str()).collect();
-        PyDataFrame::new(self.dataframe.summary(refs))
+        Ok(PyDataFrame::new(self.dataframe.summary(refs)))
     }
 
     /// Persist with the default MEMORY_AND_DISK storage level.
@@ -833,37 +836,29 @@ impl PyDataFrame {
             .to_pyerr()
     }
 
-    /// Drop rows containing nulls.
-    #[pyo3(signature = (how="any", thresh=None, subset=None))]
-    fn dropna(&self, how: &str, thresh: Option<i32>, subset: 
Option<Vec<String>>) -> PyDataFrame {
-        let owned = subset;
-        let refs = owned
-            .as_ref()
-            .map(|v| v.iter().map(|s| s.as_str()).collect());
-        PyDataFrame::new(self.dataframe.dropna(Some(how), thresh, refs))
+    /// Drop rows containing nulls, as PySpark's `DataFrame.dropna`.
+    #[pyo3(
+        signature = (how=None, thresh=None, subset=None),
+        text_signature = "($self, how='any', thresh=None, subset=None)"
+    )]
+    fn dropna(
+        &self,
+        how: Option<&Bound<'_, PyAny>>,
+        thresh: Option<&Bound<'_, PyAny>>,
+        subset: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        crate::stat_arguments::dropna(&self.dataframe, how, thresh, subset)
     }
 
-    /// Fill null values. `value` may be a scalar (optionally over `subset`) 
or a
-    /// {column: value} mapping, mirroring `pyspark.sql.DataFrame.fillna`.
+    /// Fill null values, as PySpark's `DataFrame.fillna`: `value` is a 
scalar, over
+    /// `subset`, or a {column: value} mapping.
     #[pyo3(signature = (value, subset=None))]
     fn fillna(
         &self,
         value: &Bound<'_, PyAny>,
         subset: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        if let Ok(d) = value.cast::<PyDict>() {
-            let mut pairs = Vec::with_capacity(d.len());
-            for (k, v) in d.iter() {
-                pairs.push((k.extract::<String>()?, 
crate::session::py_to_value(&v)?));
-            }
-            return Ok(PyDataFrame::new(self.dataframe.fillna_map(pairs)));
-        }
-        let val = crate::session::py_to_value(value)?;
-        let owned = to_subset(subset)?;
-        let refs: Option<Vec<&str>> = owned
-            .as_ref()
-            .map(|v| v.iter().map(|s| s.as_str()).collect());
-        Ok(PyDataFrame::new(self.dataframe.fillna_value(val, refs)))
+        crate::stat_arguments::fillna(&self.dataframe, value, subset)
     }
 
     /// Statistical functions (`df.stat`).
@@ -875,42 +870,37 @@ impl PyDataFrame {
 
     /// `df.crosstab(col1, col2)` - pyspark exposes it on DataFrame as well as
     /// `df.stat.crosstab`.
-    fn crosstab(&self, col1: &str, col2: &str) -> PyDataFrame {
-        PyDataFrame::new(self.dataframe.stat().crosstab(col1, col2))
+    fn crosstab(&self, col1: &Bound<'_, PyAny>, col2: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
+        crate::stat_arguments::crosstab(&self.dataframe.stat(), col1, col2)
     }
 
-    /// `df.freqItems(cols, support=0.01)` (also on df.stat).
-    #[pyo3(name = "freqItems", signature = (cols, support=0.01))]
-    fn freq_items(&self, cols: Vec<String>, support: f64) -> PyDataFrame {
-        let refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
-        PyDataFrame::new(self.dataframe.stat().freq_items(refs, support))
+    /// `df.freqItems(cols, support=None)` (also on df.stat).
+    #[pyo3(name = "freqItems", signature = (cols, support=None))]
+    fn freq_items(
+        &self,
+        cols: &Bound<'_, PyAny>,
+        support: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        crate::stat_arguments::freq_items(&self.dataframe.stat(), cols, 
support)
     }
 
-    /// `df.approxQuantile(col, probabilities, relativeError)` -> list[float]
-    /// (also on df.stat). The server returns one row whose column is the
-    /// array(-of-arrays) of quantiles; for a single column we return the 
inner list.
+    /// `df.approxQuantile(col, probabilities, relativeError)` (also on 
df.stat): the
+    /// quantiles of a column, or a list of them of each of a list or tuple of 
columns.
     #[pyo3(name = "approxQuantile")]
     fn approx_quantile(
         &self,
         py: Python<'_>,
-        col: &str,
-        probabilities: Vec<f64>,
-        relativeError: f64,
-    ) -> PyResult<Vec<f64>> {
-        use spark_connect::row::Value;
-        let df = self
-            .dataframe
-            .stat()
-            .approx_quantile(vec![col], probabilities, relativeError);
-        let rows = py.detach(|| df.collect()).to_pyerr()?;
-        let quantiles = match rows.first().and_then(|r| r.get(0)) {
-            Some(Value::List(outer)) => match outer.first() {
-                Some(Value::List(inner)) => inner.iter().filter_map(|v| 
v.as_f64()).collect(),
-                _ => outer.iter().filter_map(|v| v.as_f64()).collect(),
-            },
-            _ => Vec::new(),
-        };
-        Ok(quantiles)
+        col: &Bound<'_, PyAny>,
+        probabilities: &Bound<'_, PyAny>,
+        relativeError: &Bound<'_, PyAny>,
+    ) -> PyResult<Py<PyAny>> {
+        crate::stat_arguments::approx_quantile(
+            py,
+            &self.dataframe.stat(),
+            col,
+            probabilities,
+            relativeError,
+        )
     }
 
     /// Observe named metrics on the stream/df. `observation` is an Observation
@@ -986,18 +976,26 @@ impl PyDataFrame {
         ))
     }
 
-    /// Pearson correlation (or `method`) between two columns.
+    /// Pearson correlation between two columns; PySpark's `method` is only 
"pearson".
     #[pyo3(signature = (col1, col2, method=None))]
-    #[allow(unused_variables)]
-    fn corr(&self, py: Python<'_>, col1: &str, col2: &str, method: 
Option<&str>) -> PyResult<f64> {
-        py.detach(|| self.dataframe.stat().corr(col1, col2))
-            .to_pyerr()
+    fn corr(
+        &self,
+        py: Python<'_>,
+        col1: &Bound<'_, PyAny>,
+        col2: &Bound<'_, PyAny>,
+        method: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<f64> {
+        crate::stat_arguments::corr(py, &self.dataframe.stat(), col1, col2, 
method)
     }
 
     /// Sample covariance between two columns.
-    fn cov(&self, py: Python<'_>, col1: &str, col2: &str) -> PyResult<f64> {
-        py.detach(|| self.dataframe.stat().cov(col1, col2))
-            .to_pyerr()
+    fn cov(
+        &self,
+        py: Python<'_>,
+        col1: &Bound<'_, PyAny>,
+        col2: &Bound<'_, PyAny>,
+    ) -> PyResult<f64> {
+        crate::stat_arguments::cov(py, &self.dataframe.stat(), col1, col2)
     }
 
     /// Stratified sample without replacement per key.
@@ -1584,7 +1582,6 @@ impl PyDataFrame {
     #[getter]
     fn na(&self) -> PyDataFrameNaFunctions {
         PyDataFrameNaFunctions {
-            inner: self.dataframe.na(),
             df: self.dataframe.clone(),
         }
     }
@@ -1774,47 +1771,34 @@ impl PyDataFrame {
 /// Python wrapper for `DataFrameNaFunctions` (`df.na`).
 #[pyclass(name = "DataFrameNaFunctions", module = "pyspark.sql.dataframe")]
 pub struct PyDataFrameNaFunctions {
-    inner: spark_connect::group::NaFunctions,
     df: spark_connect::dataframe::DataFrame,
 }
 
 #[pymethods]
 impl PyDataFrameNaFunctions {
-    #[pyo3(signature = (how=None, thresh=None, subset=None))]
+    /// Drop rows containing nulls, as PySpark's `DataFrameNaFunctions.drop`, 
which is
+    /// `DataFrame.dropna`.
+    #[pyo3(
+        signature = (how=None, thresh=None, subset=None),
+        text_signature = "($self, how='any', thresh=None, subset=None)"
+    )]
     fn drop(
         &self,
-        how: Option<String>,
-        thresh: Option<i32>,
-        subset: Option<Vec<String>>,
-    ) -> PyDataFrame {
-        let subset_refs: Option<Vec<&str>> = subset
-            .as_ref()
-            .map(|v| v.iter().map(String::as_str).collect());
-        PyDataFrame::new(self.inner.drop(how.as_deref(), thresh, subset_refs))
+        how: Option<&Bound<'_, PyAny>>,
+        thresh: Option<&Bound<'_, PyAny>>,
+        subset: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        crate::stat_arguments::dropna(&self.df, how, thresh, subset)
     }
 
-    /// Fill nulls; `value` may be a scalar (float / int / str / bool, 
optionally over
-    /// `subset`) or a {column: value} mapping. Alias of `DataFrame.fillna`. 
PySpark fills
-    /// only columns whose type matches the value's type (server-side).
+    /// Fill nulls, as PySpark's `DataFrameNaFunctions.fill`, which is 
`DataFrame.fillna`.
     #[pyo3(signature = (value, subset=None))]
     fn fill(
         &self,
         value: &Bound<'_, PyAny>,
         subset: Option<&Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
-        if let Ok(d) = value.cast::<PyDict>() {
-            let mut pairs = Vec::with_capacity(d.len());
-            for (k, v) in d.iter() {
-                pairs.push((k.extract::<String>()?, 
crate::session::py_to_value(&v)?));
-            }
-            return Ok(PyDataFrame::new(self.df.fillna_map(pairs)));
-        }
-        let val = crate::session::py_to_value(value)?;
-        let owned = to_subset(subset)?;
-        let refs: Option<Vec<&str>> = owned
-            .as_ref()
-            .map(|v| v.iter().map(|s| s.as_str()).collect());
-        Ok(PyDataFrame::new(self.df.fillna_value(val, refs)))
+        crate::stat_arguments::fillna(&self.df, value, subset)
     }
 
     /// Replace values, as PySpark's `DataFrameNaFunctions.replace`, which is
@@ -1830,21 +1814,6 @@ impl PyDataFrameNaFunctions {
     }
 }
 
-/// Coerce a `subset` argument (a str column name, a list of names, or None) to
-/// an owned Vec<String>, mirroring reference pyspark which accepts either.
-fn to_subset(subset: Option<&Bound<'_, PyAny>>) -> 
PyResult<Option<Vec<String>>> {
-    match subset {
-        None => Ok(None),
-        Some(s) => {
-            if let Ok(name) = s.extract::<String>() {
-                Ok(Some(vec![name]))
-            } else {
-                Ok(Some(s.extract::<Vec<String>>()?))
-            }
-        }
-    }
-}
-
 /// The `value` argument of `DataFrame.replace`, where PySpark tells an 
omitted value apart
 /// from None: `replace(to_replace)` requires a dict `to_replace`, while
 /// `replace(to_replace, None)` replaces with nulls.
diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index 549565d..96c9f43 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -46,6 +46,7 @@ mod resource;
 mod row;
 mod session;
 mod stat;
+mod stat_arguments;
 mod storagelevel;
 mod streaming;
 mod tablearg;
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index ca26489..bb45d83 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -4,7 +4,6 @@ use pyo3::prelude::*;
 use spark_connect::group::StatFunctions;
 
 use crate::dataframe::PyDataFrame;
-use crate::errors::ResultExt;
 
 /// Python wrapper for DataFrame statistical functions.
 #[pyclass(name = "DataFrameStatFunctions", module = "pyspark.sql.dataframe")]
@@ -20,41 +19,46 @@ impl PyStatFunctions {
 
 #[pymethods]
 impl PyStatFunctions {
-    /// Pearson correlation between two numeric columns.
+    /// Pearson correlation between two numeric columns, as PySpark's, whose 
`method` is only
+    /// "pearson".
     ///
-    /// `corr`/`cov` execute a plan (server round-trip via `scalar()`), so 
release
+    /// `corr`/`cov` execute a plan (server round-trip via `scalar()`), so 
they release
     /// the GIL across the call - otherwise a Python thread is blocked for the 
whole
-    /// RPC. (`crosstab`/`freq_items`/`approx_quantile` return lazy DataFrames 
and do
-    /// no I/O here, so they need no detach.)
-    /// Pearson correlation. `method` is accepted for signature parity with 
reference
-    /// pyspark, which currently supports only "pearson".
+    /// RPC.
     #[pyo3(signature = (col1, col2, method=None))]
-    fn corr(&self, py: Python<'_>, col1: &str, col2: &str, method: 
Option<&str>) -> PyResult<f64> {
-        if let Some(m) = method {
-            if m != "pearson" {
-                return Err(PyErr::new::<pyo3::exceptions::PyValueError, 
_>(format!(
-                    "Currently only the calculation of the Pearson Correlation 
coefficient is supported, got {m}"
-                )));
-            }
-        }
-        py.detach(|| self.stat.corr(col1, col2)).to_pyerr()
+    fn corr(
+        &self,
+        py: Python<'_>,
+        col1: &Bound<'_, PyAny>,
+        col2: &Bound<'_, PyAny>,
+        method: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<f64> {
+        crate::stat_arguments::corr(py, &self.stat, col1, col2, method)
     }
 
     /// Sample covariance between two numeric columns.
-    fn cov(&self, py: Python<'_>, col1: &str, col2: &str) -> PyResult<f64> {
-        py.detach(|| self.stat.cov(col1, col2)).to_pyerr()
+    fn cov(
+        &self,
+        py: Python<'_>,
+        col1: &Bound<'_, PyAny>,
+        col2: &Bound<'_, PyAny>,
+    ) -> PyResult<f64> {
+        crate::stat_arguments::cov(py, &self.stat, col1, col2)
     }
 
     /// Contingency table (cross-tabulation) of two columns.
-    fn crosstab(&self, col1: &str, col2: &str) -> PyDataFrame {
-        PyDataFrame::new(self.stat.crosstab(col1, col2))
+    fn crosstab(&self, col1: &Bound<'_, PyAny>, col2: &Bound<'_, PyAny>) -> 
PyResult<PyDataFrame> {
+        crate::stat_arguments::crosstab(&self.stat, col1, col2)
     }
 
     /// Frequent items for the given columns.
-    #[pyo3(name = "freqItems", signature = (cols, support=0.01))]
-    fn freq_items(&self, cols: Vec<String>, support: f64) -> PyDataFrame {
-        let refs: Vec<&str> = cols.iter().map(|s| s.as_str()).collect();
-        PyDataFrame::new(self.stat.freq_items(refs, support))
+    #[pyo3(name = "freqItems", signature = (cols, support=None))]
+    fn freq_items(
+        &self,
+        cols: &Bound<'_, PyAny>,
+        support: Option<&Bound<'_, PyAny>>,
+    ) -> PyResult<PyDataFrame> {
+        crate::stat_arguments::freq_items(&self.stat, cols, support)
     }
 
     /// Stratified sample without replacement. Mirrors 
`DataFrameStatFunctions.sampleBy(
@@ -70,36 +74,17 @@ impl PyStatFunctions {
         Ok(PyDataFrame::new(self.stat.sample_by(col, fractions, seed)))
     }
 
-    /// Approximate quantiles of a column at the given probabilities.
-    ///
-    /// Mirrors reference `DataFrameStatFunctions.approxQuantile(col, 
probabilities,
-    /// relativeError)`, which returns a `list[float]` (the quantiles) - not a
-    /// DataFrame. The server returns a single row whose one column is the
-    /// `array<double>` of quantiles; collect it (releasing the GIL for the 
RPC) and
-    /// return that list.
+    /// Approximate quantiles of a column, or of each of a list or tuple of 
columns, at the
+    /// given probabilities, as PySpark's 
`DataFrameStatFunctions.approxQuantile`.
     #[pyo3(name = "approxQuantile")]
     fn approx_quantile(
         &self,
         py: Python<'_>,
-        col: &str,
-        probabilities: Vec<f64>,
-        relativeError: f64,
-    ) -> PyResult<Vec<f64>> {
-        use spark_connect::row::Value;
-        let df = self
-            .stat
-            .approx_quantile(vec![col], probabilities, relativeError);
-        let rows = py.detach(|| df.collect()).to_pyerr()?;
-        // The server returns a single row, col 0 = array-of-arrays (one inner 
array of
-        // quantiles per input column); for a single column we return that 
inner array.
-        let quantiles = match rows.first().and_then(|r| r.get(0)) {
-            Some(Value::List(outer)) => match outer.first() {
-                Some(Value::List(inner)) => inner.iter().filter_map(|v| 
v.as_f64()).collect(),
-                _ => outer.iter().filter_map(|v| v.as_f64()).collect(),
-            },
-            _ => Vec::new(),
-        };
-        Ok(quantiles)
+        col: &Bound<'_, PyAny>,
+        probabilities: &Bound<'_, PyAny>,
+        relativeError: &Bound<'_, PyAny>,
+    ) -> PyResult<Py<PyAny>> {
+        crate::stat_arguments::approx_quantile(py, &self.stat, col, 
probabilities, relativeError)
     }
 }
 
diff --git a/crates/pyspark-rs/src/stat_arguments.rs 
b/crates/pyspark-rs/src/stat_arguments.rs
new file mode 100644
index 0000000..099069f
--- /dev/null
+++ b/crates/pyspark-rs/src/stat_arguments.rs
@@ -0,0 +1,376 @@
+//! The NA and statistic functions of a DataFrame (`df.dropna`, `df.na.fill`, 
`df.stat.corr`,
+//! ...), which check their arguments on the client as PySpark Connect's 
DataFrame does.
+
+use pyo3::prelude::*;
+use pyo3::types::{PyDict, PyFloat, PyInt, PyList, PyString, PyTuple};
+use spark_connect::dataframe::DataFrame;
+use spark_connect::group::StatFunctions;
+
+use crate::column::raise_pyspark;
+use crate::dataframe::PyDataFrame;
+use crate::errors::ResultExt;
+
+/// PySpark's `NOT_EXPECTED_TYPE` error of the argument `name`, `value`, which 
should be of
+/// the type `expected`.
+pub(crate) fn not_expected_type(value: &Bound<'_, PyAny>, name: &str, 
expected: &str) -> PyErr {
+    let type_name = value
+        .get_type()
+        .name()
+        .map(|name| name.to_string())
+        .unwrap_or_default();
+    raise_pyspark(
+        value.py(),
+        "PySparkTypeError",
+        "NOT_EXPECTED_TYPE",
+        &[
+            ("arg_name", name),
+            ("expected_type", expected),
+            ("arg_type", &type_name),
+        ],
+    )
+}
+
+fn is_sequence(value: &Bound<'_, PyAny>) -> bool {
+    value.is_instance_of::<PyList>() || value.is_instance_of::<PyTuple>()
+}
+
+/// A bool, an int, a float or a str: a bool is an int.
+fn is_primitive(value: &Bound<'_, PyAny>) -> bool {
+    value.is_instance_of::<PyInt>()
+        || value.is_instance_of::<PyFloat>()
+        || value.is_instance_of::<PyString>()
+}
+
+/// The argument `name`, a str, as PySpark's statistic functions take a column.
+fn string_argument(value: &Bound<'_, PyAny>, name: &str) -> PyResult<String> {
+    if value.is_instance_of::<PyString>() {
+        value.extract()
+    } else {
+        Err(not_expected_type(value, name, "str"))
+    }
+}
+
+/// The columns of the `subset` of `dropna` and `fillna`, as PySpark's: a 
column name, or a
+/// list or tuple of them; `expected` is the type that the error of another 
type names.
+fn subset_columns(subset: Option<&Bound<'_, PyAny>>, expected: &str) -> 
PyResult<Vec<String>> {
+    let Some(subset) = subset else {
+        return Ok(Vec::new());
+    };
+    if subset.is_instance_of::<PyString>() {
+        return Ok(vec![subset.extract()?]);
+    }
+    if !is_sequence(subset) {
+        return Err(not_expected_type(subset, "subset", expected));
+    }
+    subset
+        .try_iter()?
+        .map(|column| {
+            let column = column?;
+            if column.is_instance_of::<PyString>() {
+                column.extract()
+            } else {
+                Err(not_expected_type(&column, "cols", "list, str or tuple"))
+            }
+        })
+        .collect()
+}
+
+fn column_refs(columns: &[String]) -> Option<Vec<&str>> {
+    (!columns.is_empty()).then(|| columns.iter().map(String::as_str).collect())
+}
+
+/// `df.dropna(how, thresh, subset)` and `df.na.drop`, as PySpark's: `how` is 
"any" or "all",
+/// else `VALUE_NOT_ALLOWED`, `thresh`, an int, overrides it, and `subset` is 
a column name,
+/// or a list or tuple of them.
+pub(crate) fn dropna(
+    df: &DataFrame,
+    how: Option<&Bound<'_, PyAny>>,
+    thresh: Option<&Bound<'_, PyAny>>,
+    subset: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+    let mut how_name = "any";
+    if let Some(how) = how {
+        match string_argument(how, "how")?.as_str() {
+            "all" => how_name = "all",
+            "any" => {}
+            _ => {
+                return Err(raise_pyspark(
+                    how.py(),
+                    "PySparkValueError",
+                    "VALUE_NOT_ALLOWED",
+                    &[("arg_name", "how"), ("allowed_values", "['any', 
'all']")],
+                ))
+            }
+        }
+    }
+    let thresh = match thresh {
+        None => None,
+        Some(thresh) if thresh.is_instance_of::<PyInt>() => 
Some(thresh.extract::<i32>()?),
+        Some(thresh) => return Err(not_expected_type(thresh, "thresh", "int")),
+    };
+    let columns = subset_columns(subset, "list, str or tuple")?;
+    Ok(PyDataFrame::new(df.dropna(
+        Some(how_name),
+        thresh,
+        column_refs(&columns),
+    )))
+}
+
+/// `df.fillna(value, subset)` and `df.na.fill`, as PySpark's: `value` is a 
bool, an int, a
+/// float or a str, over the columns of `subset`, or a non-empty dict of them 
by column names.
+pub(crate) fn fillna(
+    df: &DataFrame,
+    value: &Bound<'_, PyAny>,
+    subset: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+    let dict = value.cast::<PyDict>().ok();
+    if dict.is_none() && !is_primitive(value) {
+        return Err(not_expected_type(
+            value,
+            "value",
+            "bool, dict, float, int or str",
+        ));
+    }
+    if let Some(dict) = &dict {
+        if dict.is_empty() {
+            return Err(raise_pyspark(
+                value.py(),
+                "PySparkValueError",
+                "CANNOT_BE_EMPTY",
+                &[("item", "value")],
+            ));
+        }
+        for (column, fill) in dict.iter() {
+            if !column.is_instance_of::<PyString>() {
+                return Err(not_expected_type(&column, "key type of dict", 
"str"));
+            }
+            if !is_primitive(&fill) {
+                return Err(not_expected_type(
+                    &fill,
+                    "value type of dict",
+                    "bool, float, int or str",
+                ));
+            }
+        }
+    }
+    let columns = subset_columns(subset, "list or tuple")?;
+    if let Some(dict) = dict {
+        let mut pairs = Vec::with_capacity(dict.len());
+        for (column, fill) in dict.iter() {
+            pairs.push((column.extract()?, 
crate::session::py_to_value(&fill)?));
+        }
+        return Ok(PyDataFrame::new(df.fillna_map(pairs)));
+    }
+    Ok(PyDataFrame::new(df.fillna_value(
+        crate::session::py_to_value(value)?,
+        column_refs(&columns),
+    )))
+}
+
+/// The columns of `df.describe(*cols)`, as PySpark's: column names, or a list 
of them.
+pub(crate) fn describe_columns(cols: &[Bound<'_, PyAny>]) -> 
PyResult<Vec<String>> {
+    let cols: Vec<Bound<'_, PyAny>> = match cols {
+        [only] if only.is_instance_of::<PyList>() => 
only.try_iter()?.collect::<PyResult<_>>()?,
+        _ => cols.to_vec(),
+    };
+    let mut names = Vec::new();
+    for column in cols {
+        if column.is_instance_of::<PyString>() {
+            names.push(column.extract()?);
+        } else {
+            for name in column.try_iter()? {
+                names.push(name?.extract()?);
+            }
+        }
+    }
+    Ok(names)
+}
+
+/// The statistics of `df.summary(*statistics)`, as PySpark's: strings.
+pub(crate) fn summary_statistics(statistics: &[Bound<'_, PyAny>]) -> 
PyResult<Vec<String>> {
+    statistics
+        .iter()
+        .map(|statistic| {
+            if statistic.is_instance_of::<PyString>() {
+                statistic.extract()
+            } else {
+                Err(not_expected_type(statistic, "statistics", "list[str]"))
+            }
+        })
+        .collect()
+}
+
+/// `df.cov(col1, col2)`, of two column names, as PySpark's.
+pub(crate) fn cov(
+    py: Python<'_>,
+    stat: &StatFunctions,
+    col1: &Bound<'_, PyAny>,
+    col2: &Bound<'_, PyAny>,
+) -> PyResult<f64> {
+    let (col1, col2) = (
+        string_argument(col1, "col1")?,
+        string_argument(col2, "col2")?,
+    );
+    py.detach(|| stat.cov(&col1, &col2).map_err(Box::new))
+        .to_pyerr()
+}
+
+/// `df.corr(col1, col2, method)`, of two column names and the method 
"pearson", the default,
+/// else `VALUE_NOT_ALLOWED`, as PySpark's.
+pub(crate) fn corr(
+    py: Python<'_>,
+    stat: &StatFunctions,
+    col1: &Bound<'_, PyAny>,
+    col2: &Bound<'_, PyAny>,
+    method: Option<&Bound<'_, PyAny>>,
+) -> PyResult<f64> {
+    let (col1, col2) = (
+        string_argument(col1, "col1")?,
+        string_argument(col2, "col2")?,
+    );
+    if let Some(method) = method {
+        if method.is_truthy()? && !method.eq("pearson")? {
+            return Err(raise_pyspark(
+                py,
+                "PySparkValueError",
+                "VALUE_NOT_ALLOWED",
+                &[("arg_name", "method"), ("allowed_values", "['pearson']")],
+            ));
+        }
+    }
+    py.detach(|| stat.corr(&col1, &col2).map_err(Box::new))
+        .to_pyerr()
+}
+
+/// `df.crosstab(col1, col2)`, of two column names, as PySpark's.
+pub(crate) fn crosstab(
+    stat: &StatFunctions,
+    col1: &Bound<'_, PyAny>,
+    col2: &Bound<'_, PyAny>,
+) -> PyResult<PyDataFrame> {
+    let (col1, col2) = (
+        string_argument(col1, "col1")?,
+        string_argument(col2, "col2")?,
+    );
+    Ok(PyDataFrame::new(stat.crosstab(&col1, &col2)))
+}
+
+/// `df.freqItems(cols, support)`, as PySpark's: `cols` is a list or tuple, 
and `support` is
+/// 0.01 unless one is given.
+pub(crate) fn freq_items(
+    stat: &StatFunctions,
+    cols: &Bound<'_, PyAny>,
+    support: Option<&Bound<'_, PyAny>>,
+) -> PyResult<PyDataFrame> {
+    if !is_sequence(cols) {
+        return Err(not_expected_type(cols, "cols", "list or tuple"));
+    }
+    let cols: Vec<String> = cols.extract()?;
+    let support = match support {
+        Some(support) if support.is_truthy()? => support.extract()?,
+        _ => 0.01,
+    };
+    let refs: Vec<&str> = cols.iter().map(String::as_str).collect();
+    Ok(PyDataFrame::new(stat.freq_items(refs, support)))
+}
+
+/// `df.approxQuantile(col, probabilities, relativeError)`, as PySpark's: the 
quantiles of a
+/// column, or a list of those of each of a list or tuple of columns, at 
`probabilities`, a
+/// list or tuple of numbers from 0 to 1, with a non-negative `relativeError`.
+pub(crate) fn approx_quantile(
+    py: Python<'_>,
+    stat: &StatFunctions,
+    col: &Bound<'_, PyAny>,
+    probabilities: &Bound<'_, PyAny>,
+    relative_error: &Bound<'_, PyAny>,
+) -> PyResult<Py<PyAny>> {
+    use pyo3::IntoPyObjectExt;
+    use spark_connect::row::Value;
+
+    let one_column = col.is_instance_of::<PyString>();
+    if !one_column && !is_sequence(col) {
+        return Err(not_expected_type(col, "col", "list, str or tuple"));
+    }
+    let columns: Vec<Bound<'_, PyAny>> = if one_column {
+        vec![col.clone()]
+    } else {
+        col.try_iter()?.collect::<PyResult<_>>()?
+    };
+    let columns: Vec<String> = columns
+        .iter()
+        .map(|column| {
+            if column.is_instance_of::<PyString>() {
+                column.extract()
+            } else {
+                Err(not_expected_type(column, "columns", "list[str]"))
+            }
+        })
+        .collect::<PyResult<_>>()?;
+    if !is_sequence(probabilities) {
+        return Err(not_expected_type(
+            probabilities,
+            "probabilities",
+            "list or tuple",
+        ));
+    }
+    let mut quantiles = Vec::new();
+    for probability in probabilities.try_iter()? {
+        let probability = probability?;
+        let number =
+            probability.is_instance_of::<PyInt>() || 
probability.is_instance_of::<PyFloat>();
+        let value: f64 = if number { probability.extract()? } else { 0.0 };
+        // As PySpark's `p < 0 or p > 1`, which a NaN passes, unlike a range's 
`contains`.
+        #[allow(clippy::manual_range_contains)]
+        let out_of_range = value < 0.0 || value > 1.0;
+        if !number || out_of_range {
+            return Err(not_expected_type(
+                &probability,
+                "probabilities",
+                "list[float, int]",
+            ));
+        }
+        quantiles.push(value);
+    }
+    if !(relative_error.is_instance_of::<PyInt>() || 
relative_error.is_instance_of::<PyFloat>()) {
+        return Err(not_expected_type(
+            relative_error,
+            "relativeError",
+            "float or int",
+        ));
+    }
+    let error: f64 = relative_error.extract()?;
+    if error < 0.0 {
+        return Err(raise_pyspark(
+            py,
+            "PySparkValueError",
+            "NEGATIVE_VALUE",
+            &[
+                ("arg_name", "relativeError"),
+                ("arg_value", &relative_error.str()?.to_string()),
+            ],
+        ));
+    }
+    let refs: Vec<&str> = columns.iter().map(String::as_str).collect();
+    let df = stat.approx_quantile(refs, quantiles, error);
+    let rows = py.detach(|| df.collect().map_err(Box::new)).to_pyerr()?;
+    // One row, of the array of the quantiles of each column.
+    let per_column: Vec<Vec<f64>> = match rows.first().and_then(|row| 
row.get(0)) {
+        Some(Value::List(columns)) => columns
+            .iter()
+            .map(|quantiles| match quantiles {
+                Value::List(values) => values.iter().filter_map(|v| 
v.as_f64()).collect(),
+                _ => Vec::new(),
+            })
+            .collect(),
+        _ => Vec::new(),
+    };
+    if one_column {
+        per_column
+            .into_iter()
+            .next()
+            .unwrap_or_default()
+            .into_py_any(py)
+    } else {
+        per_column.into_py_any(py)
+    }
+}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 3d2066a..61042fa 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1449,6 +1449,26 @@ def 
test_create_dataframe_infers_by_the_session_configurations_as_pyspark(sessio
             session.conf.unset(key)
 
 
+def test_na_and_stat_functions_take_their_arguments_as_pyspark(session):
+    # As PySpark's: the subset of dropna, a column name or a list or tuple of 
them, the columns
+    # of describe, names or a list of them, and the quantiles of 
approxQuantile of a list or
+    # tuple of columns, a list of those of each.
+    df = session.createDataFrame(
+        [(1, "a", None), (None, "b", 2.0), (3, None, 3.0)], "x int, k string, 
v double"
+    )
+    kept = [Row(x=1, k="a", v=None), Row(x=3, k=None, v=3.0)]
+    assert df.dropna(subset="x").collect() == kept
+    assert df.na.drop(subset=("x",)).collect() == kept
+    assert df.dropna(how="all", thresh=3).collect() == []
+    assert df.describe(["x", "v"]).columns == ["summary", "x", "v"]
+    assert df.describe(("x", "v")).columns == ["summary", "x", "v"]
+    assert df.approxQuantile(["x", "v"], [0.0, 1.0], 0.0) == [[1.0, 3.0], 
[2.0, 3.0]]
+    assert df.stat.approxQuantile("x", (0.0, 1), 0) == [1.0, 3.0]
+    linear = session.createDataFrame([(1, 2.0), (2, 4.0), (3, 6.0)], "a int, b 
double")
+    assert linear.corr("a", "b", None) == linear.stat.corr("a", "b", 
"pearson") == 1.0
+    assert df.freqItems(["k"]).columns == ["k_freqItems"]
+
+
 def test_functions_apply_their_optional_arguments_as_pyspark(session):
     df = session.sql(
         "SELECT 'key' AS k, 'msg' AS m, parse_json('{\"a\": 1}') AS v, "
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 47a4ac4..404731a 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1585,6 +1585,54 @@ def test_row_count_and_index():
         r.index(99)
 
 
+def test_na_and_stat_functions_check_their_arguments_as_pyspark():
+    # As PySpark's NA and statistic functions of a DataFrame, which check 
their arguments on
+    # the client.
+    from pyspark.errors import PySparkTypeError, PySparkValueError
+
+    df = _offline_session().range(1)
+
+    def not_expected(arg_name, expected_type, arg_type):
+        return (PySparkTypeError, "NOT_EXPECTED_TYPE",
+                {"arg_name": arg_name, "expected_type": expected_type, 
"arg_type": arg_type})
+
+    for call, (error_type, condition, parameters) in (
+        (lambda: df.dropna(how="bogus"), (PySparkValueError, 
"VALUE_NOT_ALLOWED",
+                                          {"arg_name": "how", 
"allowed_values": "['any', 'all']"})),
+        (lambda: df.na.drop(how=1), not_expected("how", "str", "int")),
+        (lambda: df.dropna(thresh=1.5), not_expected("thresh", "int", 
"float")),
+        (lambda: df.dropna(subset=1), not_expected("subset", "list, str or 
tuple", "int")),
+        (lambda: df.na.drop(subset=["id", 1]), not_expected("cols", "list, str 
or tuple", "int")),
+        (lambda: df.fillna([1]), not_expected("value", "bool, dict, float, int 
or str", "list")),
+        (lambda: df.na.fill({}), (PySparkValueError, "CANNOT_BE_EMPTY", 
{"item": "value"})),
+        (lambda: df.fillna({1: 0}), not_expected("key type of dict", "str", 
"int")),
+        (lambda: df.fillna({"id": [1]}),
+         not_expected("value type of dict", "bool, float, int or str", 
"list")),
+        (lambda: df.fillna(0, subset=1), not_expected("subset", "list or 
tuple", "int")),
+        (lambda: df.summary(1), not_expected("statistics", "list[str]", 
"int")),
+        (lambda: df.cov("id", 1), not_expected("col2", "str", "int")),
+        (lambda: df.stat.corr(1, "id"), not_expected("col1", "str", "int")),
+        (lambda: df.corr("id", "id", "kendall"), (PySparkValueError, 
"VALUE_NOT_ALLOWED",
+                                                  {"arg_name": "method",
+                                                   "allowed_values": 
"['pearson']"})),
+        (lambda: df.stat.crosstab("id", 1), not_expected("col2", "str", 
"int")),
+        (lambda: df.freqItems("id"), not_expected("cols", "list or tuple", 
"str")),
+        (lambda: df.approxQuantile(1, [0.5], 0.0), not_expected("col", "list, 
str or tuple", "int")),
+        (lambda: df.approxQuantile(["id", 1], [0.5], 0.0), 
not_expected("columns", "list[str]", "int")),
+        (lambda: df.stat.approxQuantile("id", 0.5, 0.0),
+         not_expected("probabilities", "list or tuple", "float")),
+        (lambda: df.approxQuantile("id", [1.5], 0.0),
+         not_expected("probabilities", "list[float, int]", "float")),
+        (lambda: df.approxQuantile("id", [0.5], "0"), 
not_expected("relativeError", "float or int", "str")),
+        (lambda: df.approxQuantile("id", [0.5], -1), (PySparkValueError, 
"NEGATIVE_VALUE",
+                                                     {"arg_name": 
"relativeError", "arg_value": "-1"})),
+    ):
+        with pytest.raises(error_type) as error:
+            call()
+        assert error.value.getCondition() == condition
+        assert error.value.getMessageParameters() == parameters
+
+
 def test_avro_protobuf_partitioning_submodules():
     # The official import paths resolve (the implementations live in
     # pyspark.sql.functions and are re-exported here).


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to