This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 45ad1bb [SPARK-61049][CONNECT][PYTHON] Expose the df of df.na and
df.stat in the Python client as PySpark does
45ad1bb is described below
commit 45ad1bbda245b2f087a63d4ec78c43e29c18defb
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:52:07 2026 +0900
[SPARK-61049][CONNECT][PYTHON] Expose the df of df.na and df.stat in the
Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61049](https://issues.apache.org/jira/browse/SPARK-61049): the
`DataFrameNaFunctions` of `df.na` and the `DataFrameStatFunctions` of `df.stat`
now have the `df` of PySpark Connect's, the DataFrame itself, so `df.na.df is
df`.
### Why are the changes needed?
The client had no `df` of them: `df.na.df` raised an `AttributeError`.
### Does this PR introduce _any_ user-facing change?
Yes. `df.na.df` and `df.stat.df` are the DataFrame, as in PySpark 4.3.0.
### How was this patch tested?
- A new live test checks that `df.na.df` and `df.stat.df` are `df`, and
runs functions of them. It passes with official PySpark 4.3.0, and fails
without this change.
- A new offline test checks the same of a DataFrame of an offline session.
- The lib tests of both crates (574 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1813 passed, 8 skipped), and the
live drop-in and Arrow suites (112 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #250 from HyukjinKwon/SPARK-61049-na-stat-df.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 23 +++++++++++++++++------
crates/pyspark-rs/src/stat.rs | 13 +++++++++++--
python/tests/test_dropin_e2e.py | 9 +++++++++
python/tests/test_dropin_offline.py | 13 +++++++++++++
4 files changed, 50 insertions(+), 8 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 3a1a041..bda68b4 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1294,11 +1294,11 @@ impl PyDataFrame {
crate::stat_arguments::fillna(&self.dataframe, value, subset)
}
- /// Statistical functions (`df.stat`).
+ /// Statistical functions (`df.stat`), of this DataFrame, their `df`, as
PySpark's.
#[getter]
#[pyo3(name = "stat")]
- fn stat(&self) -> crate::stat::PyStatFunctions {
- crate::stat::PyStatFunctions::new(self.dataframe.stat())
+ fn stat(slf: &Bound<'_, Self>) -> crate::stat::PyStatFunctions {
+ crate::stat::PyStatFunctions::new(slf.borrow().dataframe.stat(),
slf.clone().unbind())
}
/// `df.crosstab(col1, col2)` - pyspark exposes it on DataFrame as well as
@@ -2098,11 +2098,13 @@ impl PyDataFrame {
PyDataStreamWriter::new(writer)
}
- /// Null-handling functions (`df.na.drop()/fill()/replace()`).
+ /// Null-handling functions (`df.na.drop()/fill()/replace()`), of this
DataFrame, their
+ /// `df`, as PySpark's.
#[getter]
- fn na(&self) -> PyDataFrameNaFunctions {
+ fn na(slf: &Bound<'_, Self>) -> PyDataFrameNaFunctions {
PyDataFrameNaFunctions {
- df: self.dataframe.clone(),
+ df: slf.borrow().dataframe.clone(),
+ frame: slf.clone().unbind(),
}
}
@@ -2313,10 +2315,19 @@ impl PyDataFrame {
#[pyclass(name = "DataFrameNaFunctions", module = "pyspark.sql.dataframe")]
pub struct PyDataFrameNaFunctions {
df: spark_connect::dataframe::DataFrame,
+ /// The DataFrame of `df.na`, as PySpark's `DataFrameNaFunctions.df`.
+ frame: Py<PyDataFrame>,
}
#[pymethods]
impl PyDataFrameNaFunctions {
+ /// The DataFrame of these functions, as PySpark's
`DataFrameNaFunctions.df`: `df.na.df` is
+ /// `df`.
+ #[getter(df)]
+ fn frame(&self, py: Python<'_>) -> Py<PyDataFrame> {
+ self.frame.clone_ref(py)
+ }
+
/// Drop rows containing nulls, as PySpark's `DataFrameNaFunctions.drop`,
which is
/// `DataFrame.dropna`.
#[pyo3(
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index cc7ced3..445d293 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -9,16 +9,25 @@ use crate::dataframe::PyDataFrame;
#[pyclass(name = "DataFrameStatFunctions", module = "pyspark.sql.dataframe")]
pub struct PyStatFunctions {
stat: StatFunctions,
+ /// The DataFrame of `df.stat`, as PySpark's `DataFrameStatFunctions.df`.
+ frame: Py<PyDataFrame>,
}
impl PyStatFunctions {
- pub fn new(stat: StatFunctions) -> Self {
- PyStatFunctions { stat }
+ pub fn new(stat: StatFunctions, frame: Py<PyDataFrame>) -> Self {
+ PyStatFunctions { stat, frame }
}
}
#[pymethods]
impl PyStatFunctions {
+ /// The DataFrame of these functions, as PySpark's
`DataFrameStatFunctions.df`: `df.stat.df`
+ /// is `df`.
+ #[getter(df)]
+ fn frame(&self, py: Python<'_>) -> Py<PyDataFrame> {
+ self.frame.clone_ref(py)
+ }
+
/// Pearson correlation between two numeric columns, as PySpark's, whose
`method` is only
/// "pearson".
///
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index fe999fb..d499e05 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1503,6 +1503,15 @@ def
test_to_pandas_converts_the_columns_as_pyspark(session):
session.conf.unset("spark.sql.session.timeZone")
+def test_na_and_stat_functions_are_of_their_dataframe_as_pyspark(session):
+ # As PySpark's: the df of df.na and df.stat is df itself.
+ df = session.createDataFrame([(1, None), (None, 2.0)], "a int, b double")
+ assert df.na.df is df
+ assert df.stat.df is df
+ assert sorted(df.na.df.na.fill(0).collect()) ==
sorted(df.na.fill(0).collect())
+ assert df.stat.df.count() == 2
+
+
def test_sample_by_samples_the_fractions_of_the_strata_as_pyspark(session):
# As PySpark's sampleBy: each stratum by its fraction, of a column name or
a Column.
df = session.createDataFrame([(1, "a"), (2, "b"), (3, "a")], "id int, k
string")
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 366d7b4..8f8e92e 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2835,6 +2835,19 @@ def test_functions_null_and_misc():
assert type(F.covar_samp(c, c)).__name__ == "Column"
+def test_na_and_stat_functions_are_of_their_dataframe_as_pyspark():
+ # As PySpark's DataFrameNaFunctions and DataFrameStatFunctions, whose df
is the DataFrame
+ # of df.na and df.stat itself.
+ from pyspark.sql import DataFrameNaFunctions, DataFrameStatFunctions
+
+ df = _offline_session().range(1)
+ assert isinstance(df.na, DataFrameNaFunctions)
+ assert isinstance(df.stat, DataFrameStatFunctions)
+ assert df.na.df is df
+ assert df.stat.df is df
+ assert df.na.df.stat.df is df
+
+
def test_sample_by_checks_its_arguments_as_pyspark():
# As PySpark's sampleBy: a column name or a Column, a dict of fractions,
and strata of
# floats, ints and strings.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]