This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 45ad1bb  [SPARK-61049][CONNECT][PYTHON] Expose the df of df.na and 
df.stat in the Python client as PySpark does
45ad1bb is described below

commit 45ad1bbda245b2f087a63d4ec78c43e29c18defb
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 11:52:07 2026 +0900

    [SPARK-61049][CONNECT][PYTHON] Expose the df of df.na and df.stat in the 
Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61049](https://issues.apache.org/jira/browse/SPARK-61049): the 
`DataFrameNaFunctions` of `df.na` and the `DataFrameStatFunctions` of `df.stat` 
now have the `df` of PySpark Connect's, the DataFrame itself, so `df.na.df is 
df`.
    
    ### Why are the changes needed?
    
    The client had no `df` of them: `df.na.df` raised an `AttributeError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `df.na.df` and `df.stat.df` are the DataFrame, as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks that `df.na.df` and `df.stat.df` are `df`, and 
runs functions of them. It passes with official PySpark 4.3.0, and fails 
without this change.
    - A new offline test checks the same of a DataFrame of an offline session.
    - The lib tests of both crates (574 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1813 passed, 8 skipped), and the 
live drop-in and Arrow suites (112 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #250 from HyukjinKwon/SPARK-61049-na-stat-df.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 23 +++++++++++++++++------
 crates/pyspark-rs/src/stat.rs       | 13 +++++++++++--
 python/tests/test_dropin_e2e.py     |  9 +++++++++
 python/tests/test_dropin_offline.py | 13 +++++++++++++
 4 files changed, 50 insertions(+), 8 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 3a1a041..bda68b4 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1294,11 +1294,11 @@ impl PyDataFrame {
         crate::stat_arguments::fillna(&self.dataframe, value, subset)
     }
 
-    /// Statistical functions (`df.stat`).
+    /// Statistical functions (`df.stat`), of this DataFrame, their `df`, as 
PySpark's.
     #[getter]
     #[pyo3(name = "stat")]
-    fn stat(&self) -> crate::stat::PyStatFunctions {
-        crate::stat::PyStatFunctions::new(self.dataframe.stat())
+    fn stat(slf: &Bound<'_, Self>) -> crate::stat::PyStatFunctions {
+        crate::stat::PyStatFunctions::new(slf.borrow().dataframe.stat(), 
slf.clone().unbind())
     }
 
     /// `df.crosstab(col1, col2)` - pyspark exposes it on DataFrame as well as
@@ -2098,11 +2098,13 @@ impl PyDataFrame {
         PyDataStreamWriter::new(writer)
     }
 
-    /// Null-handling functions (`df.na.drop()/fill()/replace()`).
+    /// Null-handling functions (`df.na.drop()/fill()/replace()`), of this 
DataFrame, their
+    /// `df`, as PySpark's.
     #[getter]
-    fn na(&self) -> PyDataFrameNaFunctions {
+    fn na(slf: &Bound<'_, Self>) -> PyDataFrameNaFunctions {
         PyDataFrameNaFunctions {
-            df: self.dataframe.clone(),
+            df: slf.borrow().dataframe.clone(),
+            frame: slf.clone().unbind(),
         }
     }
 
@@ -2313,10 +2315,19 @@ impl PyDataFrame {
 #[pyclass(name = "DataFrameNaFunctions", module = "pyspark.sql.dataframe")]
 pub struct PyDataFrameNaFunctions {
     df: spark_connect::dataframe::DataFrame,
+    /// The DataFrame of `df.na`, as PySpark's `DataFrameNaFunctions.df`.
+    frame: Py<PyDataFrame>,
 }
 
 #[pymethods]
 impl PyDataFrameNaFunctions {
+    /// The DataFrame of these functions, as PySpark's 
`DataFrameNaFunctions.df`: `df.na.df` is
+    /// `df`.
+    #[getter(df)]
+    fn frame(&self, py: Python<'_>) -> Py<PyDataFrame> {
+        self.frame.clone_ref(py)
+    }
+
     /// Drop rows containing nulls, as PySpark's `DataFrameNaFunctions.drop`, 
which is
     /// `DataFrame.dropna`.
     #[pyo3(
diff --git a/crates/pyspark-rs/src/stat.rs b/crates/pyspark-rs/src/stat.rs
index cc7ced3..445d293 100644
--- a/crates/pyspark-rs/src/stat.rs
+++ b/crates/pyspark-rs/src/stat.rs
@@ -9,16 +9,25 @@ use crate::dataframe::PyDataFrame;
 #[pyclass(name = "DataFrameStatFunctions", module = "pyspark.sql.dataframe")]
 pub struct PyStatFunctions {
     stat: StatFunctions,
+    /// The DataFrame of `df.stat`, as PySpark's `DataFrameStatFunctions.df`.
+    frame: Py<PyDataFrame>,
 }
 
 impl PyStatFunctions {
-    pub fn new(stat: StatFunctions) -> Self {
-        PyStatFunctions { stat }
+    pub fn new(stat: StatFunctions, frame: Py<PyDataFrame>) -> Self {
+        PyStatFunctions { stat, frame }
     }
 }
 
 #[pymethods]
 impl PyStatFunctions {
+    /// The DataFrame of these functions, as PySpark's 
`DataFrameStatFunctions.df`: `df.stat.df`
+    /// is `df`.
+    #[getter(df)]
+    fn frame(&self, py: Python<'_>) -> Py<PyDataFrame> {
+        self.frame.clone_ref(py)
+    }
+
     /// Pearson correlation between two numeric columns, as PySpark's, whose 
`method` is only
     /// "pearson".
     ///
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index fe999fb..d499e05 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1503,6 +1503,15 @@ def 
test_to_pandas_converts_the_columns_as_pyspark(session):
         session.conf.unset("spark.sql.session.timeZone")
 
 
+def test_na_and_stat_functions_are_of_their_dataframe_as_pyspark(session):
+    # As PySpark's: the df of df.na and df.stat is df itself.
+    df = session.createDataFrame([(1, None), (None, 2.0)], "a int, b double")
+    assert df.na.df is df
+    assert df.stat.df is df
+    assert sorted(df.na.df.na.fill(0).collect()) == 
sorted(df.na.fill(0).collect())
+    assert df.stat.df.count() == 2
+
+
 def test_sample_by_samples_the_fractions_of_the_strata_as_pyspark(session):
     # As PySpark's sampleBy: each stratum by its fraction, of a column name or 
a Column.
     df = session.createDataFrame([(1, "a"), (2, "b"), (3, "a")], "id int, k 
string")
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 366d7b4..8f8e92e 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -2835,6 +2835,19 @@ def test_functions_null_and_misc():
     assert type(F.covar_samp(c, c)).__name__ == "Column"
 
 
+def test_na_and_stat_functions_are_of_their_dataframe_as_pyspark():
+    # As PySpark's DataFrameNaFunctions and DataFrameStatFunctions, whose df 
is the DataFrame
+    # of df.na and df.stat itself.
+    from pyspark.sql import DataFrameNaFunctions, DataFrameStatFunctions
+
+    df = _offline_session().range(1)
+    assert isinstance(df.na, DataFrameNaFunctions)
+    assert isinstance(df.stat, DataFrameStatFunctions)
+    assert df.na.df is df
+    assert df.stat.df is df
+    assert df.na.df.stat.df is df
+
+
 def test_sample_by_checks_its_arguments_as_pyspark():
     # As PySpark's sampleBy: a column name or a Column, a dict of fractions, 
and strata of
     # floats, ints and strings.


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to