This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 7093835  [SPARK-61042][CONNECT][PYTHON] Return the DataFrame itself 
from persist, cache and unpersist in the Python client as PySpark does
7093835 is described below

commit 7093835cabfc72280de98144bf33fe58f9720a1b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:03:06 2026 +0900

    [SPARK-61042][CONNECT][PYTHON] Return the DataFrame itself from persist, 
cache and unpersist in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61042](https://issues.apache.org/jira/browse/SPARK-61042): 
`persist`, `cache` and `unpersist` now return the DataFrame itself, as PySpark 
Connect's do.
    
    `df.persist(storageLevel)`, `df.cache()` and `df.unpersist(blocking)` 
persist or unpersist the plan of the DataFrame and return the DataFrame itself, 
so `df.persist() is df`.
    
    ### Why are the changes needed?
    
    The client returned a new DataFrame object of the same plan, so 
`df.persist() is df` was `False`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These methods return the DataFrame itself, as in PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks that these return the DataFrame itself, at the 
storage level given. It passes with official PySpark 4.3.0, and fails without 
this change.
    - A new offline test checks the same against a mock Connect server.
    - The lib tests of both crates (571 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the 
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #240 from HyukjinKwon/SPARK-61042-persist-self.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 36 +++++++++++++++++++++++-------------
 python/tests/test_dropin_e2e.py     | 14 ++++++++++++++
 python/tests/test_dropin_offline.py | 13 +++++++++++++
 3 files changed, 50 insertions(+), 13 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index c5a7aa1..f6518ba 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1106,18 +1106,25 @@ impl PyDataFrame {
     }
 
     /// Persist with the default MEMORY_AND_DISK storage level.
-    fn cache(&self, py: Python<'_>) -> PyResult<PyDataFrame> {
-        let df = py.detach(|| self.dataframe.cache()).to_pyerr()?;
-        Ok(PyDataFrame::new(df))
+    fn cache<'py>(slf: PyRef<'py, Self>, py: Python<'py>) -> 
PyResult<PyRef<'py, Self>> {
+        // As PySpark's, which returns the DataFrame itself.
+        let df = slf.dataframe.clone();
+        py.detach(|| df.cache().map_err(Box::new)).to_pyerr()?;
+        Ok(slf)
     }
 
     /// Mark the DataFrame as non-persistent.
     #[pyo3(signature = (blocking=false))]
-    fn unpersist(&self, py: Python<'_>, blocking: bool) -> 
PyResult<PyDataFrame> {
-        let df = py
-            .detach(|| self.dataframe.unpersist(blocking))
+    fn unpersist<'py>(
+        slf: PyRef<'py, Self>,
+        py: Python<'py>,
+        blocking: bool,
+    ) -> PyResult<PyRef<'py, Self>> {
+        // As PySpark's, which returns the DataFrame itself.
+        let df = slf.dataframe.clone();
+        py.detach(|| df.unpersist(blocking).map_err(Box::new))
             .to_pyerr()?;
-        Ok(PyDataFrame::new(df))
+        Ok(slf)
     }
 
     /// Print the plan with Python's `print`, as PySpark's 
`DataFrame.explain`, which prints
@@ -1567,18 +1574,21 @@ impl PyDataFrame {
     /// Persist the DataFrame at `storageLevel`, MEMORY_AND_DISK_DESER by 
default, as
     /// PySpark's `DataFrame.persist()`.
     #[pyo3(signature = (storageLevel=None))]
-    fn persist(
-        &self,
-        py: Python<'_>,
+    fn persist<'py>(
+        slf: PyRef<'py, Self>,
+        py: Python<'py>,
         storageLevel: Option<Bound<'_, PyAny>>,
-    ) -> PyResult<PyDataFrame> {
+    ) -> PyResult<PyRef<'py, Self>> {
         use spark_connect::storage::StorageLevelExt;
         let level = match storageLevel {
             Some(level) => crate::storagelevel::storage_level_proto(&level)?,
             None => spark_connect_proto::StorageLevel::memory_and_disk_deser(),
         };
-        let df = py.detach(|| self.dataframe.persist(level)).to_pyerr()?;
-        Ok(PyDataFrame::new(df))
+        // As PySpark's, which returns the DataFrame itself.
+        let df = slf.dataframe.clone();
+        py.detach(|| df.persist(level).map_err(Box::new))
+            .to_pyerr()?;
+        Ok(slf)
     }
 
     /// Column access by attribute (`df.colname` -> Column), mirroring 
reference pyspark: the
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d4dcf83..710e58f 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1662,6 +1662,20 @@ def 
test_create_dataframe_raises_merge_errors_as_pyspark(session):
         assert error.value.getMessageParameters() == parameters, data
 
 
+def 
test_persist_cache_and_unpersist_return_the_dataframe_itself_as_pyspark(session):
+    # As PySpark's: the DataFrame itself, cached at the storage level given.
+    from pyspark.storagelevel import StorageLevel
+
+    df = session.range(3)
+    try:
+        assert df.persist(StorageLevel.DISK_ONLY) is df
+        assert df.storageLevel == StorageLevel.DISK_ONLY
+        assert df.unpersist(blocking=True) is df
+        assert df.cache() is df and df.count() == 3
+    finally:
+        df.unpersist()
+
+
 def test_create_dataframe_converts_numpy_scalars_as_pyspark(session):
     np = pytest.importorskip("numpy")
     df = session.createDataFrame(
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index bb243cb..fd9bfd9 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -399,6 +399,19 @@ def 
test_checkpoints_send_their_eager_and_storage_level_as_pyspark(mock_connect)
     assert parse_message(sent()[4][0]) == {2: [1], 3: [1], 4: [1], 5: [2]}
 
 
+def 
test_persist_cache_and_unpersist_return_the_dataframe_itself_as_pyspark(mock_connect):
+    # As PySpark's, which persist the plan of the DataFrame and return it, not 
a copy.
+    from pyspark.storagelevel import StorageLevel
+
+    spark, _ = mock_connect
+    df = spark.range(1)
+    assert df.persist() is df
+    assert df.persist(StorageLevel.DISK_ONLY) is df
+    assert df.cache() is df
+    assert df.unpersist() is df
+    assert df.unpersist(blocking=True) is df
+
+
 # --------------------------------------------------------------------------- 
session builder
 
 def test_session_builder_chaining_offline():


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to