This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7093835 [SPARK-61042][CONNECT][PYTHON] Return the DataFrame itself
from persist, cache and unpersist in the Python client as PySpark does
7093835 is described below
commit 7093835cabfc72280de98144bf33fe58f9720a1b
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 09:03:06 2026 +0900
[SPARK-61042][CONNECT][PYTHON] Return the DataFrame itself from persist,
cache and unpersist in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61042](https://issues.apache.org/jira/browse/SPARK-61042):
`persist`, `cache` and `unpersist` now return the DataFrame itself, as PySpark
Connect's do.
`df.persist(storageLevel)`, `df.cache()` and `df.unpersist(blocking)`
persist or unpersist the plan of the DataFrame and return the DataFrame itself,
so `df.persist() is df`.
### Why are the changes needed?
The client returned a new DataFrame object of the same plan, so
`df.persist() is df` was `False`.
### Does this PR introduce _any_ user-facing change?
Yes. These methods return the DataFrame itself, as in PySpark 4.3.0.
### How was this patch tested?
- A new live test checks that these return the DataFrame itself, at the
storage level given. It passes with official PySpark 4.3.0, and fails without
this change.
- A new offline test checks the same against a mock Connect server.
- The lib tests of both crates (571 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1810 passed, 8 skipped), and the
live drop-in and Arrow suites (109 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #240 from HyukjinKwon/SPARK-61042-persist-self.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 36 +++++++++++++++++++++++-------------
python/tests/test_dropin_e2e.py | 14 ++++++++++++++
python/tests/test_dropin_offline.py | 13 +++++++++++++
3 files changed, 50 insertions(+), 13 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index c5a7aa1..f6518ba 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1106,18 +1106,25 @@ impl PyDataFrame {
}
/// Persist with the default MEMORY_AND_DISK storage level.
- fn cache(&self, py: Python<'_>) -> PyResult<PyDataFrame> {
- let df = py.detach(|| self.dataframe.cache()).to_pyerr()?;
- Ok(PyDataFrame::new(df))
+ fn cache<'py>(slf: PyRef<'py, Self>, py: Python<'py>) ->
PyResult<PyRef<'py, Self>> {
+ // As PySpark's, which returns the DataFrame itself.
+ let df = slf.dataframe.clone();
+ py.detach(|| df.cache().map_err(Box::new)).to_pyerr()?;
+ Ok(slf)
}
/// Mark the DataFrame as non-persistent.
#[pyo3(signature = (blocking=false))]
- fn unpersist(&self, py: Python<'_>, blocking: bool) ->
PyResult<PyDataFrame> {
- let df = py
- .detach(|| self.dataframe.unpersist(blocking))
+ fn unpersist<'py>(
+ slf: PyRef<'py, Self>,
+ py: Python<'py>,
+ blocking: bool,
+ ) -> PyResult<PyRef<'py, Self>> {
+ // As PySpark's, which returns the DataFrame itself.
+ let df = slf.dataframe.clone();
+ py.detach(|| df.unpersist(blocking).map_err(Box::new))
.to_pyerr()?;
- Ok(PyDataFrame::new(df))
+ Ok(slf)
}
/// Print the plan with Python's `print`, as PySpark's
`DataFrame.explain`, which prints
@@ -1567,18 +1574,21 @@ impl PyDataFrame {
/// Persist the DataFrame at `storageLevel`, MEMORY_AND_DISK_DESER by
default, as
/// PySpark's `DataFrame.persist()`.
#[pyo3(signature = (storageLevel=None))]
- fn persist(
- &self,
- py: Python<'_>,
+ fn persist<'py>(
+ slf: PyRef<'py, Self>,
+ py: Python<'py>,
storageLevel: Option<Bound<'_, PyAny>>,
- ) -> PyResult<PyDataFrame> {
+ ) -> PyResult<PyRef<'py, Self>> {
use spark_connect::storage::StorageLevelExt;
let level = match storageLevel {
Some(level) => crate::storagelevel::storage_level_proto(&level)?,
None => spark_connect_proto::StorageLevel::memory_and_disk_deser(),
};
- let df = py.detach(|| self.dataframe.persist(level)).to_pyerr()?;
- Ok(PyDataFrame::new(df))
+ // As PySpark's, which returns the DataFrame itself.
+ let df = slf.dataframe.clone();
+ py.detach(|| df.persist(level).map_err(Box::new))
+ .to_pyerr()?;
+ Ok(slf)
}
/// Column access by attribute (`df.colname` -> Column), mirroring
reference pyspark: the
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d4dcf83..710e58f 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1662,6 +1662,20 @@ def
test_create_dataframe_raises_merge_errors_as_pyspark(session):
assert error.value.getMessageParameters() == parameters, data
+def
test_persist_cache_and_unpersist_return_the_dataframe_itself_as_pyspark(session):
+ # As PySpark's: the DataFrame itself, cached at the storage level given.
+ from pyspark.storagelevel import StorageLevel
+
+ df = session.range(3)
+ try:
+ assert df.persist(StorageLevel.DISK_ONLY) is df
+ assert df.storageLevel == StorageLevel.DISK_ONLY
+ assert df.unpersist(blocking=True) is df
+ assert df.cache() is df and df.count() == 3
+ finally:
+ df.unpersist()
+
+
def test_create_dataframe_converts_numpy_scalars_as_pyspark(session):
np = pytest.importorskip("numpy")
df = session.createDataFrame(
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index bb243cb..fd9bfd9 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -399,6 +399,19 @@ def
test_checkpoints_send_their_eager_and_storage_level_as_pyspark(mock_connect)
assert parse_message(sent()[4][0]) == {2: [1], 3: [1], 4: [1], 5: [2]}
+def
test_persist_cache_and_unpersist_return_the_dataframe_itself_as_pyspark(mock_connect):
+ # As PySpark's, which persist the plan of the DataFrame and return it, not
a copy.
+ from pyspark.storagelevel import StorageLevel
+
+ spark, _ = mock_connect
+ df = spark.range(1)
+ assert df.persist() is df
+ assert df.persist(StorageLevel.DISK_ONLY) is df
+ assert df.cache() is df
+ assert df.unpersist() is df
+ assert df.unpersist(blocking=True) is df
+
+
# ---------------------------------------------------------------------------
session builder
def test_session_builder_chaining_offline():
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]