This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 80d6dac  [SPARK-61039][CONNECT][PYTHON] Send the eager and 
storageLevel of checkpoint and localCheckpoint in the Python client as PySpark 
does
80d6dac is described below

commit 80d6dac1f42f011a9067b3ad70872d580238012f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 06:19:04 2026 +0900

    [SPARK-61039][CONNECT][PYTHON] Send the eager and storageLevel of 
checkpoint and localCheckpoint in the Python client as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61039](https://issues.apache.org/jira/browse/SPARK-61039): 
`checkpoint` and `localCheckpoint` now send their `eager` and `storageLevel`, 
as PySpark Connect does.
    
    `df.checkpoint(eager)` and `df.localCheckpoint(eager, storageLevel)` send 
`eager`, and for a local checkpoint the storage level, in their 
`CheckpointCommand`. A checkpoint that is not eager is materialized when it is 
first used, and a local one is kept at the storage level given.
    
    The core takes them in the new `DataFrame::checkpoint_with` and 
`local_checkpoint_with`. `DataFrame.persist`, `localCheckpoint` and 
`Catalog.cacheTable` now convert a `StorageLevel` to its proto with one helper.
    
    ### Why are the changes needed?
    
    The client sent every checkpoint as eager, at the default storage level, 
whatever the arguments.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. These checkpoints follow their `eager` and `storageLevel`, as in 
PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new offline test checks the `local`, `eager` and `storage_level` of the 
`CheckpointCommand`s that `checkpoint` and `localCheckpoint` send to a mock 
Connect server, as official PySpark 4.3.0 sends them.
    - An e2e test now checks a lazy local checkpoint on disk, and a lazy 
checkpoint, against the server.
    - The lib tests of both crates (568 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (1800 passed, 8 skipped), and the 
live drop-in and Arrow suites (105 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #233 from HyukjinKwon/SPARK-61039-checkpoint-args.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/catalog.rs              | 13 +++------
 crates/pyspark-rs/src/dataframe.rs            | 39 +++++++++++++--------------
 crates/pyspark-rs/src/storagelevel.rs         | 14 ++++++++++
 crates/spark-connect/src/dataframe.rs         | 29 +++++++++++++++++---
 crates/spark-connect/tests/e2e_new_methods.rs |  8 ++++++
 python/tests/test_dropin_offline.py           | 30 +++++++++++++++++++++
 6 files changed, 99 insertions(+), 34 deletions(-)

diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs
index 1e1e930..62c5cef 100644
--- a/crates/pyspark-rs/src/catalog.rs
+++ b/crates/pyspark-rs/src/catalog.rs
@@ -355,16 +355,9 @@ impl PyCatalog {
     /// `None` leaves it unset so the server default applies.
     #[pyo3(signature = (tableName, storageLevel=None))]
     fn cacheTable(&self, tableName: &str, storageLevel: Option<&Bound<'_, 
PyAny>>) -> PyResult<()> {
-        let level = match storageLevel {
-            Some(obj) => Some(spark_connect_proto::StorageLevel {
-                use_disk: obj.getattr("useDisk")?.extract()?,
-                use_memory: obj.getattr("useMemory")?.extract()?,
-                use_off_heap: obj.getattr("useOffHeap")?.extract()?,
-                deserialized: obj.getattr("deserialized")?.extract()?,
-                replication: obj.getattr("replication")?.extract()?,
-            }),
-            None => None,
-        };
+        let level = storageLevel
+            .map(crate::storagelevel::storage_level_proto)
+            .transpose()?;
         self.catalog
             .cache_table_with_storage_level(tableName, level)
             .to_pyerr()
diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index aa47f24..0c200cb 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1278,28 +1278,35 @@ impl PyDataFrame {
             .to_pyerr()
     }
 
-    /// Reliable checkpoint (materialize + truncate lineage).
+    /// Reliable checkpoint (materialize + truncate lineage), at once if 
`eager`, as PySpark's.
     #[pyo3(signature = (eager=true))]
-    #[allow(unused_variables)]
     fn checkpoint(&self, py: Python<'_>, eager: bool) -> PyResult<PyDataFrame> 
{
         Ok(PyDataFrame::new(
-            py.detach(|| self.dataframe.checkpoint()).to_pyerr()?,
+            py.detach(|| 
self.dataframe.checkpoint_with(eager).map_err(Box::new))
+                .to_pyerr()?,
         ))
     }
 
-    /// Local checkpoint. `storageLevel` is accepted for signature parity with 
reference
-    /// pyspark; the connect core's local checkpoint does not yet take a 
storage level,
-    /// so a supplied value is currently a no-op.
+    /// Local checkpoint, at once if `eager`, kept at `storageLevel`, else at 
the server's
+    /// default, as PySpark's.
     #[pyo3(name = "localCheckpoint", signature = (eager=true, 
storageLevel=None))]
-    #[allow(unused_variables)]
     fn local_checkpoint(
         &self,
         py: Python<'_>,
         eager: bool,
         storageLevel: Option<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
+        let level = storageLevel
+            .filter(|level| !level.is_none())
+            .map(|level| crate::storagelevel::storage_level_proto(&level))
+            .transpose()?;
         Ok(PyDataFrame::new(
-            py.detach(|| self.dataframe.local_checkpoint()).to_pyerr()?,
+            py.detach(|| {
+                self.dataframe
+                    .local_checkpoint_with(eager, level)
+                    .map_err(Box::new)
+            })
+            .to_pyerr()?,
         ))
     }
 
@@ -1530,8 +1537,8 @@ impl PyDataFrame {
         self.unpivot(ids, values, variableColumnName, valueColumnName)
     }
 
-    /// Persist the DataFrame. A storage-level argument is accepted for API 
parity;
-    /// the default (MEMORY_AND_DISK) is used.
+    /// Persist the DataFrame at `storageLevel`, MEMORY_AND_DISK_DESER by 
default, as
+    /// PySpark's `DataFrame.persist()`.
     #[pyo3(signature = (storageLevel=None))]
     fn persist(
         &self,
@@ -1539,18 +1546,8 @@ impl PyDataFrame {
         storageLevel: Option<Bound<'_, PyAny>>,
     ) -> PyResult<PyDataFrame> {
         use spark_connect::storage::StorageLevelExt;
-        // Honor the requested StorageLevel instead of always using 
MEMORY_AND_DISK.
-        // A pyspark StorageLevel exposes 
useDisk/useMemory/useOffHeap/deserialized/
-        // replication; map them onto the proto. Default (None) is 
MEMORY_AND_DISK_DESER,
-        // matching reference DataFrame.persist().
         let level = match storageLevel {
-            Some(obj) => spark_connect_proto::StorageLevel {
-                use_disk: obj.getattr("useDisk")?.extract()?,
-                use_memory: obj.getattr("useMemory")?.extract()?,
-                use_off_heap: obj.getattr("useOffHeap")?.extract()?,
-                deserialized: obj.getattr("deserialized")?.extract()?,
-                replication: obj.getattr("replication")?.extract()?,
-            },
+            Some(level) => crate::storagelevel::storage_level_proto(&level)?,
             None => spark_connect_proto::StorageLevel::memory_and_disk_deser(),
         };
         let df = py.detach(|| self.dataframe.persist(level)).to_pyerr()?;
diff --git a/crates/pyspark-rs/src/storagelevel.rs 
b/crates/pyspark-rs/src/storagelevel.rs
index 3fb410a..4157dce 100644
--- a/crates/pyspark-rs/src/storagelevel.rs
+++ b/crates/pyspark-rs/src/storagelevel.rs
@@ -37,6 +37,20 @@ impl PyStorageLevel {
     }
 }
 
+/// The proto of a pyspark `StorageLevel`, of its flags, as PySpark's 
`storage_level_to_proto`
+/// reads them: of `DataFrame.persist`, `localCheckpoint` and 
`Catalog.cacheTable`.
+pub(crate) fn storage_level_proto(
+    level: &Bound<'_, PyAny>,
+) -> PyResult<spark_connect_proto::StorageLevel> {
+    Ok(spark_connect_proto::StorageLevel {
+        use_disk: level.getattr("useDisk")?.extract()?,
+        use_memory: level.getattr("useMemory")?.extract()?,
+        use_off_heap: level.getattr("useOffHeap")?.extract()?,
+        deserialized: level.getattr("deserialized")?.extract()?,
+        replication: level.getattr("replication")?.extract()?,
+    })
+}
+
 #[pymethods]
 impl PyStorageLevel {
     #[new]
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index 8d1c2ac..29b56b5 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1191,22 +1191,45 @@ impl DataFrame {
 
     /// Checkpoint this DataFrame to disk.
     pub fn checkpoint(&self) -> Result<DataFrame> {
-        self.checkpoint_impl(false, true)
+        self.checkpoint_with(true)
+    }
+
+    /// Checkpoint this DataFrame, at once if `eager`, else when it is first 
used, mirroring
+    /// `DataFrame.checkpoint(eager)`.
+    pub fn checkpoint_with(&self, eager: bool) -> Result<DataFrame> {
+        self.checkpoint_impl(false, eager, None)
     }
 
     /// Create a local checkpoint of this DataFrame.
     pub fn local_checkpoint(&self) -> Result<DataFrame> {
-        self.checkpoint_impl(true, true)
+        self.local_checkpoint_with(true, None)
+    }
+
+    /// Create a local checkpoint of this DataFrame, at once if `eager`, kept 
at
+    /// `storage_level`, else at the server's default, mirroring
+    /// `DataFrame.localCheckpoint(eager, storageLevel)`.
+    pub fn local_checkpoint_with(
+        &self,
+        eager: bool,
+        storage_level: Option<proto::StorageLevel>,
+    ) -> Result<DataFrame> {
+        self.checkpoint_impl(true, eager, storage_level)
     }
 
     /// Execute a `CheckpointCommand` and return a DataFrame referencing the 
resulting
     /// cached remote relation. Mirrors 
`DataFrame.checkpoint`/`localCheckpoint`, which
     /// materialize server-side and return a handle to the checkpointed data.
-    fn checkpoint_impl(&self, local: bool, eager: bool) -> Result<DataFrame> {
+    fn checkpoint_impl(
+        &self,
+        local: bool,
+        eager: bool,
+        storage_level: Option<proto::StorageLevel>,
+    ) -> Result<DataFrame> {
         let mut cmd = proto::CheckpointCommand::default();
         cmd.relation = Some(self.relation());
         cmd.local = local;
         cmd.eager = eager;
+        cmd.storage_level = storage_level;
         let (responses, info) = execute_command_with_info(
             &self.session,
             proto::command::CommandType::CheckpointCommand(cmd),
diff --git a/crates/spark-connect/tests/e2e_new_methods.rs 
b/crates/spark-connect/tests/e2e_new_methods.rs
index 125fa51..3fa54e3 100644
--- a/crates/spark-connect/tests/e2e_new_methods.rs
+++ b/crates/spark-connect/tests/e2e_new_methods.rs
@@ -69,6 +69,14 @@ fn checkpoints_and_input_files_local() {
 
     let lc = df.local_checkpoint().expect("local_checkpoint");
     assert_eq!(lc.collect().expect("collect lc").len(), 10);
+    // A local checkpoint that is not eager, kept on disk, as 
localCheckpoint(False, DISK_ONLY).
+    use spark_connect::storage::StorageLevelExt;
+    let lazy = df
+        .local_checkpoint_with(false, 
Some(spark_connect_proto::StorageLevel::disk_only()))
+        .expect("local_checkpoint_with");
+    assert_eq!(lazy.count().expect("count lazy"), 10);
+    // A reliable checkpoint needs a checkpoint directory, which this server 
has none of.
+    assert!(df.checkpoint_with(false).is_err());
 
     // inputFiles on a computed relation is empty; is_local is false for a 
range.
     assert!(df.input_files().expect("input_files").is_empty());
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 593892b..6e3c250 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -367,6 +367,36 @@ def test_storage_level():
     assert StorageLevel.DISK_ONLY != StorageLevel.MEMORY_ONLY
 
 
+def 
test_checkpoints_send_their_eager_and_storage_level_as_pyspark(mock_connect):
+    # As PySpark's, checkpoint(eager) and localCheckpoint(eager, storageLevel) 
send them in
+    # their CheckpointCommand, of which the mock answers with a 
CachedRemoteRelation.
+    from pyspark.storagelevel import StorageLevel
+
+    spark, service = mock_connect
+    # ExecutePlanResponse.checkpoint_command_result=19 -> relation=1 -> 
relation_id=1.
+    service.execute_payload = message_field(19, message_field(1, 
message_field(1, "r1")))
+
+    def sent():
+        # ExecutePlanRequest.plan=3 -> Plan.command=2 -> 
Command.checkpoint_command=14: local=2,
+        # eager=3, storage_level=4, where false is not sent.
+        plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+        return parse_message(parse_message(plan[2][0])[14][0])
+
+    df = spark.range(1)
+    for checkpoint, local, eager in ((lambda: df.checkpoint(), [], [1]),
+                                     (lambda: df.checkpoint(eager=False), [], 
[]),
+                                     (lambda: df.localCheckpoint(), [1], [1]),
+                                     (lambda: df.localCheckpoint(False), [1], 
[])):
+        assert type(checkpoint()).__name__ == "DataFrame"
+        command = sent()
+        assert (command.get(2, []), command.get(3, []), command.get(4)) == 
(local, eager, None)
+    df.localCheckpoint(storageLevel=StorageLevel.DISK_ONLY)
+    # StorageLevel: use_disk=1, use_memory=2, use_off_heap=3, deserialized=4, 
replication=5.
+    assert parse_message(sent()[4][0]) == {1: [1], 5: [1]}
+    df.localCheckpoint(True, StorageLevel(False, True, True, True, 2))
+    assert parse_message(sent()[4][0]) == {2: [1], 3: [1], 4: [1], 5: [2]}
+
+
 # --------------------------------------------------------------------------- 
session builder
 
 def test_session_builder_chaining_offline():


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to