This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 80d6dac [SPARK-61039][CONNECT][PYTHON] Send the eager and
storageLevel of checkpoint and localCheckpoint in the Python client as PySpark
does
80d6dac is described below
commit 80d6dac1f42f011a9067b3ad70872d580238012f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 06:19:04 2026 +0900
[SPARK-61039][CONNECT][PYTHON] Send the eager and storageLevel of
checkpoint and localCheckpoint in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61039](https://issues.apache.org/jira/browse/SPARK-61039):
`checkpoint` and `localCheckpoint` now send their `eager` and `storageLevel`,
as PySpark Connect does.
`df.checkpoint(eager)` and `df.localCheckpoint(eager, storageLevel)` send
`eager`, and for a local checkpoint the storage level, in their
`CheckpointCommand`. A checkpoint that is not eager is materialized when it is
first used, and a local one is kept at the storage level given.
The core takes them in the new `DataFrame::checkpoint_with` and
`local_checkpoint_with`. `DataFrame.persist`, `localCheckpoint` and
`Catalog.cacheTable` now convert a `StorageLevel` to its proto with one helper.
### Why are the changes needed?
The client sent every checkpoint as eager, at the default storage level,
whatever the arguments.
### Does this PR introduce _any_ user-facing change?
Yes. These checkpoints follow their `eager` and `storageLevel`, as in
PySpark 4.3.0.
### How was this patch tested?
- A new offline test checks the `local`, `eager` and `storage_level` of the
`CheckpointCommand`s that `checkpoint` and `localCheckpoint` send to a mock
Connect server, as official PySpark 4.3.0 sends them.
- An e2e test now checks a lazy local checkpoint on disk, and a lazy
checkpoint, against the server.
- The lib tests of both crates (568 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1800 passed, 8 skipped), and the
live drop-in and Arrow suites (105 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #233 from HyukjinKwon/SPARK-61039-checkpoint-args.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/catalog.rs | 13 +++------
crates/pyspark-rs/src/dataframe.rs | 39 +++++++++++++--------------
crates/pyspark-rs/src/storagelevel.rs | 14 ++++++++++
crates/spark-connect/src/dataframe.rs | 29 +++++++++++++++++---
crates/spark-connect/tests/e2e_new_methods.rs | 8 ++++++
python/tests/test_dropin_offline.py | 30 +++++++++++++++++++++
6 files changed, 99 insertions(+), 34 deletions(-)
diff --git a/crates/pyspark-rs/src/catalog.rs b/crates/pyspark-rs/src/catalog.rs
index 1e1e930..62c5cef 100644
--- a/crates/pyspark-rs/src/catalog.rs
+++ b/crates/pyspark-rs/src/catalog.rs
@@ -355,16 +355,9 @@ impl PyCatalog {
/// `None` leaves it unset so the server default applies.
#[pyo3(signature = (tableName, storageLevel=None))]
fn cacheTable(&self, tableName: &str, storageLevel: Option<&Bound<'_,
PyAny>>) -> PyResult<()> {
- let level = match storageLevel {
- Some(obj) => Some(spark_connect_proto::StorageLevel {
- use_disk: obj.getattr("useDisk")?.extract()?,
- use_memory: obj.getattr("useMemory")?.extract()?,
- use_off_heap: obj.getattr("useOffHeap")?.extract()?,
- deserialized: obj.getattr("deserialized")?.extract()?,
- replication: obj.getattr("replication")?.extract()?,
- }),
- None => None,
- };
+ let level = storageLevel
+ .map(crate::storagelevel::storage_level_proto)
+ .transpose()?;
self.catalog
.cache_table_with_storage_level(tableName, level)
.to_pyerr()
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index aa47f24..0c200cb 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1278,28 +1278,35 @@ impl PyDataFrame {
.to_pyerr()
}
- /// Reliable checkpoint (materialize + truncate lineage).
+ /// Reliable checkpoint (materialize + truncate lineage), at once if
`eager`, as PySpark's.
#[pyo3(signature = (eager=true))]
- #[allow(unused_variables)]
fn checkpoint(&self, py: Python<'_>, eager: bool) -> PyResult<PyDataFrame>
{
Ok(PyDataFrame::new(
- py.detach(|| self.dataframe.checkpoint()).to_pyerr()?,
+ py.detach(||
self.dataframe.checkpoint_with(eager).map_err(Box::new))
+ .to_pyerr()?,
))
}
- /// Local checkpoint. `storageLevel` is accepted for signature parity with
reference
- /// pyspark; the connect core's local checkpoint does not yet take a
storage level,
- /// so a supplied value is currently a no-op.
+ /// Local checkpoint, at once if `eager`, kept at `storageLevel`, else at
the server's
+ /// default, as PySpark's.
#[pyo3(name = "localCheckpoint", signature = (eager=true,
storageLevel=None))]
- #[allow(unused_variables)]
fn local_checkpoint(
&self,
py: Python<'_>,
eager: bool,
storageLevel: Option<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
+ let level = storageLevel
+ .filter(|level| !level.is_none())
+ .map(|level| crate::storagelevel::storage_level_proto(&level))
+ .transpose()?;
Ok(PyDataFrame::new(
- py.detach(|| self.dataframe.local_checkpoint()).to_pyerr()?,
+ py.detach(|| {
+ self.dataframe
+ .local_checkpoint_with(eager, level)
+ .map_err(Box::new)
+ })
+ .to_pyerr()?,
))
}
@@ -1530,8 +1537,8 @@ impl PyDataFrame {
self.unpivot(ids, values, variableColumnName, valueColumnName)
}
- /// Persist the DataFrame. A storage-level argument is accepted for API
parity;
- /// the default (MEMORY_AND_DISK) is used.
+ /// Persist the DataFrame at `storageLevel`, MEMORY_AND_DISK_DESER by
default, as
+ /// PySpark's `DataFrame.persist()`.
#[pyo3(signature = (storageLevel=None))]
fn persist(
&self,
@@ -1539,18 +1546,8 @@ impl PyDataFrame {
storageLevel: Option<Bound<'_, PyAny>>,
) -> PyResult<PyDataFrame> {
use spark_connect::storage::StorageLevelExt;
- // Honor the requested StorageLevel instead of always using
MEMORY_AND_DISK.
- // A pyspark StorageLevel exposes
useDisk/useMemory/useOffHeap/deserialized/
- // replication; map them onto the proto. Default (None) is
MEMORY_AND_DISK_DESER,
- // matching reference DataFrame.persist().
let level = match storageLevel {
- Some(obj) => spark_connect_proto::StorageLevel {
- use_disk: obj.getattr("useDisk")?.extract()?,
- use_memory: obj.getattr("useMemory")?.extract()?,
- use_off_heap: obj.getattr("useOffHeap")?.extract()?,
- deserialized: obj.getattr("deserialized")?.extract()?,
- replication: obj.getattr("replication")?.extract()?,
- },
+ Some(level) => crate::storagelevel::storage_level_proto(&level)?,
None => spark_connect_proto::StorageLevel::memory_and_disk_deser(),
};
let df = py.detach(|| self.dataframe.persist(level)).to_pyerr()?;
diff --git a/crates/pyspark-rs/src/storagelevel.rs
b/crates/pyspark-rs/src/storagelevel.rs
index 3fb410a..4157dce 100644
--- a/crates/pyspark-rs/src/storagelevel.rs
+++ b/crates/pyspark-rs/src/storagelevel.rs
@@ -37,6 +37,20 @@ impl PyStorageLevel {
}
}
+/// The proto of a pyspark `StorageLevel`, of its flags, as PySpark's
`storage_level_to_proto`
+/// reads them: of `DataFrame.persist`, `localCheckpoint` and
`Catalog.cacheTable`.
+pub(crate) fn storage_level_proto(
+ level: &Bound<'_, PyAny>,
+) -> PyResult<spark_connect_proto::StorageLevel> {
+ Ok(spark_connect_proto::StorageLevel {
+ use_disk: level.getattr("useDisk")?.extract()?,
+ use_memory: level.getattr("useMemory")?.extract()?,
+ use_off_heap: level.getattr("useOffHeap")?.extract()?,
+ deserialized: level.getattr("deserialized")?.extract()?,
+ replication: level.getattr("replication")?.extract()?,
+ })
+}
+
#[pymethods]
impl PyStorageLevel {
#[new]
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index 8d1c2ac..29b56b5 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -1191,22 +1191,45 @@ impl DataFrame {
/// Checkpoint this DataFrame to disk.
pub fn checkpoint(&self) -> Result<DataFrame> {
- self.checkpoint_impl(false, true)
+ self.checkpoint_with(true)
+ }
+
+ /// Checkpoint this DataFrame, at once if `eager`, else when it is first
used, mirroring
+ /// `DataFrame.checkpoint(eager)`.
+ pub fn checkpoint_with(&self, eager: bool) -> Result<DataFrame> {
+ self.checkpoint_impl(false, eager, None)
}
/// Create a local checkpoint of this DataFrame.
pub fn local_checkpoint(&self) -> Result<DataFrame> {
- self.checkpoint_impl(true, true)
+ self.local_checkpoint_with(true, None)
+ }
+
+ /// Create a local checkpoint of this DataFrame, at once if `eager`, kept
at
+ /// `storage_level`, else at the server's default, mirroring
+ /// `DataFrame.localCheckpoint(eager, storageLevel)`.
+ pub fn local_checkpoint_with(
+ &self,
+ eager: bool,
+ storage_level: Option<proto::StorageLevel>,
+ ) -> Result<DataFrame> {
+ self.checkpoint_impl(true, eager, storage_level)
}
/// Execute a `CheckpointCommand` and return a DataFrame referencing the
resulting
/// cached remote relation. Mirrors
`DataFrame.checkpoint`/`localCheckpoint`, which
/// materialize server-side and return a handle to the checkpointed data.
- fn checkpoint_impl(&self, local: bool, eager: bool) -> Result<DataFrame> {
+ fn checkpoint_impl(
+ &self,
+ local: bool,
+ eager: bool,
+ storage_level: Option<proto::StorageLevel>,
+ ) -> Result<DataFrame> {
let mut cmd = proto::CheckpointCommand::default();
cmd.relation = Some(self.relation());
cmd.local = local;
cmd.eager = eager;
+ cmd.storage_level = storage_level;
let (responses, info) = execute_command_with_info(
&self.session,
proto::command::CommandType::CheckpointCommand(cmd),
diff --git a/crates/spark-connect/tests/e2e_new_methods.rs
b/crates/spark-connect/tests/e2e_new_methods.rs
index 125fa51..3fa54e3 100644
--- a/crates/spark-connect/tests/e2e_new_methods.rs
+++ b/crates/spark-connect/tests/e2e_new_methods.rs
@@ -69,6 +69,14 @@ fn checkpoints_and_input_files_local() {
let lc = df.local_checkpoint().expect("local_checkpoint");
assert_eq!(lc.collect().expect("collect lc").len(), 10);
+ // A local checkpoint that is not eager, kept on disk, as
localCheckpoint(False, DISK_ONLY).
+ use spark_connect::storage::StorageLevelExt;
+ let lazy = df
+ .local_checkpoint_with(false,
Some(spark_connect_proto::StorageLevel::disk_only()))
+ .expect("local_checkpoint_with");
+ assert_eq!(lazy.count().expect("count lazy"), 10);
+ // A reliable checkpoint needs a checkpoint directory, which this server
has none of.
+ assert!(df.checkpoint_with(false).is_err());
// inputFiles on a computed relation is empty; is_local is false for a
range.
assert!(df.input_files().expect("input_files").is_empty());
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 593892b..6e3c250 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -367,6 +367,36 @@ def test_storage_level():
assert StorageLevel.DISK_ONLY != StorageLevel.MEMORY_ONLY
+def
test_checkpoints_send_their_eager_and_storage_level_as_pyspark(mock_connect):
+ # As PySpark's, checkpoint(eager) and localCheckpoint(eager, storageLevel)
send them in
+ # their CheckpointCommand, of which the mock answers with a
CachedRemoteRelation.
+ from pyspark.storagelevel import StorageLevel
+
+ spark, service = mock_connect
+ # ExecutePlanResponse.checkpoint_command_result=19 -> relation=1 ->
relation_id=1.
+ service.execute_payload = message_field(19, message_field(1,
message_field(1, "r1")))
+
+ def sent():
+ # ExecutePlanRequest.plan=3 -> Plan.command=2 ->
Command.checkpoint_command=14: local=2,
+ # eager=3, storage_level=4, where false is not sent.
+ plan = parse_message(parse_message(service.execute_requests[-1])[3][0])
+ return parse_message(parse_message(plan[2][0])[14][0])
+
+ df = spark.range(1)
+ for checkpoint, local, eager in ((lambda: df.checkpoint(), [], [1]),
+ (lambda: df.checkpoint(eager=False), [],
[]),
+ (lambda: df.localCheckpoint(), [1], [1]),
+ (lambda: df.localCheckpoint(False), [1],
[])):
+ assert type(checkpoint()).__name__ == "DataFrame"
+ command = sent()
+ assert (command.get(2, []), command.get(3, []), command.get(4)) ==
(local, eager, None)
+ df.localCheckpoint(storageLevel=StorageLevel.DISK_ONLY)
+ # StorageLevel: use_disk=1, use_memory=2, use_off_heap=3, deserialized=4,
replication=5.
+ assert parse_message(sent()[4][0]) == {1: [1], 5: [1]}
+ df.localCheckpoint(True, StorageLevel(False, True, True, True, 2))
+ assert parse_message(sent()[4][0]) == {2: [1], 3: [1], 4: [1], 5: [2]}
+
+
# ---------------------------------------------------------------------------
session builder
def test_session_builder_chaining_offline():
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]