This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 596382a [SPARK-61001][CONNECT][PYTHON] Raise the errors of
toLocalIterator, and execute with its first row, as PySpark does
596382a is described below
commit 596382a6cdb3f63fcce1f0dba96431ef3ac0097c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 20:21:54 2026 +0900
[SPARK-61001][CONNECT][PYTHON] Raise the errors of toLocalIterator, and
execute with its first row, as PySpark does
### What changes were proposed in this pull request?
[SPARK-61001](https://issues.apache.org/jira/browse/SPARK-61001):
`DataFrame.toLocalIterator()` now raises the error of an execution that fails
while it is iterated, as PySpark does, with or without `prefetchPartitions`.
The error ends the iteration, as it ends PySpark's generator.
PySpark's `toLocalIterator` is a generator, so the execution starts with
the first row rather than with the call. Ours now does the same: an error of
the execution, such as an analysis error, is raised by the iteration too, and
an iterator that is never read executes nothing.
### Why are the changes needed?
The iteration ended without an error, so the rows of a failing query looked
complete. For `spark.sql("SELECT assert_true(id < 0) AS x FROM range(3)")`,
`list(df.toLocalIterator())` raises `SparkRuntimeException`
`USER_RAISED_EXCEPTION` in PySpark 4.3.0, and was `[]` here.
### Does this PR introduce _any_ user-facing change?
Yes, `toLocalIterator()` raises the errors of PySpark 4.3.0, and executes
with its first row.
### How was this patch tested?
- A new live test checks the error of a failing query, with and without
prefetching, and the end of the iteration after it. It also passes with
official PySpark 4.3.0, and fails without this change.
- A new offline test checks, against a mock Connect server, that
`toLocalIterator()` sends no request until the first row, which raises the
error of a failing `ExecutePlan`, and that the iteration then ends.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (660 passed, and 123 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (911 passed, 9 skipped),
and the live drop-in and Arrow suites (84 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #203 from HyukjinKwon/SPARK-61001-iterator-errors.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 57 ++++++++++++++++++++++++++-----------
python/tests/test_dropin_e2e.py | 12 ++++++++
python/tests/test_dropin_offline.py | 16 +++++++++++
3 files changed, 68 insertions(+), 17 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index 11a760b..dc9ead8 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1289,19 +1289,13 @@ impl PyDataFrame {
/// Return an iterator over rows without materializing the entire result
in memory.
///
- /// Mirrors
`pyspark.sql.DataFrame.toLocalIterator(prefetchPartitions=False)`.
+ /// Mirrors
`pyspark.sql.DataFrame.toLocalIterator(prefetchPartitions=False)`, a generator:
+ /// the execution starts with the first row, and its errors are raised by
the iteration.
/// Yields Row objects lazily as batches are fetched from the server.
#[pyo3(name = "toLocalIterator")]
#[pyo3(signature = (prefetchPartitions=false))]
- fn to_local_iterator(
- &self,
- py: Python<'_>,
- prefetchPartitions: bool,
- ) -> PyResult<PyLocalRowIterator> {
- let iterator = py
- .detach(|| self.dataframe.to_local_iterator(prefetchPartitions))
- .to_pyerr()?;
- Ok(PyLocalRowIterator::new(iterator))
+ fn to_local_iterator(&self, prefetchPartitions: bool) ->
PyLocalRowIterator {
+ PyLocalRowIterator::new(self.dataframe.clone(), prefetchPartitions)
}
/// Collect the DataFrame into a `pyarrow.Table`.
@@ -2821,19 +2815,38 @@ fn rows_to_py(
/// Implements `__iter__` and `__next__` to make it a proper Python iterator.
#[pyclass(name = "LocalRowIterator", module = "pyspark.sql.dataframe")]
pub struct PyLocalRowIterator {
- iterator: LocalRowIterator,
+ dataframe: DataFrame,
+ prefetch_partitions: bool,
+ /// The rows of the execution, which the first row starts, as PySpark's
generator does.
+ iterator: Option<LocalRowIterator>,
+ /// Whether the iteration ended, with its last row or an error, as a
generator ends.
+ finished: bool,
/// The converters of the fields of user-defined types of the rows, once
the schema of the
/// result is known: with the first row.
converters: Option<Option<FieldConverters>>,
}
impl PyLocalRowIterator {
- pub fn new(iterator: LocalRowIterator) -> Self {
+ pub fn new(dataframe: DataFrame, prefetch_partitions: bool) -> Self {
PyLocalRowIterator {
- iterator,
+ dataframe,
+ prefetch_partitions,
+ iterator: None,
+ finished: false,
converters: None,
}
}
+
+ /// The next row, starting the execution for the first.
+ fn next_row(&mut self) ->
Option<spark_connect_core::error::Result<spark_connect::row::Row>> {
+ if self.iterator.is_none() {
+ match self.dataframe.to_local_iterator(self.prefetch_partitions) {
+ Ok(iterator) => self.iterator = Some(iterator),
+ Err(error) => return Some(Err(error)),
+ }
+ }
+ self.iterator.as_mut()?.next()
+ }
}
#[pymethods]
@@ -2847,18 +2860,28 @@ impl PyLocalRowIterator {
///
/// Releases the GIL during the blocking fetch to allow other threads to
run.
fn __next__(&mut self, py: Python<'_>) -> PyResult<Option<PyRow>> {
+ if self.finished {
+ return Ok(None);
+ }
// Release the GIL while fetching from the Rust iterator (which may
block on network I/O).
- match py.detach(|| self.iterator.next()) {
+ match py.detach(|| self.next_row()) {
Some(Ok(row)) => {
if self.converters.is_none() {
- let schema = self.iterator.schema();
+ let schema =
self.iterator.as_ref().and_then(LocalRowIterator::schema);
self.converters = Some(udt_field_converters(py,
schema.as_ref())?);
}
let converters = self.converters.clone().flatten();
Ok(Some(PyRow::with_converters(row, converters)))
}
- Some(Err(_e)) => Ok(None), // On error, stop iterating
- None => Ok(None),
+ // As PySpark raises the error of the execution, which ends the
iteration.
+ Some(Err(error)) => {
+ self.finished = true;
+ Err(crate::errors::spark_error_to_py_exception(error))
+ }
+ None => {
+ self.finished = true;
+ Ok(None)
+ }
}
}
}
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 2896f8e..26a0d0c 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -54,6 +54,18 @@ def session():
session.stop()
[email protected]("prefetch", [False, True])
+def
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session,
prefetch):
+ # As official PySpark 4.3.0: the error of a query that fails while it is
iterated is
+ # raised by the iteration, which then ends, rather than ending it as if
complete.
+ df = session.sql("SELECT assert_true(id < 0) AS x FROM range(3)")
+ rows = df.toLocalIterator(prefetch)
+ with pytest.raises(Exception) as error:
+ next(rows)
+ assert error.value.getCondition() == "USER_RAISED_EXCEPTION"
+ assert list(rows) == []
+
+
def test_conf_get_with_a_none_default(session):
key = "spark.connect.rust.test.unset"
assert session.conf.get(key, None) is None
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b0d18b1..27efd02 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6066,6 +6066,22 @@ def
test_pandas_conversion_and_its_imports_are_pyspark_modules():
assert
inspect.isgeneratorfunction(pyspark.util._load_from_socket.__wrapped__)
+def
test_to_local_iterator_executes_with_its_first_row_as_pyspark(mock_connect):
+ # As PySpark's generator: the execution starts with the first row, which
raises its
+ # error, and the iteration then ends.
+ from pyspark.errors import PySparkException
+
+ spark, service = mock_connect
+ service.execute_failure = "plan rejected"
+ rows = spark.range(3).toLocalIterator()
+ assert service.execute_calls == 0
+ with pytest.raises(PySparkException, match="plan rejected"):
+ next(rows)
+ assert service.execute_calls == 1
+ assert list(rows) == []
+ assert service.execute_calls == 1
+
+
def test_execute_plan_error_carries_operation_id(mock_connect):
from pyspark.errors import PySparkException
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]