This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 596382a  [SPARK-61001][CONNECT][PYTHON] Raise the errors of 
toLocalIterator, and execute with its first row, as PySpark does
596382a is described below

commit 596382a6cdb3f63fcce1f0dba96431ef3ac0097c
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 20:21:54 2026 +0900

    [SPARK-61001][CONNECT][PYTHON] Raise the errors of toLocalIterator, and 
execute with its first row, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61001](https://issues.apache.org/jira/browse/SPARK-61001): 
`DataFrame.toLocalIterator()` now raises the error of an execution that fails 
while it is iterated, as PySpark does, with or without `prefetchPartitions`. 
The error ends the iteration, as it ends PySpark's generator.
    
    PySpark's `toLocalIterator` is a generator, so the execution starts with 
the first row rather than with the call. Ours now does the same: an error of 
the execution, such as an analysis error, is raised by the iteration too, and 
an iterator that is never read executes nothing.
    
    ### Why are the changes needed?
    
    The iteration ended without an error, so the rows of a failing query looked 
complete. For `spark.sql("SELECT assert_true(id < 0) AS x FROM range(3)")`, 
`list(df.toLocalIterator())` raises `SparkRuntimeException` 
`USER_RAISED_EXCEPTION` in PySpark 4.3.0, and was `[]` here.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `toLocalIterator()` raises the errors of PySpark 4.3.0, and executes 
with its first row.
    
    ### How was this patch tested?
    
    - A new live test checks the error of a failing query, with and without 
prefetching, and the end of the iteration after it. It also passes with 
official PySpark 4.3.0, and fails without this change.
    - A new offline test checks, against a mock Connect server, that 
`toLocalIterator()` sends no request until the first row, which raises the 
error of a failing `ExecutePlan`, and that the iteration then ends.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (660 passed, and 123 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (911 passed, 9 skipped), 
and the live drop-in and Arrow suites (84 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #203 from HyukjinKwon/SPARK-61001-iterator-errors.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  | 57 ++++++++++++++++++++++++++-----------
 python/tests/test_dropin_e2e.py     | 12 ++++++++
 python/tests/test_dropin_offline.py | 16 +++++++++++
 3 files changed, 68 insertions(+), 17 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index 11a760b..dc9ead8 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -1289,19 +1289,13 @@ impl PyDataFrame {
 
     /// Return an iterator over rows without materializing the entire result 
in memory.
     ///
-    /// Mirrors 
`pyspark.sql.DataFrame.toLocalIterator(prefetchPartitions=False)`.
+    /// Mirrors 
`pyspark.sql.DataFrame.toLocalIterator(prefetchPartitions=False)`, a generator:
+    /// the execution starts with the first row, and its errors are raised by 
the iteration.
     /// Yields Row objects lazily as batches are fetched from the server.
     #[pyo3(name = "toLocalIterator")]
     #[pyo3(signature = (prefetchPartitions=false))]
-    fn to_local_iterator(
-        &self,
-        py: Python<'_>,
-        prefetchPartitions: bool,
-    ) -> PyResult<PyLocalRowIterator> {
-        let iterator = py
-            .detach(|| self.dataframe.to_local_iterator(prefetchPartitions))
-            .to_pyerr()?;
-        Ok(PyLocalRowIterator::new(iterator))
+    fn to_local_iterator(&self, prefetchPartitions: bool) -> 
PyLocalRowIterator {
+        PyLocalRowIterator::new(self.dataframe.clone(), prefetchPartitions)
     }
 
     /// Collect the DataFrame into a `pyarrow.Table`.
@@ -2821,19 +2815,38 @@ fn rows_to_py(
 /// Implements `__iter__` and `__next__` to make it a proper Python iterator.
 #[pyclass(name = "LocalRowIterator", module = "pyspark.sql.dataframe")]
 pub struct PyLocalRowIterator {
-    iterator: LocalRowIterator,
+    dataframe: DataFrame,
+    prefetch_partitions: bool,
+    /// The rows of the execution, which the first row starts, as PySpark's 
generator does.
+    iterator: Option<LocalRowIterator>,
+    /// Whether the iteration ended, with its last row or an error, as a 
generator ends.
+    finished: bool,
     /// The converters of the fields of user-defined types of the rows, once 
the schema of the
     /// result is known: with the first row.
     converters: Option<Option<FieldConverters>>,
 }
 
 impl PyLocalRowIterator {
-    pub fn new(iterator: LocalRowIterator) -> Self {
+    pub fn new(dataframe: DataFrame, prefetch_partitions: bool) -> Self {
         PyLocalRowIterator {
-            iterator,
+            dataframe,
+            prefetch_partitions,
+            iterator: None,
+            finished: false,
             converters: None,
         }
     }
+
+    /// The next row, starting the execution for the first.
+    fn next_row(&mut self) -> 
Option<spark_connect_core::error::Result<spark_connect::row::Row>> {
+        if self.iterator.is_none() {
+            match self.dataframe.to_local_iterator(self.prefetch_partitions) {
+                Ok(iterator) => self.iterator = Some(iterator),
+                Err(error) => return Some(Err(error)),
+            }
+        }
+        self.iterator.as_mut()?.next()
+    }
 }
 
 #[pymethods]
@@ -2847,18 +2860,28 @@ impl PyLocalRowIterator {
     ///
     /// Releases the GIL during the blocking fetch to allow other threads to 
run.
     fn __next__(&mut self, py: Python<'_>) -> PyResult<Option<PyRow>> {
+        if self.finished {
+            return Ok(None);
+        }
         // Release the GIL while fetching from the Rust iterator (which may 
block on network I/O).
-        match py.detach(|| self.iterator.next()) {
+        match py.detach(|| self.next_row()) {
             Some(Ok(row)) => {
                 if self.converters.is_none() {
-                    let schema = self.iterator.schema();
+                    let schema = 
self.iterator.as_ref().and_then(LocalRowIterator::schema);
                     self.converters = Some(udt_field_converters(py, 
schema.as_ref())?);
                 }
                 let converters = self.converters.clone().flatten();
                 Ok(Some(PyRow::with_converters(row, converters)))
             }
-            Some(Err(_e)) => Ok(None), // On error, stop iterating
-            None => Ok(None),
+            // As PySpark raises the error of the execution, which ends the 
iteration.
+            Some(Err(error)) => {
+                self.finished = true;
+                Err(crate::errors::spark_error_to_py_exception(error))
+            }
+            None => {
+                self.finished = true;
+                Ok(None)
+            }
         }
     }
 }
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 2896f8e..26a0d0c 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -54,6 +54,18 @@ def session():
         session.stop()
 
 
[email protected]("prefetch", [False, True])
+def 
test_to_local_iterator_raises_the_error_of_the_execution_as_pyspark(session, 
prefetch):
+    # As official PySpark 4.3.0: the error of a query that fails while it is 
iterated is
+    # raised by the iteration, which then ends, rather than ending it as if 
complete.
+    df = session.sql("SELECT assert_true(id < 0) AS x FROM range(3)")
+    rows = df.toLocalIterator(prefetch)
+    with pytest.raises(Exception) as error:
+        next(rows)
+    assert error.value.getCondition() == "USER_RAISED_EXCEPTION"
+    assert list(rows) == []
+
+
 def test_conf_get_with_a_none_default(session):
     key = "spark.connect.rust.test.unset"
     assert session.conf.get(key, None) is None
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index b0d18b1..27efd02 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6066,6 +6066,22 @@ def 
test_pandas_conversion_and_its_imports_are_pyspark_modules():
     assert 
inspect.isgeneratorfunction(pyspark.util._load_from_socket.__wrapped__)
 
 
+def 
test_to_local_iterator_executes_with_its_first_row_as_pyspark(mock_connect):
+    # As PySpark's generator: the execution starts with the first row, which 
raises its
+    # error, and the iteration then ends.
+    from pyspark.errors import PySparkException
+
+    spark, service = mock_connect
+    service.execute_failure = "plan rejected"
+    rows = spark.range(3).toLocalIterator()
+    assert service.execute_calls == 0
+    with pytest.raises(PySparkException, match="plan rejected"):
+        next(rows)
+    assert service.execute_calls == 1
+    assert list(rows) == []
+    assert service.execute_calls == 1
+
+
 def test_execute_plan_error_carries_operation_id(mock_connect):
     from pyspark.errors import PySparkException
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to