This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a5ec907  [SPARK-60961][CONNECT][PYTHON] Raise 
SparkConnectGrpcException for a gRPC error without an error of the server, as 
PySpark does
a5ec907 is described below

commit a5ec907f0d57172418630f9b214299b61a85276f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 16:47:13 2026 +0900

    [SPARK-60961][CONNECT][PYTHON] Raise SparkConnectGrpcException for a gRPC 
error without an error of the server, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60961](https://issues.apache.org/jira/browse/SPARK-60961): a gRPC 
error that carries no error of the server now raises 
`SparkConnectGrpcException` of its status code, which `getGrpcStatusCode()` 
returns, as PySpark's `_handle_rpc_error` raises it. Examples are an 
`UNAVAILABLE` of an unreachable server once the retries are exhausted, an 
`UNAUTHENTICATED` or `PERMISSION_DENIED` of a proxy, and a 
`RESOURCE_EXHAUSTED`. The message is the transport's, which names the status 
and give [...]
    
    The mock Connect server of the tests can now fail an `ExecutePlan` with any 
status code.
    
    ### Why are the changes needed?
    
    These errors raised `PySparkRuntimeError`, which is not a 
`SparkConnectGrpcException`. So `except SparkConnectGrpcException` no longer 
caught them, as it did while that class was an alias, and they had no status 
code.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these errors raise the exception of PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - Against a mock Connect server, an `ExecutePlan` failing with 
`INVALID_ARGUMENT`, `PERMISSION_DENIED`, `UNAUTHENTICATED`, 
`RESOURCE_EXHAUSTED` or `INTERNAL`, and an `AnalyzePlan` failing with 
`INVALID_ARGUMENT`, raise the same exception class and status code as in 
official PySpark 4.3.0, with the operation id of the request. Before this 
change, all of them differed. A `DEADLINE_EXCEEDED` of an execution's stream 
differs: PySpark 4.3.0 reattaches it under its per-RPC deadlines, which  [...]
    - New offline tests check these cases. They fail without this change.
    - The lib tests and every integration test, e2e included, against a local 
Spark 4.3.0 server (648 passed, and 123 of the transport crate), the build of 
all tests with `wasm-udf`, the offline Python suites (726 passed, 9 skipped), 
and the live drop-in and Arrow suites (76 passed) and live cases (10 passed). 
Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #197 from HyukjinKwon/SPARK-60961-grpc-exception.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/errors.rs      | 34 +++++++++++++++++++++++++++-------
 python/tests/_mock_connect_server.py | 10 ++++++++--
 python/tests/test_dropin_offline.py  | 28 ++++++++++++++++++++++++++++
 3 files changed, 63 insertions(+), 9 deletions(-)

diff --git a/crates/pyspark-rs/src/errors.rs b/crates/pyspark-rs/src/errors.rs
index 4ea7a0c..d82bc46 100644
--- a/crates/pyspark-rs/src/errors.rs
+++ b/crates/pyspark-rs/src/errors.rs
@@ -126,6 +126,23 @@ fn server_error_to_py<'py>(
     )
 }
 
+/// The exception for a gRPC error that carries no error of the server, such 
as an
+/// `UNAVAILABLE` of an unreachable server or a `PERMISSION_DENIED` of a 
proxy, as the
+/// reference client's `_handle_rpc_error` raises it: a 
`SparkConnectGrpcException` of the
+/// status code.
+fn grpc_error_to_py<'py>(py: Python<'py>, message: &str, code: i32) -> 
PyResult<Bound<'py, PyAny>> {
+    let connect = py.import("pyspark.errors.exceptions.connect")?;
+    let kwargs = PyDict::new(py);
+    kwargs.set_item("message", message)?;
+    kwargs.set_item(
+        "grpc_status_code",
+        connect.call_method1("_status_code", (code,))?,
+    )?;
+    connect
+        .getattr("SparkConnectGrpcException")?
+        .call((), Some(&kwargs))
+}
+
 /// Maps a SparkError to a Python exception, attempting to raise the correct 
typed
 /// exception based on the error kind. Falls back to PyRuntimeError if import 
fails.
 pub fn spark_error_to_py_exception(err: SparkError) -> PyErr {
@@ -135,14 +152,17 @@ pub fn spark_error_to_py_exception(err: SparkError) -> 
PyErr {
 
     // Try to acquire the GIL and import the exception class from 
pyspark.errors
     Python::attach(|py| {
-        if let Some(server) = &err.server {
-            // Should the connect classes not import, fall back to the class 
by kind below.
-            if let Ok(exc_instance) = server_error_to_py(py, &err, server) {
-                if let Some(op) = &operation_id {
-                    let _ = exc_instance.setattr("_operation_id", op);
-                }
-                return PyErr::from_value(exc_instance);
+        // Should the connect classes not import, fall back to the class by 
kind below.
+        let connect_error = match (&err.server, err.grpc_code) {
+            (Some(server), _) => server_error_to_py(py, &err, server).ok(),
+            (None, Some(code)) => grpc_error_to_py(py, &msg, code).ok(),
+            (None, None) => None,
+        };
+        if let Some(exc_instance) = connect_error {
+            if let Some(op) = &operation_id {
+                let _ = exc_instance.setattr("_operation_id", op);
             }
+            return PyErr::from_value(exc_instance);
         }
         // First, verify and get the exception class from pyspark.errors
         let errors_module = match py.import("pyspark.errors") {
diff --git a/python/tests/_mock_connect_server.py 
b/python/tests/_mock_connect_server.py
index b98ba0e..46278a0 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -147,7 +147,8 @@ def serve(connection):
 
         # ExecutePlan records each request and streams one response carrying
         # `execute_payload` (raw ExecutePlanResponse fields), then 
ResultComplete; with
-        # `execute_failure` set it aborts instead. Responses echo the request's
+        # `execute_failure` set it aborts instead, with INVALID_ARGUMENT and 
that message, or
+        # with the status code name and message of a tuple. Responses echo the 
request's
         # operation id, as the server does.
         execute_requests = []
         execute_payload = b""
@@ -173,7 +174,12 @@ def serve(connection):
                     "Python thread could not run while the statement ran",
                 )
             if self.execute_failure:
-                context.abort(grpc.StatusCode.INVALID_ARGUMENT, 
self.execute_failure)
+                code, message = (
+                    self.execute_failure
+                    if isinstance(self.execute_failure, tuple)
+                    else ("INVALID_ARGUMENT", self.execute_failure)
+                )
+                context.abort(getattr(grpc.StatusCode, code), message)
             if self.execute_error:
                 message, reason, metadata = self.execute_error
                 # google.rpc: Status.code=1, message=2, details=3; 
Any.type_url=1, value=2;
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index eaf0602..a8dd237 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5580,6 +5580,34 @@ def 
test_execute_plan_error_carries_operation_id(mock_connect):
     assert error.value.operation_id == 
_request_operation_id(service.execute_requests[-1])
 
 
[email protected]("code", [
+    "INVALID_ARGUMENT", "PERMISSION_DENIED", "UNAUTHENTICATED", 
"RESOURCE_EXHAUSTED", "INTERNAL",
+])
+def 
test_grpc_errors_without_details_are_spark_connect_grpc_exceptions(mock_connect,
 code):
+    # A gRPC error that carries no error of the server, as of a proxy, is a
+    # SparkConnectGrpcException of its status code, as PySpark's 
_handle_rpc_error raises it.
+    # The cases agree with official PySpark 4.3.0.
+    from pyspark.errors.exceptions.connect import SparkConnectGrpcException, 
StatusCode
+
+    spark, service = mock_connect
+    service.execute_failure = (code, "denied by the proxy")
+    with pytest.raises(SparkConnectGrpcException, match="denied by the proxy") 
as error:
+        spark.range(1).collect()
+    assert error.value.getGrpcStatusCode() == getattr(StatusCode, code)
+    assert error.value.getCondition() is None
+    assert error.value.operation_id == 
_request_operation_id(service.execute_requests[-1])
+
+
+def 
test_analyze_errors_without_details_are_spark_connect_grpc_exceptions(mock_connect):
+    from pyspark.errors.exceptions.connect import SparkConnectGrpcException, 
StatusCode
+
+    spark, service = mock_connect
+    service.failure = "rpc"  # INVALID_ARGUMENT "schema unavailable", of 
AnalyzePlan.
+    with pytest.raises(SparkConnectGrpcException, match="schema unavailable") 
as error:
+        spark.range(1).schema
+    assert error.value.getGrpcStatusCode() == StatusCode.INVALID_ARGUMENT
+
+
 def _error_info(classes, **metadata):
     """ErrorInfo metadata as the server sends it, with the exception classes, 
the most
     specific first."""


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to