This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new a5ec907 [SPARK-60961][CONNECT][PYTHON] Raise
SparkConnectGrpcException for a gRPC error without an error of the server, as
PySpark does
a5ec907 is described below
commit a5ec907f0d57172418630f9b214299b61a85276f
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 16:47:13 2026 +0900
[SPARK-60961][CONNECT][PYTHON] Raise SparkConnectGrpcException for a gRPC
error without an error of the server, as PySpark does
### What changes were proposed in this pull request?
[SPARK-60961](https://issues.apache.org/jira/browse/SPARK-60961): a gRPC
error that carries no error of the server now raises
`SparkConnectGrpcException` of its status code, which `getGrpcStatusCode()`
returns, as PySpark's `_handle_rpc_error` raises it. Examples are an
`UNAVAILABLE` of an unreachable server once the retries are exhausted, an
`UNAUTHENTICATED` or `PERMISSION_DENIED` of a proxy, and a
`RESOURCE_EXHAUSTED`. The message is the transport's, which names the status
and give [...]
The mock Connect server of the tests can now fail an `ExecutePlan` with any
status code.
### Why are the changes needed?
These errors raised `PySparkRuntimeError`, which is not a
`SparkConnectGrpcException`. So `except SparkConnectGrpcException` no longer
caught them, as it did while that class was an alias, and they had no status
code.
### Does this PR introduce _any_ user-facing change?
Yes, these errors raise the exception of PySpark 4.3.0.
### How was this patch tested?
- Against a mock Connect server, an `ExecutePlan` failing with
`INVALID_ARGUMENT`, `PERMISSION_DENIED`, `UNAUTHENTICATED`,
`RESOURCE_EXHAUSTED` or `INTERNAL`, and an `AnalyzePlan` failing with
`INVALID_ARGUMENT`, raise the same exception class and status code as in
official PySpark 4.3.0, with the operation id of the request. Before this
change, all of them differed. A `DEADLINE_EXCEEDED` of an execution's stream
differs: PySpark 4.3.0 reattaches it under its per-RPC deadlines, which [...]
- New offline tests check these cases. They fail without this change.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (648 passed, and 123 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (726 passed, 9 skipped),
and the live drop-in and Arrow suites (76 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #197 from HyukjinKwon/SPARK-60961-grpc-exception.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/errors.rs | 34 +++++++++++++++++++++++++++-------
python/tests/_mock_connect_server.py | 10 ++++++++--
python/tests/test_dropin_offline.py | 28 ++++++++++++++++++++++++++++
3 files changed, 63 insertions(+), 9 deletions(-)
diff --git a/crates/pyspark-rs/src/errors.rs b/crates/pyspark-rs/src/errors.rs
index 4ea7a0c..d82bc46 100644
--- a/crates/pyspark-rs/src/errors.rs
+++ b/crates/pyspark-rs/src/errors.rs
@@ -126,6 +126,23 @@ fn server_error_to_py<'py>(
)
}
+/// The exception for a gRPC error that carries no error of the server, such
as an
+/// `UNAVAILABLE` of an unreachable server or a `PERMISSION_DENIED` of a
proxy, as the
+/// reference client's `_handle_rpc_error` raises it: a
`SparkConnectGrpcException` of the
+/// status code.
+fn grpc_error_to_py<'py>(py: Python<'py>, message: &str, code: i32) ->
PyResult<Bound<'py, PyAny>> {
+ let connect = py.import("pyspark.errors.exceptions.connect")?;
+ let kwargs = PyDict::new(py);
+ kwargs.set_item("message", message)?;
+ kwargs.set_item(
+ "grpc_status_code",
+ connect.call_method1("_status_code", (code,))?,
+ )?;
+ connect
+ .getattr("SparkConnectGrpcException")?
+ .call((), Some(&kwargs))
+}
+
/// Maps a SparkError to a Python exception, attempting to raise the correct
typed
/// exception based on the error kind. Falls back to PyRuntimeError if import
fails.
pub fn spark_error_to_py_exception(err: SparkError) -> PyErr {
@@ -135,14 +152,17 @@ pub fn spark_error_to_py_exception(err: SparkError) ->
PyErr {
// Try to acquire the GIL and import the exception class from
pyspark.errors
Python::attach(|py| {
- if let Some(server) = &err.server {
- // Should the connect classes not import, fall back to the class
by kind below.
- if let Ok(exc_instance) = server_error_to_py(py, &err, server) {
- if let Some(op) = &operation_id {
- let _ = exc_instance.setattr("_operation_id", op);
- }
- return PyErr::from_value(exc_instance);
+ // Should the connect classes not import, fall back to the class by
kind below.
+ let connect_error = match (&err.server, err.grpc_code) {
+ (Some(server), _) => server_error_to_py(py, &err, server).ok(),
+ (None, Some(code)) => grpc_error_to_py(py, &msg, code).ok(),
+ (None, None) => None,
+ };
+ if let Some(exc_instance) = connect_error {
+ if let Some(op) = &operation_id {
+ let _ = exc_instance.setattr("_operation_id", op);
}
+ return PyErr::from_value(exc_instance);
}
// First, verify and get the exception class from pyspark.errors
let errors_module = match py.import("pyspark.errors") {
diff --git a/python/tests/_mock_connect_server.py
b/python/tests/_mock_connect_server.py
index b98ba0e..46278a0 100644
--- a/python/tests/_mock_connect_server.py
+++ b/python/tests/_mock_connect_server.py
@@ -147,7 +147,8 @@ def serve(connection):
# ExecutePlan records each request and streams one response carrying
# `execute_payload` (raw ExecutePlanResponse fields), then
ResultComplete; with
- # `execute_failure` set it aborts instead. Responses echo the request's
+ # `execute_failure` set it aborts instead, with INVALID_ARGUMENT and
that message, or
+ # with the status code name and message of a tuple. Responses echo the
request's
# operation id, as the server does.
execute_requests = []
execute_payload = b""
@@ -173,7 +174,12 @@ def serve(connection):
"Python thread could not run while the statement ran",
)
if self.execute_failure:
- context.abort(grpc.StatusCode.INVALID_ARGUMENT,
self.execute_failure)
+ code, message = (
+ self.execute_failure
+ if isinstance(self.execute_failure, tuple)
+ else ("INVALID_ARGUMENT", self.execute_failure)
+ )
+ context.abort(getattr(grpc.StatusCode, code), message)
if self.execute_error:
message, reason, metadata = self.execute_error
# google.rpc: Status.code=1, message=2, details=3;
Any.type_url=1, value=2;
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index eaf0602..a8dd237 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -5580,6 +5580,34 @@ def
test_execute_plan_error_carries_operation_id(mock_connect):
assert error.value.operation_id ==
_request_operation_id(service.execute_requests[-1])
[email protected]("code", [
+ "INVALID_ARGUMENT", "PERMISSION_DENIED", "UNAUTHENTICATED",
"RESOURCE_EXHAUSTED", "INTERNAL",
+])
+def
test_grpc_errors_without_details_are_spark_connect_grpc_exceptions(mock_connect,
code):
+ # A gRPC error that carries no error of the server, as of a proxy, is a
+ # SparkConnectGrpcException of its status code, as PySpark's
_handle_rpc_error raises it.
+ # The cases agree with official PySpark 4.3.0.
+ from pyspark.errors.exceptions.connect import SparkConnectGrpcException,
StatusCode
+
+ spark, service = mock_connect
+ service.execute_failure = (code, "denied by the proxy")
+ with pytest.raises(SparkConnectGrpcException, match="denied by the proxy")
as error:
+ spark.range(1).collect()
+ assert error.value.getGrpcStatusCode() == getattr(StatusCode, code)
+ assert error.value.getCondition() is None
+ assert error.value.operation_id ==
_request_operation_id(service.execute_requests[-1])
+
+
+def
test_analyze_errors_without_details_are_spark_connect_grpc_exceptions(mock_connect):
+ from pyspark.errors.exceptions.connect import SparkConnectGrpcException,
StatusCode
+
+ spark, service = mock_connect
+ service.failure = "rpc" # INVALID_ARGUMENT "schema unavailable", of
AnalyzePlan.
+ with pytest.raises(SparkConnectGrpcException, match="schema unavailable")
as error:
+ spark.range(1).schema
+ assert error.value.getGrpcStatusCode() == StatusCode.INVALID_ARGUMENT
+
+
def _error_info(classes, **metadata):
"""ErrorInfo metadata as the server sends it, with the exception classes,
the most
specific first."""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]