This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 3d9602c587b4 [SPARK-58846][CONNECT][PYTHON][FOLLOWUP] Preserve
RpcDeadlines positional arguments
3d9602c587b4 is described below
commit 3d9602c587b4b579c64937ef83803402ed186040
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Aug 28 13:36:25 2026 +0800
[SPARK-58846][CONNECT][PYTHON][FOLLOWUP] Preserve RpcDeadlines positional
arguments
### What changes were proposed in this pull request?
This is a follow-up to #58085.
This PR appends `RpcDeadlines.release_relation` after all existing
dataclass fields instead of
inserting it between `release_session` and `artifact_status`. It also
clarifies that ordinary
non-reattachable query `ExecutePlan` calls have no deadline, while relation
cleanup is the bounded
exception.
The deadline tests now cover the new default and disabled values and verify
that the existing
positional constructor arguments retain their original bindings.
### Why are the changes needed?
The generated dataclass constructor accepts positional arguments. Inserting
`release_relation`
before existing fields silently shifted the bindings for `artifact_status`,
`clone_session`,
`get_status`, and `fetch_error_details`. Appending the field preserves
those bindings.
The documentation also made a categorical statement that non-reattachable
`ExecutePlan` calls have
no deadline immediately before documenting the relation-cleanup exception.
Qualifying the statement
removes that contradiction.
### Does this PR introduce _any_ user-facing change?
Yes, within the unreleased `master` branch only. Existing positional
`RpcDeadlines` constructor
arguments keep their original meanings. The release RPC deadline behavior
introduced by #58085 is
unchanged. No released Spark version is affected.
### How was this patch tested?
Built Spark with Hive support and ran:
`conda run -n spark-dev-313 python/run-tests --testnames
pyspark.sql.tests.connect.client.test_client_retries`
The test suite passed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #58326 from zhengruifeng/py-connect-rpc-deadlines-followup-dev-7.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 463b400cb4d2b06ac0cfa2e889a6f5ffff9d1703)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/sql/connect/client/core.py | 8 ++++----
.../pyspark/sql/tests/connect/client/test_client_retries.py | 11 +++++++++++
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/sql/connect/client/core.py
b/python/pyspark/sql/connect/client/core.py
index 7685571c7253..e47bc3c1ae10 100644
--- a/python/pyspark/sql/connect/client/core.py
+++ b/python/pyspark/sql/connect/client/core.py
@@ -141,8 +141,8 @@ class RpcDeadlines:
Note on ``reattachable_execute_plan`` and ``reattach_execute``: these
timeouts apply to each
individual gRPC stream segment, not to the overall query execution
lifetime. When a deadline
fires, the server-side operation continues running; the client opens a new
ReattachExecute
- stream to resume receiving results. Non-reattachable ExecutePlan has no
deadline because a
- timeout there would kill the execution with no recovery path.
+ stream to resume receiving results. Non-reattachable query ExecutePlan
calls have no deadline
+ because a timeout there would kill the execution with no recovery path.
Note on ``release_relation``: the RemoveRemoteCachedRelation cleanup
command is sent over a
blocking, non-reattachable ExecutePlan call issued from
@@ -160,11 +160,11 @@ class RpcDeadlines:
config: Optional[float] = 10 * 60 # 10 min
interrupt: Optional[float] = 10 * 60 # 10 min
release_session: Optional[float] = 10 * 60 # 10 min
- release_relation: Optional[float] = 60 # 1 min; short: per-batch finalizer
artifact_status: Optional[float] = 10 * 60 # 10 min
clone_session: Optional[float] = 10 * 60 # 10 min
get_status: Optional[float] = 10 * 60 # 10 min
fetch_error_details: Optional[float] = 10 * 60 # 10 min
+ release_relation: Optional[float] = 60 # 1 min; short: per-batch finalizer
def __post_init__(self) -> None:
for field in fields(self):
@@ -191,11 +191,11 @@ class RpcDeadlines:
config=None,
interrupt=None,
release_session=None,
- release_relation=None,
artifact_status=None,
clone_session=None,
get_status=None,
fetch_error_details=None,
+ release_relation=None,
)
diff --git a/python/pyspark/sql/tests/connect/client/test_client_retries.py
b/python/pyspark/sql/tests/connect/client/test_client_retries.py
index f6d352ad2728..6a7a08ffe254 100644
--- a/python/pyspark/sql/tests/connect/client/test_client_retries.py
+++ b/python/pyspark/sql/tests/connect/client/test_client_retries.py
@@ -401,6 +401,7 @@ class SparkConnectClientRetriesTestCase(unittest.TestCase):
self.assertIsNone(d.clone_session)
self.assertIsNone(d.get_status)
self.assertIsNone(d.fetch_error_details)
+ self.assertIsNone(d.release_relation)
def test_rpc_deadlines_defaults_are_set(self):
"""RpcDeadlines() default instance should have documented timeout
values."""
@@ -416,6 +417,16 @@ class SparkConnectClientRetriesTestCase(unittest.TestCase):
self.assertEqual(d.clone_session, 10 * 60)
self.assertEqual(d.get_status, 10 * 60)
self.assertEqual(d.fetch_error_details, 10 * 60)
+ self.assertEqual(d.release_relation, 60)
+
+ def test_rpc_deadlines_preserves_existing_positional_arguments(self):
+ """New deadline fields should not shift existing positional
arguments."""
+ d = RpcDeadlines(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)
+ self.assertEqual(d.artifact_status, 8)
+ self.assertEqual(d.clone_session, 9)
+ self.assertEqual(d.get_status, 10)
+ self.assertEqual(d.fetch_error_details, 11)
+ self.assertEqual(d.release_relation, 60)
def test_rpc_deadlines_rejects_non_positive_values(self):
"""RpcDeadlines should raise ValueError for any non-None field that is
<= 0."""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
