This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 3945555 [SPARK-61090][CONNECT] Harden reattach cleanup regression
tests
3945555 is described below
commit 3945555b92f853f77b20011aca79b565e77d7664
Author: DB Tsai <[email protected]>
AuthorDate: Sat Nov 7 18:15:05 2026 +0900
[SPARK-61090][CONNECT] Harden reattach cleanup regression tests
### What changes were proposed in this pull request?
Harden the loopback gRPC cleanup tests from #172. Capture initial execution
and reattach requests and verify their identity, request options,
authentication and custom headers. Add disconnected-cursor and permanent
reattach-failure coverage, give scenarios independent test names, bound the
outside-runtime Drop wait, and report excess release counts directly.
### Why are the changes needed?
Tracks [SPARK-61090](https://issues.apache.org/jira/browse/SPARK-61090).
These assertions cover previously unobserved request headers and cleanup
failure paths. The fixture also reports a stuck Drop or an excess release count
more clearly.
### Does this PR introduce _any_ user-facing change?
No. The patch changes one integration-test file.
### How was this patch tested?
- `git diff --check`: passed.
- Genuine local Claude Code reviewed the complete follow-up over four
rounds. Codex fixed worthwhile findings; the final whole-patch review reports
no actionable source finding. Three final style suggestions are accepted
nonblocking nits.
- Focused `reattach_cleanup` tests and `cargo fmt --all --check` were
attempted through the campaign Cargo lock. Both failed before Cargo launch
because this host has no Cargo/Rust toolchain.
- Required compilation, formatting, tests, Clippy and Python
parity/official-test checks remain pending. This PR stays draft; no executed
test pass or review-and-validation convergence is claimed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6), with Claude Code 2.1.314 for review.
This pull request and its description were written by Isaac.
Closes #284 from dbtsai/codex/postmerge-pr-172-cleanup-test-hardening.
Authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../spark-connect-core/tests/reattach_cleanup.rs | 270 +++++++++++++++------
1 file changed, 190 insertions(+), 80 deletions(-)
diff --git a/crates/spark-connect-core/tests/reattach_cleanup.rs
b/crates/spark-connect-core/tests/reattach_cleanup.rs
index e68fe47..9199853 100644
--- a/crates/spark-connect-core/tests/reattach_cleanup.rs
+++ b/crates/spark-connect-core/tests/reattach_cleanup.rs
@@ -35,7 +35,7 @@ use spark_connect_proto::{
ReleaseExecuteResponse, UserContext,
};
use tokio::net::TcpListener;
-use tokio::sync::Notify;
+use tokio::sync::{oneshot, Notify};
use tonic::body::Body;
use tonic::codegen::{http, BoxFuture, Service};
use tonic::metadata::MetadataMap;
@@ -59,6 +59,8 @@ enum Scenario {
Error,
Eof,
Disconnect,
+ /// A retryable INTERNAL status carrying Spark's cursor-disconnect marker.
+ CursorDisconnect,
/// The ExecutePlan call fails itself, with a trailers-only response, as
for an
/// analysis error.
Reject,
@@ -70,10 +72,23 @@ struct CapturedRelease {
metadata: MetadataMap,
}
+#[derive(Clone)]
+struct CapturedExecute {
+ request: ExecutePlanRequest,
+ metadata: MetadataMap,
+}
+
+#[derive(Clone)]
+struct CapturedReattach {
+ request: ReattachExecuteRequest,
+ metadata: MetadataMap,
+}
+
struct State {
scenario: Scenario,
+ executions: Mutex<Vec<CapturedExecute>>,
releases: Mutex<Vec<CapturedRelease>>,
- reattaches: Mutex<Vec<ReattachExecuteRequest>>,
+ reattaches: Mutex<Vec<CapturedReattach>>,
/// What each ReattachExecute call does, in order: fail with a code, or
stream responses.
/// When none is left, a call completes the result.
reattach_script: Mutex<VecDeque<Result<Vec<ExecutePlanResponse>, Code>>>,
@@ -117,6 +132,7 @@ impl Fixture {
async fn start(scenario: Scenario) -> Self {
let state = Arc::new(State {
scenario,
+ executions: Mutex::new(Vec::new()),
releases: Mutex::new(Vec::new()),
reattaches: Mutex::new(Vec::new()),
reattach_script: Mutex::new(VecDeque::new()),
@@ -177,7 +193,8 @@ impl Fixture {
}
async fn assert_release_all_count(&self, expected: usize) {
- self.wait_for(|s| s.all_releases().len() == expected).await;
+ self.wait_for(|s| s.all_releases().len() >= expected).await;
+ assert_eq!(self.state.all_releases().len(), expected);
assert!(
tokio::time::timeout(
Duration::from_millis(100),
@@ -226,13 +243,17 @@ fn assert_identity(captured: &CapturedRelease) {
original.operation_id
);
assert_eq!(captured.request.client_type, original.client_type);
+ assert_metadata(&captured.metadata);
+}
+
+fn assert_metadata(metadata: &MetadataMap) {
assert_eq!(
- captured.metadata.get("authorization").unwrap(),
+ metadata.get("authorization").unwrap(),
"Bearer fixture-token"
);
- assert_eq!(captured.metadata.get("user_id").unwrap(), "transport-user");
- assert_eq!(captured.metadata.get("x-cleanup").unwrap(), "fixture-value");
- assert!(captured.metadata.get("user-agent").is_some());
+ assert_eq!(metadata.get("user_id").unwrap(), "transport-user");
+ assert_eq!(metadata.get("x-cleanup").unwrap(), "fixture-value");
+ assert!(metadata.get("user-agent").is_some());
}
#[derive(Clone)]
@@ -287,6 +308,11 @@ impl ServerStreamingService<ExecutePlanRequest> for
Execute {
fn call(&mut self, request: Request<ExecutePlanRequest>) -> Self::Future {
self.0.record_timeout("ExecutePlan", request.metadata());
+ let metadata = request.metadata().clone();
+ self.0.executions.lock().unwrap().push(CapturedExecute {
+ request: request.into_inner(),
+ metadata,
+ });
if let Scenario::Reject = self.0.scenario {
return Box::pin(async { Err(Status::invalid_argument("plan
sentinel")) });
}
@@ -303,6 +329,10 @@ impl ServerStreamingService<ExecutePlanRequest> for
Execute {
Ok(response("first", false)),
Err(Status::unavailable("stream disconnected")),
])),
+ Scenario::CursorDisconnect => Box::pin(stream::iter([
+ Ok(response("first", false)),
+ Err(Status::internal("INVALID_CURSOR.DISCONNECTED")),
+ ])),
Scenario::Reject => unreachable!("rejected before streaming"),
};
Box::pin(async { Ok(Response::new(responses)) })
@@ -319,7 +349,11 @@ impl ServerStreamingService<ReattachExecuteRequest> for
Reattach {
fn call(&mut self, request: Request<ReattachExecuteRequest>) ->
Self::Future {
let state = self.0.clone();
state.record_timeout("ReattachExecute", request.metadata());
- state.reattaches.lock().unwrap().push(request.into_inner());
+ let metadata = request.metadata().clone();
+ state.reattaches.lock().unwrap().push(CapturedReattach {
+ request: request.into_inner(),
+ metadata,
+ });
state.changed.notify_waiters();
Box::pin(async move {
if state.block_reattach.load(Ordering::SeqCst) {
@@ -387,6 +421,19 @@ async fn
completion_releases_once_without_waiting_for_release_reply() {
result.response_type,
Some(ResponseType::ResultComplete(_))
));
+ let captured = fixture.state.executions.lock().unwrap()[0].clone();
+ assert_metadata(&captured.metadata);
+ let executed = captured.request;
+ let original = execute_request();
+ assert_eq!(executed.session_id, original.session_id);
+ assert_eq!(executed.user_context, original.user_context);
+ assert_eq!(executed.operation_id, original.operation_id);
+ assert_eq!(executed.client_type, original.client_type);
+ assert!(executed.request_options.iter().any(|option| matches!(
+ &option.request_option,
+
Some(spark_connect_proto::execute_plan_request::request_option::RequestOption::ReattachOptions(options))
+ if options.reattachable
+ )));
// Observe terminal cleanup while the stream is alive: Drop must not mask
it.
fixture.wait_for(|s| !s.all_releases().is_empty()).await;
assert_eq!(fixture.state.release_finished.load(Ordering::SeqCst), 0);
@@ -401,35 +448,50 @@ async fn
completion_releases_once_without_waiting_for_release_reply() {
}
#[tokio::test]
-async fn dropping_unfinished_stream_releases_inside_or_outside_runtime() {
- for outside_runtime in [false, true] {
- let fixture = Fixture::start(Scenario::Open).await;
- let mut stream =
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+async fn dropping_unfinished_stream_releases_inside_runtime() {
+ assert_unfinished_drop(false).await;
+}
+
+#[tokio::test]
+async fn dropping_unfinished_stream_releases_outside_runtime() {
+ assert_unfinished_drop(true).await;
+}
+
+async fn assert_unfinished_drop(outside_runtime: bool) {
+ let fixture = Fixture::start(Scenario::Open).await;
+ let mut stream =
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+ .await
+ .unwrap();
+ assert_eq!(
+ bounded(stream.message())
.await
- .unwrap();
- assert_eq!(
- bounded(stream.message())
- .await
- .unwrap()
- .unwrap()
- .response_id,
- "first"
- );
- fixture
- .wait_for(|s| !s.releases.lock().unwrap().is_empty())
- .await;
- let until = fixture.state.releases.lock().unwrap()[0].clone();
- assert_identity(&until);
- assert!(matches!(until.request.release,
- Some(Release::ReleaseUntil(ref until)) if until.response_id ==
"first"));
- if outside_runtime {
- std::thread::spawn(move || drop(stream)).join().unwrap();
- } else {
+ .unwrap()
+ .unwrap()
+ .response_id,
+ "first"
+ );
+ fixture
+ .wait_for(|s| !s.releases.lock().unwrap().is_empty())
+ .await;
+ let until = fixture.state.releases.lock().unwrap()[0].clone();
+ assert_identity(&until);
+ assert!(matches!(until.request.release,
+ Some(Release::ReleaseUntil(ref until)) if until.response_id ==
"first"));
+ if outside_runtime {
+ let (finished, completion) = oneshot::channel();
+ drop(std::thread::spawn(move || {
drop(stream);
- }
- fixture.assert_release_all_count(1).await;
- assert_identity(&fixture.state.all_releases()[0]);
+ let _ = finished.send(());
+ }));
+ tokio::time::timeout(WAIT, completion)
+ .await
+ .expect("drop did not return")
+ .expect("drop thread panicked");
+ } else {
+ drop(stream);
}
+ fixture.assert_release_all_count(1).await;
+ assert_identity(&fixture.state.all_releases()[0]);
}
#[tokio::test]
@@ -518,53 +580,101 @@ async fn
permanent_release_failure_does_not_retry_or_change_success() {
}
#[tokio::test]
-async fn detached_stream_reattaches_before_releasing_the_execution() {
- for scenario in [Scenario::Eof, Scenario::Disconnect] {
- let fixture = Fixture::start(scenario).await;
- fixture.state.block_reattach.store(true, Ordering::SeqCst);
- let mut stream =
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+async fn eof_reattaches_before_releasing_the_execution() {
+ assert_detached_reattachment(Scenario::Eof).await;
+}
+
+#[tokio::test]
+async fn unavailable_reattaches_before_releasing_the_execution() {
+ assert_detached_reattachment(Scenario::Disconnect).await;
+}
+
+#[tokio::test]
+async fn disconnected_cursor_reattaches_before_releasing_the_execution() {
+ assert_detached_reattachment(Scenario::CursorDisconnect).await;
+}
+
+async fn assert_detached_reattachment(scenario: Scenario) {
+ let fixture = Fixture::start(scenario).await;
+ fixture.state.block_reattach.store(true, Ordering::SeqCst);
+ let mut stream =
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+ .await
+ .unwrap();
+ assert_eq!(
+ bounded(stream.message())
.await
- .unwrap();
- assert_eq!(
- bounded(stream.message())
- .await
- .unwrap()
- .unwrap()
- .response_id,
- "first"
- );
- let next = tokio::spawn(async move {
- let result = bounded(stream.message()).await;
- (stream, result)
- });
- fixture
- .wait_for(|s| !s.reattaches.lock().unwrap().is_empty())
- .await;
- let reattach = fixture.state.reattaches.lock().unwrap()[0].clone();
- assert_eq!(
- reattach.operation_id,
- execute_request().operation_id.unwrap()
- );
- assert_eq!(reattach.last_response_id.as_deref(), Some("first"));
- assert!(
- tokio::time::timeout(
- Duration::from_millis(100),
- fixture.wait_for(|s| !s.all_releases().is_empty())
- )
+ .unwrap()
+ .unwrap()
+ .response_id,
+ "first"
+ );
+ let next = tokio::spawn(async move {
+ let result = bounded(stream.message()).await;
+ (stream, result)
+ });
+ fixture
+ .wait_for(|s| !s.reattaches.lock().unwrap().is_empty())
+ .await;
+ let captured = fixture.state.reattaches.lock().unwrap()[0].clone();
+ let reattach = captured.request;
+ let original = execute_request();
+ assert_eq!(reattach.session_id, original.session_id);
+ assert_eq!(reattach.user_context, original.user_context);
+ assert_eq!(reattach.client_type, original.client_type);
+ assert_metadata(&captured.metadata);
+ assert_eq!(reattach.operation_id, original.operation_id.unwrap());
+ assert_eq!(reattach.last_response_id.as_deref(), Some("first"));
+ assert!(
+ tokio::time::timeout(
+ Duration::from_millis(100),
+ fixture.wait_for(|s| !s.all_releases().is_empty())
+ )
+ .await
+ .is_err(),
+ "released before reattachment completed"
+ );
+ fixture.state.reattach_gate.notify_one();
+ let (stream, result) = tokio::time::timeout(WAIT,
next).await.unwrap().unwrap();
+ assert!(matches!(
+ result.unwrap().unwrap().response_type,
+ Some(ResponseType::ResultComplete(_))
+ ));
+ fixture.wait_for(|s| !s.all_releases().is_empty()).await;
+ drop(stream);
+ fixture.assert_release_all_count(1).await;
+}
+
+#[tokio::test]
+async fn permanent_reattach_failure_releases_and_preserves_the_error() {
+ let fixture = Fixture::start(Scenario::Eof).await;
+ fixture
+ .state
+ .reattach_script
+ .lock()
+ .unwrap()
+ .push_back(Err(Code::PermissionDenied));
+ let mut stream =
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+ .await
+ .unwrap();
+ assert_eq!(
+ bounded(stream.message())
.await
- .is_err(),
- "released before reattachment completed"
- );
- fixture.state.reattach_gate.notify_one();
- let (stream, result) = tokio::time::timeout(WAIT,
next).await.unwrap().unwrap();
- assert!(matches!(
- result.unwrap().unwrap().response_type,
- Some(ResponseType::ResultComplete(_))
- ));
- fixture.wait_for(|s| !s.all_releases().is_empty()).await;
- drop(stream);
- fixture.assert_release_all_count(1).await;
- }
+ .unwrap()
+ .unwrap()
+ .response_id,
+ "first"
+ );
+ let error = bounded(stream.message()).await.unwrap_err();
+ assert!(error.message.contains("reattach sentinel"), "{error:?}");
+ assert_eq!(error.grpc_code, Some(Code::PermissionDenied as i32));
+ assert_eq!(error.operation_id, execute_request().operation_id);
+ // Observe failure cleanup before Drop can send a fallback release.
+ fixture.wait_for(|s| !s.all_releases().is_empty()).await;
+ assert_identity(&fixture.state.all_releases()[0]);
+ assert!(bounded(stream.message()).await.unwrap().is_none());
+ drop(stream);
+ fixture.assert_release_all_count(1).await;
+ assert_eq!(fixture.state.reattaches.lock().unwrap().len(), 1);
}
#[tokio::test]
@@ -644,7 +754,7 @@ async fn
execution_streams_carry_their_deadlines_and_reattach_past_them() {
let reattaches = fixture.state.reattaches.lock().unwrap().clone();
let resumed: Vec<_> = reattaches
.iter()
- .map(|r| r.last_response_id.clone())
+ .map(|r| r.request.last_response_id.clone())
.collect();
assert_eq!(
resumed,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]