This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 3945555  [SPARK-61090][CONNECT] Harden reattach cleanup regression 
tests
3945555 is described below

commit 3945555b92f853f77b20011aca79b565e77d7664
Author: DB Tsai <[email protected]>
AuthorDate: Sat Nov 7 18:15:05 2026 +0900

    [SPARK-61090][CONNECT] Harden reattach cleanup regression tests
    
    ### What changes were proposed in this pull request?
    
    Harden the loopback gRPC cleanup tests from #172. Capture initial execution 
and reattach requests and verify their identity, request options, 
authentication and custom headers. Add disconnected-cursor and permanent 
reattach-failure coverage, give scenarios independent test names, bound the 
outside-runtime Drop wait, and report excess release counts directly.
    
    ### Why are the changes needed?
    
    Tracks [SPARK-61090](https://issues.apache.org/jira/browse/SPARK-61090).
    
    These assertions cover previously unobserved request headers and cleanup 
failure paths. The fixture also reports a stuck Drop or an excess release count 
more clearly.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. The patch changes one integration-test file.
    
    ### How was this patch tested?
    
    - `git diff --check`: passed.
    - Genuine local Claude Code reviewed the complete follow-up over four 
rounds. Codex fixed worthwhile findings; the final whole-patch review reports 
no actionable source finding. Three final style suggestions are accepted 
nonblocking nits.
    - Focused `reattach_cleanup` tests and `cargo fmt --all --check` were 
attempted through the campaign Cargo lock. Both failed before Cargo launch 
because this host has no Cargo/Rust toolchain.
    - Required compilation, formatting, tests, Clippy and Python 
parity/official-test checks remain pending. This PR stays draft; no executed 
test pass or review-and-validation convergence is claimed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6), with Claude Code 2.1.314 for review.
    
    This pull request and its description were written by Isaac.
    
    Closes #284 from dbtsai/codex/postmerge-pr-172-cleanup-test-hardening.
    
    Authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 .../spark-connect-core/tests/reattach_cleanup.rs   | 270 +++++++++++++++------
 1 file changed, 190 insertions(+), 80 deletions(-)

diff --git a/crates/spark-connect-core/tests/reattach_cleanup.rs 
b/crates/spark-connect-core/tests/reattach_cleanup.rs
index e68fe47..9199853 100644
--- a/crates/spark-connect-core/tests/reattach_cleanup.rs
+++ b/crates/spark-connect-core/tests/reattach_cleanup.rs
@@ -35,7 +35,7 @@ use spark_connect_proto::{
     ReleaseExecuteResponse, UserContext,
 };
 use tokio::net::TcpListener;
-use tokio::sync::Notify;
+use tokio::sync::{oneshot, Notify};
 use tonic::body::Body;
 use tonic::codegen::{http, BoxFuture, Service};
 use tonic::metadata::MetadataMap;
@@ -59,6 +59,8 @@ enum Scenario {
     Error,
     Eof,
     Disconnect,
+    /// A retryable INTERNAL status carrying Spark's cursor-disconnect marker.
+    CursorDisconnect,
     /// The ExecutePlan call fails itself, with a trailers-only response, as 
for an
     /// analysis error.
     Reject,
@@ -70,10 +72,23 @@ struct CapturedRelease {
     metadata: MetadataMap,
 }
 
+#[derive(Clone)]
+struct CapturedExecute {
+    request: ExecutePlanRequest,
+    metadata: MetadataMap,
+}
+
+#[derive(Clone)]
+struct CapturedReattach {
+    request: ReattachExecuteRequest,
+    metadata: MetadataMap,
+}
+
 struct State {
     scenario: Scenario,
+    executions: Mutex<Vec<CapturedExecute>>,
     releases: Mutex<Vec<CapturedRelease>>,
-    reattaches: Mutex<Vec<ReattachExecuteRequest>>,
+    reattaches: Mutex<Vec<CapturedReattach>>,
     /// What each ReattachExecute call does, in order: fail with a code, or 
stream responses.
     /// When none is left, a call completes the result.
     reattach_script: Mutex<VecDeque<Result<Vec<ExecutePlanResponse>, Code>>>,
@@ -117,6 +132,7 @@ impl Fixture {
     async fn start(scenario: Scenario) -> Self {
         let state = Arc::new(State {
             scenario,
+            executions: Mutex::new(Vec::new()),
             releases: Mutex::new(Vec::new()),
             reattaches: Mutex::new(Vec::new()),
             reattach_script: Mutex::new(VecDeque::new()),
@@ -177,7 +193,8 @@ impl Fixture {
     }
 
     async fn assert_release_all_count(&self, expected: usize) {
-        self.wait_for(|s| s.all_releases().len() == expected).await;
+        self.wait_for(|s| s.all_releases().len() >= expected).await;
+        assert_eq!(self.state.all_releases().len(), expected);
         assert!(
             tokio::time::timeout(
                 Duration::from_millis(100),
@@ -226,13 +243,17 @@ fn assert_identity(captured: &CapturedRelease) {
         original.operation_id
     );
     assert_eq!(captured.request.client_type, original.client_type);
+    assert_metadata(&captured.metadata);
+}
+
+fn assert_metadata(metadata: &MetadataMap) {
     assert_eq!(
-        captured.metadata.get("authorization").unwrap(),
+        metadata.get("authorization").unwrap(),
         "Bearer fixture-token"
     );
-    assert_eq!(captured.metadata.get("user_id").unwrap(), "transport-user");
-    assert_eq!(captured.metadata.get("x-cleanup").unwrap(), "fixture-value");
-    assert!(captured.metadata.get("user-agent").is_some());
+    assert_eq!(metadata.get("user_id").unwrap(), "transport-user");
+    assert_eq!(metadata.get("x-cleanup").unwrap(), "fixture-value");
+    assert!(metadata.get("user-agent").is_some());
 }
 
 #[derive(Clone)]
@@ -287,6 +308,11 @@ impl ServerStreamingService<ExecutePlanRequest> for 
Execute {
 
     fn call(&mut self, request: Request<ExecutePlanRequest>) -> Self::Future {
         self.0.record_timeout("ExecutePlan", request.metadata());
+        let metadata = request.metadata().clone();
+        self.0.executions.lock().unwrap().push(CapturedExecute {
+            request: request.into_inner(),
+            metadata,
+        });
         if let Scenario::Reject = self.0.scenario {
             return Box::pin(async { Err(Status::invalid_argument("plan 
sentinel")) });
         }
@@ -303,6 +329,10 @@ impl ServerStreamingService<ExecutePlanRequest> for 
Execute {
                 Ok(response("first", false)),
                 Err(Status::unavailable("stream disconnected")),
             ])),
+            Scenario::CursorDisconnect => Box::pin(stream::iter([
+                Ok(response("first", false)),
+                Err(Status::internal("INVALID_CURSOR.DISCONNECTED")),
+            ])),
             Scenario::Reject => unreachable!("rejected before streaming"),
         };
         Box::pin(async { Ok(Response::new(responses)) })
@@ -319,7 +349,11 @@ impl ServerStreamingService<ReattachExecuteRequest> for 
Reattach {
     fn call(&mut self, request: Request<ReattachExecuteRequest>) -> 
Self::Future {
         let state = self.0.clone();
         state.record_timeout("ReattachExecute", request.metadata());
-        state.reattaches.lock().unwrap().push(request.into_inner());
+        let metadata = request.metadata().clone();
+        state.reattaches.lock().unwrap().push(CapturedReattach {
+            request: request.into_inner(),
+            metadata,
+        });
         state.changed.notify_waiters();
         Box::pin(async move {
             if state.block_reattach.load(Ordering::SeqCst) {
@@ -387,6 +421,19 @@ async fn 
completion_releases_once_without_waiting_for_release_reply() {
         result.response_type,
         Some(ResponseType::ResultComplete(_))
     ));
+    let captured = fixture.state.executions.lock().unwrap()[0].clone();
+    assert_metadata(&captured.metadata);
+    let executed = captured.request;
+    let original = execute_request();
+    assert_eq!(executed.session_id, original.session_id);
+    assert_eq!(executed.user_context, original.user_context);
+    assert_eq!(executed.operation_id, original.operation_id);
+    assert_eq!(executed.client_type, original.client_type);
+    assert!(executed.request_options.iter().any(|option| matches!(
+        &option.request_option,
+        
Some(spark_connect_proto::execute_plan_request::request_option::RequestOption::ReattachOptions(options))
+            if options.reattachable
+    )));
     // Observe terminal cleanup while the stream is alive: Drop must not mask 
it.
     fixture.wait_for(|s| !s.all_releases().is_empty()).await;
     assert_eq!(fixture.state.release_finished.load(Ordering::SeqCst), 0);
@@ -401,35 +448,50 @@ async fn 
completion_releases_once_without_waiting_for_release_reply() {
 }
 
 #[tokio::test]
-async fn dropping_unfinished_stream_releases_inside_or_outside_runtime() {
-    for outside_runtime in [false, true] {
-        let fixture = Fixture::start(Scenario::Open).await;
-        let mut stream = 
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+async fn dropping_unfinished_stream_releases_inside_runtime() {
+    assert_unfinished_drop(false).await;
+}
+
+#[tokio::test]
+async fn dropping_unfinished_stream_releases_outside_runtime() {
+    assert_unfinished_drop(true).await;
+}
+
+async fn assert_unfinished_drop(outside_runtime: bool) {
+    let fixture = Fixture::start(Scenario::Open).await;
+    let mut stream = 
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+        .await
+        .unwrap();
+    assert_eq!(
+        bounded(stream.message())
             .await
-            .unwrap();
-        assert_eq!(
-            bounded(stream.message())
-                .await
-                .unwrap()
-                .unwrap()
-                .response_id,
-            "first"
-        );
-        fixture
-            .wait_for(|s| !s.releases.lock().unwrap().is_empty())
-            .await;
-        let until = fixture.state.releases.lock().unwrap()[0].clone();
-        assert_identity(&until);
-        assert!(matches!(until.request.release,
-            Some(Release::ReleaseUntil(ref until)) if until.response_id == 
"first"));
-        if outside_runtime {
-            std::thread::spawn(move || drop(stream)).join().unwrap();
-        } else {
+            .unwrap()
+            .unwrap()
+            .response_id,
+        "first"
+    );
+    fixture
+        .wait_for(|s| !s.releases.lock().unwrap().is_empty())
+        .await;
+    let until = fixture.state.releases.lock().unwrap()[0].clone();
+    assert_identity(&until);
+    assert!(matches!(until.request.release,
+        Some(Release::ReleaseUntil(ref until)) if until.response_id == 
"first"));
+    if outside_runtime {
+        let (finished, completion) = oneshot::channel();
+        drop(std::thread::spawn(move || {
             drop(stream);
-        }
-        fixture.assert_release_all_count(1).await;
-        assert_identity(&fixture.state.all_releases()[0]);
+            let _ = finished.send(());
+        }));
+        tokio::time::timeout(WAIT, completion)
+            .await
+            .expect("drop did not return")
+            .expect("drop thread panicked");
+    } else {
+        drop(stream);
     }
+    fixture.assert_release_all_count(1).await;
+    assert_identity(&fixture.state.all_releases()[0]);
 }
 
 #[tokio::test]
@@ -518,53 +580,101 @@ async fn 
permanent_release_failure_does_not_retry_or_change_success() {
 }
 
 #[tokio::test]
-async fn detached_stream_reattaches_before_releasing_the_execution() {
-    for scenario in [Scenario::Eof, Scenario::Disconnect] {
-        let fixture = Fixture::start(scenario).await;
-        fixture.state.block_reattach.store(true, Ordering::SeqCst);
-        let mut stream = 
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+async fn eof_reattaches_before_releasing_the_execution() {
+    assert_detached_reattachment(Scenario::Eof).await;
+}
+
+#[tokio::test]
+async fn unavailable_reattaches_before_releasing_the_execution() {
+    assert_detached_reattachment(Scenario::Disconnect).await;
+}
+
+#[tokio::test]
+async fn disconnected_cursor_reattaches_before_releasing_the_execution() {
+    assert_detached_reattachment(Scenario::CursorDisconnect).await;
+}
+
+async fn assert_detached_reattachment(scenario: Scenario) {
+    let fixture = Fixture::start(scenario).await;
+    fixture.state.block_reattach.store(true, Ordering::SeqCst);
+    let mut stream = 
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+        .await
+        .unwrap();
+    assert_eq!(
+        bounded(stream.message())
             .await
-            .unwrap();
-        assert_eq!(
-            bounded(stream.message())
-                .await
-                .unwrap()
-                .unwrap()
-                .response_id,
-            "first"
-        );
-        let next = tokio::spawn(async move {
-            let result = bounded(stream.message()).await;
-            (stream, result)
-        });
-        fixture
-            .wait_for(|s| !s.reattaches.lock().unwrap().is_empty())
-            .await;
-        let reattach = fixture.state.reattaches.lock().unwrap()[0].clone();
-        assert_eq!(
-            reattach.operation_id,
-            execute_request().operation_id.unwrap()
-        );
-        assert_eq!(reattach.last_response_id.as_deref(), Some("first"));
-        assert!(
-            tokio::time::timeout(
-                Duration::from_millis(100),
-                fixture.wait_for(|s| !s.all_releases().is_empty())
-            )
+            .unwrap()
+            .unwrap()
+            .response_id,
+        "first"
+    );
+    let next = tokio::spawn(async move {
+        let result = bounded(stream.message()).await;
+        (stream, result)
+    });
+    fixture
+        .wait_for(|s| !s.reattaches.lock().unwrap().is_empty())
+        .await;
+    let captured = fixture.state.reattaches.lock().unwrap()[0].clone();
+    let reattach = captured.request;
+    let original = execute_request();
+    assert_eq!(reattach.session_id, original.session_id);
+    assert_eq!(reattach.user_context, original.user_context);
+    assert_eq!(reattach.client_type, original.client_type);
+    assert_metadata(&captured.metadata);
+    assert_eq!(reattach.operation_id, original.operation_id.unwrap());
+    assert_eq!(reattach.last_response_id.as_deref(), Some("first"));
+    assert!(
+        tokio::time::timeout(
+            Duration::from_millis(100),
+            fixture.wait_for(|s| !s.all_releases().is_empty())
+        )
+        .await
+        .is_err(),
+        "released before reattachment completed"
+    );
+    fixture.state.reattach_gate.notify_one();
+    let (stream, result) = tokio::time::timeout(WAIT, 
next).await.unwrap().unwrap();
+    assert!(matches!(
+        result.unwrap().unwrap().response_type,
+        Some(ResponseType::ResultComplete(_))
+    ));
+    fixture.wait_for(|s| !s.all_releases().is_empty()).await;
+    drop(stream);
+    fixture.assert_release_all_count(1).await;
+}
+
+#[tokio::test]
+async fn permanent_reattach_failure_releases_and_preserves_the_error() {
+    let fixture = Fixture::start(Scenario::Eof).await;
+    fixture
+        .state
+        .reattach_script
+        .lock()
+        .unwrap()
+        .push_back(Err(Code::PermissionDenied));
+    let mut stream = 
bounded(fixture.client.execute_plan_reattachable(execute_request()))
+        .await
+        .unwrap();
+    assert_eq!(
+        bounded(stream.message())
             .await
-            .is_err(),
-            "released before reattachment completed"
-        );
-        fixture.state.reattach_gate.notify_one();
-        let (stream, result) = tokio::time::timeout(WAIT, 
next).await.unwrap().unwrap();
-        assert!(matches!(
-            result.unwrap().unwrap().response_type,
-            Some(ResponseType::ResultComplete(_))
-        ));
-        fixture.wait_for(|s| !s.all_releases().is_empty()).await;
-        drop(stream);
-        fixture.assert_release_all_count(1).await;
-    }
+            .unwrap()
+            .unwrap()
+            .response_id,
+        "first"
+    );
+    let error = bounded(stream.message()).await.unwrap_err();
+    assert!(error.message.contains("reattach sentinel"), "{error:?}");
+    assert_eq!(error.grpc_code, Some(Code::PermissionDenied as i32));
+    assert_eq!(error.operation_id, execute_request().operation_id);
+    // Observe failure cleanup before Drop can send a fallback release.
+    fixture.wait_for(|s| !s.all_releases().is_empty()).await;
+    assert_identity(&fixture.state.all_releases()[0]);
+    assert!(bounded(stream.message()).await.unwrap().is_none());
+    drop(stream);
+    fixture.assert_release_all_count(1).await;
+    assert_eq!(fixture.state.reattaches.lock().unwrap().len(), 1);
 }
 
 #[tokio::test]
@@ -644,7 +754,7 @@ async fn 
execution_streams_carry_their_deadlines_and_reattach_past_them() {
     let reattaches = fixture.state.reattaches.lock().unwrap().clone();
     let resumed: Vec<_> = reattaches
         .iter()
-        .map(|r| r.last_response_id.clone())
+        .map(|r| r.request.last_response_id.clone())
         .collect();
     assert_eq!(
         resumed,


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to