This is an automated email from the ASF dual-hosted git repository.

dbtsai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 308662e  [SPARK-60942][CONNECT] Simplify artifact hashes and clarify 
task handling
308662e is described below

commit 308662e747e53173d2f9f12b24750c7165912d5e
Author: DB Tsai <[email protected]>
AuthorDate: Thu Nov 5 16:08:15 2026 -0800

    [SPARK-60942][CONNECT] Simplify artifact hashes and clarify task handling
    
    ### What changes were proposed in this pull request?
    
    Follow up on #164 under the original 
[SPARK-60942](https://issues.apache.org/jira/browse/SPARK-60942).
    
    - Explain that rethrowing a blocking hashing task's panic preserves inline 
hashing behavior.
    - Replace the async-task cancellation test with a queued `spawn_blocking` 
task, using explicit abort to model runtime shutdown cancelling queued hashing 
work.
    - Use the digest's `LowerHex` implementation for SHA-256 strings and cover 
a digest with a leading zero.
    
    ### Why are the changes needed?
    
    Document the intentional panic behavior and the shutdown scenario behind 
the cancellation test, and simplify hash encoding while preserving its 
64-character output.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    - 115 core and 419 API library tests passed, including the updated 
cancellation and leading-zero cases.
    - All Rust integration targets compiled, including `wasm-udf`; formatting 
and the release wheel build passed.
    - 613 offline/mock Python tests passed (8 existing skips).
    - On an isolated Spark 4.3 server, 10 Arrow e2e cases, 34 Arrow-input 
cases, and 33 official creation/collection parity cases passed (2 existing 
parity skips). API introspection found only the intentional `Column.to_plan` 
difference.
    - Advisory workspace Clippy reports errors in unchanged test and binding 
code; diagnostic locations were checked against base `014674e`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex
    
    This pull request and its description were written by Isaac.
    
    Closes #170 from dbtsai/codex/spark-60942-hashing-followup.
    
    Authored-by: DB Tsai <[email protected]>
    Signed-off-by: DB Tsai <[email protected]>
    Co-authored-by: Isaac <[email protected]>
---
 crates/spark-connect-core/src/client.rs | 55 +++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 16 deletions(-)

diff --git a/crates/spark-connect-core/src/client.rs 
b/crates/spark-connect-core/src/client.rs
index a72de22..60fc68b 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -63,7 +63,6 @@ const ARTIFACT_HASH_INLINE_MAX_BLOBS: usize = 128;
 /// blocking hashing starts, the task retains the blob buffers until it 
finishes.
 async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) -> Result<(Vec<Vec<u8>>, 
Vec<String>)> {
     use sha2::{Digest, Sha256};
-    use std::fmt::Write;
 
     let hash_inline = blobs.len() <= ARTIFACT_HASH_INLINE_MAX_BLOBS
         && blobs
@@ -75,13 +74,7 @@ async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) -> 
Result<(Vec<Vec<u8>>, Vec<S
     let hash = move || {
         let hashes = blobs
             .iter()
-            .map(|blob| {
-                let mut hash = String::with_capacity(64);
-                for byte in Sha256::digest(blob) {
-                    write!(&mut hash, "{byte:02x}").expect("writing to a 
String cannot fail");
-                }
-                hash
-            })
+            .map(|blob| format!("{:x}", Sha256::digest(blob)))
             .collect();
         (blobs, hashes)
     };
@@ -95,6 +88,7 @@ async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) -> 
Result<(Vec<Vec<u8>>, Vec<S
 
 fn artifact_hashing_task_error(error: tokio::task::JoinError) -> SparkError {
     if error.is_panic() {
+        // Preserve inline hashing panic behavior when hashing is offloaded.
         std::panic::resume_unwind(error.into_panic());
     }
     SparkError::connect_msg(format!("Artifact hashing task failed: {error}"))
@@ -1266,8 +1260,13 @@ mod tests {
 
     #[test]
     fn test_small_artifact_hashing_stays_inline_and_preserves_buffers() {
-        let (pending, hashes) =
-            hash_with_busy_blocking_pool(vec![Vec::new(), b"abc".to_vec(), 
b"abc".to_vec()]);
+        // The last digest starts with a zero, which hex encoding must 
preserve.
+        let (pending, hashes) = hash_with_busy_blocking_pool(vec![
+            Vec::new(),
+            b"abc".to_vec(),
+            b"abc".to_vec(),
+            b"s".to_vec(),
+        ]);
         assert!(
             !pending,
             "small artifact batches must not wait for a blocking thread"
@@ -1278,6 +1277,7 @@ mod tests {
                 
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
                 
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
                 
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+                
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89",
             ]
         );
     }
@@ -1344,12 +1344,35 @@ mod tests {
         assert_eq!(panic.downcast_ref::<&str>(), Some(&"hashing panic"));
     }
 
-    #[tokio::test]
-    async fn test_artifact_hashing_reports_task_cancellation() {
-        let task = tokio::spawn(std::future::pending::<()>());
-        task.abort();
-        let error = artifact_hashing_task_error(task.await.unwrap_err());
-        assert!(error.to_string().contains("Artifact hashing task failed:"));
+    #[test]
+    fn test_artifact_hashing_task_error_reports_queued_task_cancellation() {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .max_blocking_threads(1)
+            .build()
+            .unwrap();
+        runtime.block_on(async {
+            let (started, ready) = tokio::sync::oneshot::channel();
+            let (release, wait) = std::sync::mpsc::channel();
+            let blocker = tokio::task::spawn_blocking(move || {
+                started.send(()).unwrap();
+                let _ = wait.recv();
+            });
+            ready.await.unwrap();
+
+            // Explicit abort models runtime shutdown cancelling queued 
hashing work;
+            // production hashing does not abort its handle.
+            let task = tokio::task::spawn_blocking(|| ());
+            task.abort();
+            // Release the blocking worker before assertions to avoid hanging 
shutdown.
+            release.send(()).unwrap();
+            blocker.await.unwrap();
+
+            let cancelled = task.await.unwrap_err();
+            assert!(cancelled.is_cancelled());
+            let error = artifact_hashing_task_error(cancelled);
+            assert!(error.to_string().contains("Artifact hashing task 
failed:"));
+        });
     }
 
     #[tokio::test]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to