This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 308662e [SPARK-60942][CONNECT] Simplify artifact hashes and clarify
task handling
308662e is described below
commit 308662e747e53173d2f9f12b24750c7165912d5e
Author: DB Tsai <[email protected]>
AuthorDate: Thu Nov 5 16:08:15 2026 -0800
[SPARK-60942][CONNECT] Simplify artifact hashes and clarify task handling
### What changes were proposed in this pull request?
Follow up on #164 under the original
[SPARK-60942](https://issues.apache.org/jira/browse/SPARK-60942).
- Explain that rethrowing a blocking hashing task's panic preserves inline
hashing behavior.
- Replace the async-task cancellation test with a queued `spawn_blocking`
task, using explicit abort to model runtime shutdown cancelling queued hashing
work.
- Use the digest's `LowerHex` implementation for SHA-256 strings and cover
a digest with a leading zero.
### Why are the changes needed?
Document the intentional panic behavior and the shutdown scenario behind
the cancellation test, and simplify hash encoding while preserving its
64-character output.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- 115 core and 419 API library tests passed, including the updated
cancellation and leading-zero cases.
- All Rust integration targets compiled, including `wasm-udf`; formatting
and the release wheel build passed.
- 613 offline/mock Python tests passed (8 existing skips).
- On an isolated Spark 4.3 server, 10 Arrow e2e cases, 34 Arrow-input
cases, and 33 official creation/collection parity cases passed (2 existing
parity skips). API introspection found only the intentional `Column.to_plan`
difference.
- Advisory workspace Clippy reports errors in unchanged test and binding
code; diagnostic locations were checked against base `014674e`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex
This pull request and its description were written by Isaac.
Closes #170 from dbtsai/codex/spark-60942-hashing-followup.
Authored-by: DB Tsai <[email protected]>
Signed-off-by: DB Tsai <[email protected]>
Co-authored-by: Isaac <[email protected]>
---
crates/spark-connect-core/src/client.rs | 55 +++++++++++++++++++++++----------
1 file changed, 39 insertions(+), 16 deletions(-)
diff --git a/crates/spark-connect-core/src/client.rs
b/crates/spark-connect-core/src/client.rs
index a72de22..60fc68b 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -63,7 +63,6 @@ const ARTIFACT_HASH_INLINE_MAX_BLOBS: usize = 128;
/// blocking hashing starts, the task retains the blob buffers until it
finishes.
async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) -> Result<(Vec<Vec<u8>>,
Vec<String>)> {
use sha2::{Digest, Sha256};
- use std::fmt::Write;
let hash_inline = blobs.len() <= ARTIFACT_HASH_INLINE_MAX_BLOBS
&& blobs
@@ -75,13 +74,7 @@ async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) ->
Result<(Vec<Vec<u8>>, Vec<S
let hash = move || {
let hashes = blobs
.iter()
- .map(|blob| {
- let mut hash = String::with_capacity(64);
- for byte in Sha256::digest(blob) {
- write!(&mut hash, "{byte:02x}").expect("writing to a
String cannot fail");
- }
- hash
- })
+ .map(|blob| format!("{:x}", Sha256::digest(blob)))
.collect();
(blobs, hashes)
};
@@ -95,6 +88,7 @@ async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) ->
Result<(Vec<Vec<u8>>, Vec<S
fn artifact_hashing_task_error(error: tokio::task::JoinError) -> SparkError {
if error.is_panic() {
+ // Preserve inline hashing panic behavior when hashing is offloaded.
std::panic::resume_unwind(error.into_panic());
}
SparkError::connect_msg(format!("Artifact hashing task failed: {error}"))
@@ -1266,8 +1260,13 @@ mod tests {
#[test]
fn test_small_artifact_hashing_stays_inline_and_preserves_buffers() {
- let (pending, hashes) =
- hash_with_busy_blocking_pool(vec![Vec::new(), b"abc".to_vec(),
b"abc".to_vec()]);
+ // The last digest starts with a zero, which hex encoding must
preserve.
+ let (pending, hashes) = hash_with_busy_blocking_pool(vec![
+ Vec::new(),
+ b"abc".to_vec(),
+ b"abc".to_vec(),
+ b"s".to_vec(),
+ ]);
assert!(
!pending,
"small artifact batches must not wait for a blocking thread"
@@ -1278,6 +1277,7 @@ mod tests {
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89",
]
);
}
@@ -1344,12 +1344,35 @@ mod tests {
assert_eq!(panic.downcast_ref::<&str>(), Some(&"hashing panic"));
}
- #[tokio::test]
- async fn test_artifact_hashing_reports_task_cancellation() {
- let task = tokio::spawn(std::future::pending::<()>());
- task.abort();
- let error = artifact_hashing_task_error(task.await.unwrap_err());
- assert!(error.to_string().contains("Artifact hashing task failed:"));
+ #[test]
+ fn test_artifact_hashing_task_error_reports_queued_task_cancellation() {
+ let runtime = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .max_blocking_threads(1)
+ .build()
+ .unwrap();
+ runtime.block_on(async {
+ let (started, ready) = tokio::sync::oneshot::channel();
+ let (release, wait) = std::sync::mpsc::channel();
+ let blocker = tokio::task::spawn_blocking(move || {
+ started.send(()).unwrap();
+ let _ = wait.recv();
+ });
+ ready.await.unwrap();
+
+ // Explicit abort models runtime shutdown cancelling queued
hashing work;
+ // production hashing does not abort its handle.
+ let task = tokio::task::spawn_blocking(|| ());
+ task.abort();
+ // Release the blocking worker before assertions to avoid hanging
shutdown.
+ release.send(()).unwrap();
+ blocker.await.unwrap();
+
+ let cancelled = task.await.unwrap_err();
+ assert!(cancelled.is_cancelled());
+ let error = artifact_hashing_task_error(cancelled);
+ assert!(error.to_string().contains("Artifact hashing task
failed:"));
+ });
}
#[tokio::test]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]