This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new a98551e  [SPARK-60942][CONNECT] Tighten artifact hashing and wire tests
a98551e is described below

commit a98551e501d593b5f5d01048037d4325b548a614
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 08:38:41 2026 +0900

    [SPARK-60942][CONNECT] Tighten artifact hashing and wire tests
    
    ### What changes were proposed in this pull request?
    
    Follow up on #164 by lowering the inline artifact-hashing cutoff from 1 MiB 
to 256 KiB and updating the boundary-test digests and helper documentation. 
Tighten the Python attribution assertion to require `user_agent + " spark/"`, 
and make file-upload frame-coverage failures explain the fixture's 32 KiB 
chunk-size assumption.
    
    ### Why are the changes needed?
    
    A smaller cutoff limits synchronous work on async workers, including CPUs 
without SHA acceleration. The tighter attribution assertion rejects a bare 
custom user-agent prefix. The fixture message makes a lost chunk-coverage 
failure easier to diagnose after a chunk-size change.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Cache batches totaling more than 256 KiB now offload hashing to the 
blocking pool. The independent 128-blob limit still applies.
    
    ### How was this patch tested?
    
    - Existing boundary tests verify inline completion at 256 KiB and 
offloading when the combined input exceeds it by one byte, with known hashes 
and preserved buffers.
    - 115 core unit tests, 416 API unit tests, and 585 offline/mock Python 
tests passed (8 existing skips).
    - All API integration targets compile with and without `wasm-udf`; 
formatting and whitespace checks pass.
    - Against an isolated Spark 4.3 server, native Arrow e2e/input tests and 
the official creation/collection parity modules passed. API introspection found 
only the intentional `Column.to_plan` difference.
    - Advisory workspace Clippy reports errors in files unchanged from master; 
no error file is modified by this PR.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex
    
    This pull request and its description were written by Isaac.
    
    Closes #166 from dbtsai/codex/pr164-review-followup.
    
    Authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect-core/src/client.rs | 11 ++++++-----
 python/tests/test_arrow_handoff.py      | 10 ++++++----
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/crates/spark-connect-core/src/client.rs 
b/crates/spark-connect-core/src/client.rs
index 1e49a3f..a72de22 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -53,11 +53,12 @@ async fn wait_ready(grpc: &mut 
tonic::client::Grpc<Channel>) -> Result<()> {
 }
 
 /// Avoid a blocking-pool round trip for small artifact batches.
-const ARTIFACT_HASH_INLINE_MAX_BYTES: usize = 1024 * 1024;
+/// Keep synchronous work small on async workers, including CPUs without SHA 
acceleration.
+const ARTIFACT_HASH_INLINE_MAX_BYTES: usize = 256 * 1024;
 /// Bound per-blob digest and encoding work even when every blob is empty.
 const ARTIFACT_HASH_INLINE_MAX_BLOBS: usize = 128;
 
-/// Hash up to 128 blobs totaling at most 1 MiB inline; offload larger batches.
+/// Hash up to 128 blobs totaling at most 256 KiB inline; offload larger 
batches.
 /// Returns the original buffers for upload framing. If the caller is dropped 
after
 /// blocking hashing starts, the task retains the blob buffers until it 
finishes.
 async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) -> Result<(Vec<Vec<u8>>, 
Vec<String>)> {
@@ -1288,7 +1289,7 @@ mod tests {
         assert!(!pending, "the inline size limit is inclusive");
         assert_eq!(
             hashes,
-            
["9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360"]
+            
["dd3dde87623d9a6b354c68c943d189c89c63652d945e7bbdf0986cae91a49521"]
         );
     }
 
@@ -1306,8 +1307,8 @@ mod tests {
         assert_eq!(
             hashes,
             [
-                
"85a84a75886e8a526dbec4e16e3375faa307b4aead79c9ed3264c0477a6f6eba",
-                
"8d666ffa0196841cce7c504d43bf27e311775220d2490a23a2f984a43d901015",
+                
"b44ffb72fcc259676bd80495fef1b44b808ca8f1ffe1b1706a4d7911b0e31f11",
+                
"7e009ea4ef882e385b3c0bcbbfa8d009bb0a633bdd764415c09182ee0e75da73",
             ]
         );
     }
diff --git a/python/tests/test_arrow_handoff.py 
b/python/tests/test_arrow_handoff.py
index a379eb8..459a65e 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -591,7 +591,7 @@ def _assert_artifact_contexts(contexts, *, session_id, 
user_id, user_agent):
     assert len(client_types) == 1
     client_type = client_types.pop()
     assert client_type
-    assert client_type.startswith(user_agent)
+    assert client_type.startswith(user_agent + " spark/")
     for context in contexts:
         grpc_user_agent = context["grpc_user_agent"]
         assert grpc_user_agent == client_type or 
grpc_user_agent.startswith(client_type + " ")
@@ -656,9 +656,11 @@ def test_file_upload_attribution_on_every_frame(
             for path, destination in zip([small, large], destinations):
                 spark.copyFromLocalToFs(str(path), destination)
             artifact_names = ["forward_to_fs/" + destination for destination 
in destinations]
-        assert {context["payload"] for context in state["upload_contexts"]} == 
{
-            "batch", "begin_chunk", "chunk"
-        }
+        payloads = {context["payload"] for context in state["upload_contexts"]}
+        assert payloads == {"batch", "begin_chunk", "chunk"}, (
+            f"Expected all artifact frame kinds, got {payloads}. "
+            "This fixture assumes a 32 KiB CHUNK_SIZE; adjust its sizes if 
CHUNK_SIZE changes."
+        )
         assert state["cache"][artifact_names[0]] == small.read_bytes()
         assert state["cache"][artifact_names[1]] == large.read_bytes()
         assert state["uploads"] == (1 if upload == "file" else 2)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to