This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new a98551e [SPARK-60942][CONNECT] Tighten artifact hashing and wire tests
a98551e is described below
commit a98551e501d593b5f5d01048037d4325b548a614
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 08:38:41 2026 +0900
[SPARK-60942][CONNECT] Tighten artifact hashing and wire tests
### What changes were proposed in this pull request?
Follow up on #164 by lowering the inline artifact-hashing cutoff from 1 MiB
to 256 KiB and updating the boundary-test digests and helper documentation.
Tighten the Python attribution assertion to require `user_agent + " spark/"`,
and make file-upload frame-coverage failures explain the fixture's 32 KiB
chunk-size assumption.
### Why are the changes needed?
A smaller cutoff limits synchronous work on async workers, including CPUs
without SHA acceleration. The tighter attribution assertion rejects a bare
custom user-agent prefix. The fixture message makes a lost chunk-coverage
failure easier to diagnose after a chunk-size change.
### Does this PR introduce _any_ user-facing change?
Cache batches totaling more than 256 KiB now offload hashing to the
blocking pool. The independent 128-blob limit still applies.
### How was this patch tested?
- Existing boundary tests verify inline completion at 256 KiB and
offloading when the combined input exceeds it by one byte, with known hashes
and preserved buffers.
- 115 core unit tests, 416 API unit tests, and 585 offline/mock Python
tests passed (8 existing skips).
- All API integration targets compile with and without `wasm-udf`;
formatting and whitespace checks pass.
- Against an isolated Spark 4.3 server, native Arrow e2e/input tests and
the official creation/collection parity modules passed. API introspection found
only the intentional `Column.to_plan` difference.
- Advisory workspace Clippy reports errors in files unchanged from master;
no error file is modified by this PR.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex
This pull request and its description were written by Isaac.
Closes #166 from dbtsai/codex/pr164-review-followup.
Authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect-core/src/client.rs | 11 ++++++-----
python/tests/test_arrow_handoff.py | 10 ++++++----
2 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/crates/spark-connect-core/src/client.rs
b/crates/spark-connect-core/src/client.rs
index 1e49a3f..a72de22 100644
--- a/crates/spark-connect-core/src/client.rs
+++ b/crates/spark-connect-core/src/client.rs
@@ -53,11 +53,12 @@ async fn wait_ready(grpc: &mut
tonic::client::Grpc<Channel>) -> Result<()> {
}
/// Avoid a blocking-pool round trip for small artifact batches.
-const ARTIFACT_HASH_INLINE_MAX_BYTES: usize = 1024 * 1024;
+/// Keep synchronous work small on async workers, including CPUs without SHA
acceleration.
+const ARTIFACT_HASH_INLINE_MAX_BYTES: usize = 256 * 1024;
/// Bound per-blob digest and encoding work even when every blob is empty.
const ARTIFACT_HASH_INLINE_MAX_BLOBS: usize = 128;
-/// Hash up to 128 blobs totaling at most 1 MiB inline; offload larger batches.
+/// Hash up to 128 blobs totaling at most 256 KiB inline; offload larger
batches.
/// Returns the original buffers for upload framing. If the caller is dropped
after
/// blocking hashing starts, the task retains the blob buffers until it
finishes.
async fn hash_artifact_blobs(blobs: Vec<Vec<u8>>) -> Result<(Vec<Vec<u8>>,
Vec<String>)> {
@@ -1288,7 +1289,7 @@ mod tests {
assert!(!pending, "the inline size limit is inclusive");
assert_eq!(
hashes,
-
["9bc1b2a288b26af7257a36277ae3816a7d4f16e89c1e7e77d0a5c48bad62b360"]
+
["dd3dde87623d9a6b354c68c943d189c89c63652d945e7bbdf0986cae91a49521"]
);
}
@@ -1306,8 +1307,8 @@ mod tests {
assert_eq!(
hashes,
[
-
"85a84a75886e8a526dbec4e16e3375faa307b4aead79c9ed3264c0477a6f6eba",
-
"8d666ffa0196841cce7c504d43bf27e311775220d2490a23a2f984a43d901015",
+
"b44ffb72fcc259676bd80495fef1b44b808ca8f1ffe1b1706a4d7911b0e31f11",
+
"7e009ea4ef882e385b3c0bcbbfa8d009bb0a633bdd764415c09182ee0e75da73",
]
);
}
diff --git a/python/tests/test_arrow_handoff.py
b/python/tests/test_arrow_handoff.py
index a379eb8..459a65e 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -591,7 +591,7 @@ def _assert_artifact_contexts(contexts, *, session_id,
user_id, user_agent):
assert len(client_types) == 1
client_type = client_types.pop()
assert client_type
- assert client_type.startswith(user_agent)
+ assert client_type.startswith(user_agent + " spark/")
for context in contexts:
grpc_user_agent = context["grpc_user_agent"]
assert grpc_user_agent == client_type or
grpc_user_agent.startswith(client_type + " ")
@@ -656,9 +656,11 @@ def test_file_upload_attribution_on_every_frame(
for path, destination in zip([small, large], destinations):
spark.copyFromLocalToFs(str(path), destination)
artifact_names = ["forward_to_fs/" + destination for destination
in destinations]
- assert {context["payload"] for context in state["upload_contexts"]} ==
{
- "batch", "begin_chunk", "chunk"
- }
+ payloads = {context["payload"] for context in state["upload_contexts"]}
+ assert payloads == {"batch", "begin_chunk", "chunk"}, (
+ f"Expected all artifact frame kinds, got {payloads}. "
+ "This fixture assumes a 32 KiB CHUNK_SIZE; adjust its sizes if
CHUNK_SIZE changes."
+ )
assert state["cache"][artifact_names[0]] == small.read_bytes()
assert state["cache"][artifact_names[1]] == large.read_bytes()
assert state["uploads"] == (1 if upload == "file" else 2)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]