This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7b539b8 [SPARK-60698][CONNECT][PYTHON] Distinguish missing and null
Arrow settings in tests
7b539b8 is described below
commit 7b539b8d72a15134de0e9975ac438f27e7a4c07b
Author: DB Tsai <[email protected]>
AuthorDate: Sun Nov 8 05:26:29 2026 +0900
[SPARK-60698][CONNECT][PYTHON] Distinguish missing and null Arrow settings
in tests
### What changes were proposed in this pull request?
The Arrow input settings test parametrized missing and null values, but its
mock omitted both from `ConfigResponse`. Preserve null-valued keys with the
optional protobuf value unset and assert the two response shapes. Cover missing
and null values for all eight required settings, document the one-shot
configuration error hook, and assert the exact snapshot request in the schema
handoff test.
### Why are the changes needed?
Follow-up to #134
([SPARK-60698](https://issues.apache.org/jira/browse/SPARK-60698)). The two
previous cases produced identical wire responses, leaving the null-value path
untested. Only three of the eight required settings had missing-value coverage.
### Does this PR introduce _any_ user-facing change?
No, tests only.
### How was this patch tested?
A private wire-level reproduction fails on current master and passes after
this change. It executes the actual nested mock Config handler through
installed official protobuf classes; all 16 missing/null response cases pass,
including response ordering and optional-value presence. Python syntax and `git
diff --check` pass. Tooling unit tests: 12 passed, 2 skipped.
Draft: native runtime and mandatory pre-push validation remain pending.
This host has no Cargo/Rust/protoc, so the five required Cargo checks and
extension build could not start. The Arrow suite fails at collection without a
current-built extension (`Row` cannot be imported); API introspection and the
official parity gate remain unrun. Wire-level checks do not prove the Rust
error path. No full validation convergence is claimed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6). Independently source-reviewed using
installed local Claude Code.
This pull request and its description were written by Isaac.
Closes #290 from dbtsai/codex/postmerge-pr-134-config-snapshot-tests.
Authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/tests/test_arrow_handoff.py | 28 ++++++++++++++++++----------
1 file changed, 18 insertions(+), 10 deletions(-)
diff --git a/python/tests/test_arrow_handoff.py
b/python/tests/test_arrow_handoff.py
index 3911ad8..a26457d 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -462,7 +462,8 @@ def _arrow_input_service(
# The tests may queue status codes in `status_failures` and
`upload_failures`: each
# ArtifactStatus or AddArtifacts call fails with the next one, while any
is left.
state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": [],
- "config": config, "config_requests": [], "input_tables": [],
"tables": [],
+ "config": config, "config_requests": [], "config_responses": [],
+ "input_tables": [], "tables": [],
"upload_contexts": [], "status_contexts": [],
"status_failures": [], "upload_failures": [], "upload_attempts":
0,
"ddl_strings": []}
@@ -489,13 +490,16 @@ def _arrow_input_service(
state["config_requests"].append((operation, keys))
if config_error:
context.abort(grpc.StatusCode.UNAVAILABLE, "configuration
unavailable")
+ # Injected errors abort only the next Config call.
if error := state.pop("config_error", None):
context.abort(error, "configuration read failed")
# Deliberately reverse responses and omit missing keys to test key
mapping.
- return protocol("ConfigResponse", session_id=request.session_id,
- pairs=[dict(key=k, value=config[k]) for k in
reversed(keys)
- if config.get(k) is not None]
- ).SerializeToString()
+ # A present null key leaves the optional protobuf value unset.
+ response = protocol("ConfigResponse", session_id=request.session_id,
+ pairs=[dict(key=k, value=config[k]) for k in
reversed(keys)
+ if k in config])
+ state["config_responses"].append(response)
+ return response.SerializeToString()
def top_level_split(text):
parts, depth, start = [], 0, 0
@@ -838,9 +842,7 @@ def test_arrow_input_schema_handoff(protocol, cached,
explicit):
assert frame.toArrow().column(0).to_pylist() == [2]
assert state["result"] == (2, 1, 2)
assert state["arrow_schemas"] == [table.schema]
- assert len(state["config_requests"]) == 1
- assert state["config_requests"][0][0] == "get"
- assert len(state["config_requests"][0][1]) == 8
+ assert state["config_requests"] == [("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)]
relation = state["relations"][0]
if cached:
local = relation.chunked_cached_local_relation
@@ -1064,9 +1066,9 @@ def
test_arrow_input_batched_normalization_configs(protocol, timestamp_type, zon
assert not state["relations"][0].local_relation.HasField("schema")
[email protected]("missing", _NORMALIZATION_KEYS)
[email protected]("missing", _NORMALIZATION_KEYS + _UPLOAD_KEYS)
@pytest.mark.parametrize("present_null", [False, True])
-def test_arrow_input_missing_normalization_config(protocol, missing,
present_null):
+def test_arrow_input_missing_config(protocol, missing, present_null):
with _arrow_input_service(protocol) as (spark, state):
if present_null:
state["config"][missing] = None
@@ -1076,6 +1078,12 @@ def
test_arrow_input_missing_normalization_config(protocol, missing, present_nul
with pytest.raises(Exception, match="SQL_CONF_NOT_FOUND"):
spark.createDataFrame(pa.table({"x": [1, None], "s": ["hello",
None]}))
assert state["config_requests"] == [("get", _NORMALIZATION_KEYS +
_UPLOAD_KEYS)]
+ pairs = {pair.key: pair for pair in state["config_responses"][0].pairs}
+ if present_null:
+ assert missing in pairs
+ assert not pairs[missing].HasField("value")
+ else:
+ assert missing not in pairs
assert state["uploads"] == 0
assert state["input_tables"] == []
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]