This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 7b539b8  [SPARK-60698][CONNECT][PYTHON] Distinguish missing and null 
Arrow settings in tests
7b539b8 is described below

commit 7b539b8d72a15134de0e9975ac438f27e7a4c07b
Author: DB Tsai <[email protected]>
AuthorDate: Sun Nov 8 05:26:29 2026 +0900

    [SPARK-60698][CONNECT][PYTHON] Distinguish missing and null Arrow settings 
in tests
    
    ### What changes were proposed in this pull request?
    
    The Arrow input settings test parametrized missing and null values, but its 
mock omitted both from `ConfigResponse`. Preserve null-valued keys with the 
optional protobuf value unset and assert the two response shapes. Cover missing 
and null values for all eight required settings, document the one-shot 
configuration error hook, and assert the exact snapshot request in the schema 
handoff test.
    
    ### Why are the changes needed?
    
    Follow-up to #134 
([SPARK-60698](https://issues.apache.org/jira/browse/SPARK-60698)). The two 
previous cases produced identical wire responses, leaving the null-value path 
untested. Only three of the eight required settings had missing-value coverage.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, tests only.
    
    ### How was this patch tested?
    
    A private wire-level reproduction fails on current master and passes after 
this change. It executes the actual nested mock Config handler through 
installed official protobuf classes; all 16 missing/null response cases pass, 
including response ordering and optional-value presence. Python syntax and `git 
diff --check` pass. Tooling unit tests: 12 passed, 2 skipped.
    
    Draft: native runtime and mandatory pre-push validation remain pending. 
This host has no Cargo/Rust/protoc, so the five required Cargo checks and 
extension build could not start. The Arrow suite fails at collection without a 
current-built extension (`Row` cannot be imported); API introspection and the 
official parity gate remain unrun. Wire-level checks do not prove the Rust 
error path. No full validation convergence is claimed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6). Independently source-reviewed using 
installed local Claude Code.
    
    This pull request and its description were written by Isaac.
    
    Closes #290 from dbtsai/codex/postmerge-pr-134-config-snapshot-tests.
    
    Authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 python/tests/test_arrow_handoff.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/python/tests/test_arrow_handoff.py 
b/python/tests/test_arrow_handoff.py
index 3911ad8..a26457d 100644
--- a/python/tests/test_arrow_handoff.py
+++ b/python/tests/test_arrow_handoff.py
@@ -462,7 +462,8 @@ def _arrow_input_service(
     # The tests may queue status codes in `status_failures` and 
`upload_failures`: each
     # ArtifactStatus or AddArtifacts call fails with the next one, while any 
is left.
     state = {"cache": {}, "uploads": 0, "relations": [], "message_sizes": [],
-             "config": config, "config_requests": [], "input_tables": [], 
"tables": [],
+             "config": config, "config_requests": [], "config_responses": [],
+             "input_tables": [], "tables": [],
              "upload_contexts": [], "status_contexts": [],
              "status_failures": [], "upload_failures": [], "upload_attempts": 
0,
              "ddl_strings": []}
@@ -489,13 +490,16 @@ def _arrow_input_service(
         state["config_requests"].append((operation, keys))
         if config_error:
             context.abort(grpc.StatusCode.UNAVAILABLE, "configuration 
unavailable")
+        # Injected errors abort only the next Config call.
         if error := state.pop("config_error", None):
             context.abort(error, "configuration read failed")
         # Deliberately reverse responses and omit missing keys to test key 
mapping.
-        return protocol("ConfigResponse", session_id=request.session_id,
-                        pairs=[dict(key=k, value=config[k]) for k in 
reversed(keys)
-                               if config.get(k) is not None]
-                        ).SerializeToString()
+        # A present null key leaves the optional protobuf value unset.
+        response = protocol("ConfigResponse", session_id=request.session_id,
+                            pairs=[dict(key=k, value=config[k]) for k in 
reversed(keys)
+                                   if k in config])
+        state["config_responses"].append(response)
+        return response.SerializeToString()
 
     def top_level_split(text):
         parts, depth, start = [], 0, 0
@@ -838,9 +842,7 @@ def test_arrow_input_schema_handoff(protocol, cached, 
explicit):
         assert frame.toArrow().column(0).to_pylist() == [2]
         assert state["result"] == (2, 1, 2)
         assert state["arrow_schemas"] == [table.schema]
-        assert len(state["config_requests"]) == 1
-        assert state["config_requests"][0][0] == "get"
-        assert len(state["config_requests"][0][1]) == 8
+        assert state["config_requests"] == [("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)]
         relation = state["relations"][0]
         if cached:
             local = relation.chunked_cached_local_relation
@@ -1064,9 +1066,9 @@ def 
test_arrow_input_batched_normalization_configs(protocol, timestamp_type, zon
         assert not state["relations"][0].local_relation.HasField("schema")
 
 
[email protected]("missing", _NORMALIZATION_KEYS)
[email protected]("missing", _NORMALIZATION_KEYS + _UPLOAD_KEYS)
 @pytest.mark.parametrize("present_null", [False, True])
-def test_arrow_input_missing_normalization_config(protocol, missing, 
present_null):
+def test_arrow_input_missing_config(protocol, missing, present_null):
     with _arrow_input_service(protocol) as (spark, state):
         if present_null:
             state["config"][missing] = None
@@ -1076,6 +1078,12 @@ def 
test_arrow_input_missing_normalization_config(protocol, missing, present_nul
         with pytest.raises(Exception, match="SQL_CONF_NOT_FOUND"):
             spark.createDataFrame(pa.table({"x": [1, None], "s": ["hello", 
None]}))
         assert state["config_requests"] == [("get", _NORMALIZATION_KEYS + 
_UPLOAD_KEYS)]
+        pairs = {pair.key: pair for pair in state["config_responses"][0].pairs}
+        if present_null:
+            assert missing in pairs
+            assert not pairs[missing].HasField("value")
+        else:
+            assert missing not in pairs
         assert state["uploads"] == 0
         assert state["input_tables"] == []
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to