This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new f0fc7c3 [SPARK-60687][CONNECT][TESTS] Reuse shared capture fixtures
for container inputs
f0fc7c3 is described below
commit f0fc7c3ac82336db9138e00436fdb1f8722c212c
Author: DB Tsai <[email protected]>
AuthorDate: Sun Nov 8 05:54:58 2026 +0900
[SPARK-60687][CONNECT][TESTS] Reuse shared capture fixtures for container
inputs
The container-input IPC tests duplicated the protobuf decoder and started a
dedicated capture server/process for each case. Their session cleanup relied on
an autouse guard that ran after that server had stopped, and the fixture
permanently appended to `sys.path`.
Reuse the existing shared mock session and Arrow IPC helper through a small
pandas adapter. This preserves the input assertions, uses the shared fixture's
session teardown and `ReleaseSession` support, checks pandas before acquiring
resources, and names the positional-column and named-column variants.
Validation: all 35 selected container-input and native-number tests pass.
Python syntax and `git diff --check` pass. Runtime tests use an older prebuilt
native extension; they do not prove an exact current-target build.
This PR remains a draft. Independent review of the complete final patch
left no remaining valid in-scope issues, including nits. Exact native build and
required Rust compile/test/fmt/Clippy and official API/parity checks remain
pending because the local toolchain is unavailable. No overall validation
convergence is claimed.
Follow-up to #129.
This pull request and its description were written by OpenAI Codex.
Closes #291 from dbtsai/codex/postmerge-pr-129-stop-capture-session.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/tests/test_dropin_offline.py | 158 ++++--------------------------------
1 file changed, 16 insertions(+), 142 deletions(-)
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index b694a16..a8666e6 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -6479,7 +6479,7 @@ def
test_local_input_native_numbers_inside_nested_values(mock_connect):
assert dict(actual["mapping"]) == {"x": 2**63 - 1}
-# Exact list/tuple conversion and separate-process input IPC regressions.
+# Exact list/tuple conversion and pandas input IPC regressions.
from datetime import date as container_input_date, datetime as
container_input_datetime
from decimal import Decimal as ContainerInputDecimal
@@ -6567,35 +6567,7 @@ def
test_container_input_recursive_leaf_and_row_types(container_input_capture_in
assert type(result["decimal"]) is ContainerInputDecimal
-def _container_input_fields(data):
- """Decode only the small protobuf envelope around a local Arrow
relation."""
- def varint(pos):
- value, shift = 0, 0
- while True:
- byte = data[pos]
- pos += 1
- value |= (byte & 127) << shift
- if byte < 128:
- return value, pos
- shift += 7
-
- result, pos = {}, 0
- while pos < len(data):
- tag, pos = varint(pos)
- wire = tag & 7
- if wire == 2:
- length, pos = varint(pos)
- result[tag >> 3] = data[pos:pos + length]
- pos += length
- elif wire == 0:
- _, pos = varint(pos)
- elif wire in (1, 5):
- pos += 8 if wire == 1 else 4
- else:
- raise AssertionError(f"unexpected protobuf wire type: {wire}")
- return result
-
-
+# DDL handler for the exact-date capture server below.
def _ddl_parse(request, context):
# As the server, of the DDL schemas that the client asks it to parse
(`_parse_ddl`).
import grpc
@@ -6606,130 +6578,34 @@ def _ddl_parse(request, context):
return response
-def _container_input_capture_server(connection):
- from concurrent.futures import ThreadPoolExecutor
- import grpc
-
- captured = []
-
- def execute(request, context):
- # ExecutePlanRequest.plan=3 -> Plan.root=1. Names/pandas input
- # uses Relation.to_df=18 -> ToDF.input=1 before local_relation=11.
- relation = _container_input_fields(
- _container_input_fields(_container_input_fields(request)[3])[1])
- if 18 in relation:
- relation =
_container_input_fields(_container_input_fields(relation[18])[1])
- assert 11 in relation, relation
- captured.append(_container_input_fields(relation[11])[1])
- context.abort(grpc.StatusCode.INVALID_ARGUMENT, "captured container
input")
- yield b"" # Make this a streaming handler.
-
- with ThreadPoolExecutor(max_workers=2) as executor:
- server = grpc.server(executor)
- def config(request, context):
- # Explicit-schema pandas input reads the Arrow conversion settings
first.
- request = parse_message(request)
- keys = [key.decode() for key in
- parse_message(parse_message(request[3][0])[2][0]).get(1,
[])]
- defaults = {
- "spark.sql.timestampType": "TIMESTAMP_LTZ",
- "spark.sql.session.timeZone": "UTC",
- "spark.sql.execution.arrow.useLargeVarTypes": "false",
- "spark.sql.session.localRelationCacheThreshold": str(1 << 60),
- "spark.sql.session.localRelationSizeLimit": str(1 << 60),
- "spark.sql.session.localRelationChunkSizeRows": str(1 << 30),
- "spark.sql.session.localRelationChunkSizeBytes": str(1 << 40),
- "spark.sql.session.localRelationBatchOfChunksSizeBytes": str(1
<< 40),
- "spark.sql.execution.pandas.convertToArrowArraySafely": "true",
- "spark.sql.execution.pandas.inferPandasDictAsMap": "false",
- }
- return message_field(1, request[1][0]) + b"".join(
- message_field(2, message_field(1, key) + message_field(2,
defaults[key]))
- for key in keys)
-
- server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
- "spark.connect.SparkConnectService", {
- "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute),
- "AnalyzePlan": grpc.unary_unary_rpc_method_handler(_ddl_parse),
- "Config": grpc.unary_unary_rpc_method_handler(config),
- }),))
- port = server.add_insecure_port("127.0.0.1:0")
- server.start()
- connection.send(port)
- try:
- while True:
- command = connection.recv()
- if command == "stop":
- break
- assert command == "get"
- connection.send(captured.pop(0))
- finally:
- server.stop(0).wait()
-
-
@pytest.fixture
-def container_input_capture_input():
- import multiprocessing
- import sys
- from pathlib import Path
-
- # importlib-mode pytest does not add the tests namespace to sys.path.
- # Spawned children need it to import this server target; append it so the
- # installed wheel still takes precedence over the source pyspark package.
- test_container_input_root = str(Path(__file__).resolve().parents[1])
- if test_container_input_root not in sys.path:
- sys.path.append(test_container_input_root)
- pytest.importorskip("grpc")
- from pyspark.sql import SparkSession
-
- context = multiprocessing.get_context("spawn")
- parent, child = context.Pipe()
- process = context.Process(target=_container_input_capture_server,
args=(child,))
- process.start()
- try:
- assert parent.poll(20), "capture service failed to start"
- spark =
SparkSession.builder.remote(f"sc://127.0.0.1:{parent.recv()}").getOrCreate()
+def container_input_capture_input(request):
+ # Check the optional dependency before acquiring the shared server/session.
+ pd = pytest.importorskip("pandas")
+ mock_connect = request.getfixturevalue("mock_connect")
- def capture(data, schema):
- import pyarrow as pa
- pd = pytest.importorskip("pandas")
- # The optimized converter remains the pandas/generic path. Local
- # rows use infer.convert, with deliberately different container
- # eligibility and numeric-protocol semantics.
- if not isinstance(data, pd.DataFrame):
- data = pd.DataFrame(data, dtype=object)
- df = spark.createDataFrame(data, schema=schema)
- with pytest.raises(Exception, match="captured container input"):
- df.collect()
- parent.send("get")
- assert parent.poll(10), "capture service failed to return the IPC
payload"
- return pa.ipc.open_stream(parent.recv()).read_all()
+ def capture(data, schema):
+ # Keep input on the pandas path (generic conversion or Arrow import).
+ # Local rows use infer.convert with different container/numeric
semantics.
+ if not isinstance(data, pd.DataFrame):
+ data = pd.DataFrame(data, dtype=object)
+ return _native_number_input_table(mock_connect, data, schema)
- yield capture
- finally:
- if process.is_alive():
- parent.send("stop")
- process.join(10)
- if process.is_alive():
- process.terminate()
- process.join(10)
- parent.close()
- child.close()
+ return capture
[email protected]("source", ["pandas_from_rows", "pandas"])
[email protected]("source", ["positional_columns", "named_columns"])
@pytest.mark.parametrize("schema_kind", ["inferred", "names", "explicit",
"ddl"])
@pytest.mark.parametrize("container", [list, tuple])
def test_container_input_array_input_ipc(container_input_capture_input,
source, schema_kind, container):
import pyarrow as pa
- from pyspark.sql import types as T
values = [container([1, None, -3]), container(), None, container([255,
256])]
schema = {"inferred": None, "names": ["items"], "ddl": "items
array<bigint>",
"explicit": T.StructType([T.StructField("items",
T.ArrayType(T.LongType()))])}[schema_kind]
data = [(value,) for value in values]
- if source == "pandas":
- pd = pytest.importorskip("pandas")
+ if source == "named_columns":
+ import pandas as pd
data = pd.DataFrame(data, columns=["items"])
table = container_input_capture_input(data, schema)
assert table.column(0).type == pa.list_(pa.int64())
@@ -6737,8 +6613,6 @@ def
test_container_input_array_input_ipc(container_input_capture_input, source,
def
test_container_input_nested_array_struct_binary_ipc(container_input_capture_input):
- from pyspark.sql import types as T
-
schema = T.StructType([
T.StructField("nested", T.ArrayType(T.ArrayType(T.LongType()))),
T.StructField("record", T.StructType([
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]