This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 8a6fa21 [SPARK-60687][CONNECT] Fast-path exact list and tuple input
containers
8a6fa21 is described below
commit 8a6fa210f737ff9a29246e2321f1d642cf7794d6
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 17:43:38 2026 +0900
[SPARK-60687][CONNECT] Fast-path exact list and tuple input containers
### What changes were proposed in this pull request?
`py_to_value`, the generic Python-to-`Value` conversion, now converts an
exact built-in `list` or `tuple` to `Value::List` directly, converting its
elements recursively, before its generic checks (bool extraction, `Decimal`
detection and the numeric extraction attempts), which a list or tuple never
passes. Subclasses of `list` and `tuple` keep the generic path, so a subclass
that defines `__index__`, `__float__` or `__iter__` converts as before. Rows
stay structs, bytes stay binary.
This follows #111 (SPARK-60661), which did the same for exact `int` and
`float`. `py_to_value` converts the cells of pandas input
(`createDataFrame(pandas_df)` turns every column into Python objects first) and
the values of `fillna`, `pivot` and `Row` conversion; local rows go through
`infer.convert`.
### Why are the changes needed?
[SPARK-60687](https://issues.apache.org/jira/browse/SPARK-60687): each list
or tuple went through a failing `bool` extraction, the `Decimal` check and
failing numeric extractions, each building and dropping a Python error, before
it was recognized. Measured on macOS against a local Spark 4.3.0 server, as the
client CPU time of `createDataFrame` of a pandas DataFrame of 10,000 rows with
a list, a tuple and a nested list column (median of 7, two rounds): 84 ms on
the current master, 25- [...]
### Does this PR introduce _any_ user-facing change?
No, other than speed. The converted values are the same.
### How was this patch tested?
- New offline tests capture the Arrow IPC of pandas input with lists and
tuples in a separate-process mock server: nulls, empty and nested arrays,
binary values, nested rows, arrays with inferred, named, explicit and DDL
schemas, unsupported elements, and subclasses with `__index__`, `__float__` and
`__iter__`.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (775 passed), the build of all tests
with `wasm-udf`, the offline Python suites (900 passed, 9 skipped), the live
drop-in and Arrow suites (81 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master and the
measurement by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #129 from dbtsai/codex/spark-60687-exact-containers.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 17 +++
python/tests/test_dropin_offline.py | 238 ++++++++++++++++++++++++++++++++++++
2 files changed, 255 insertions(+)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index e305851..dd8009f 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1536,6 +1536,23 @@ pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) ->
PyResult<Value> {
return Ok(Value::Double(f.value()));
}
+ // Exact built-in containers cannot implement numeric protocols. Convert
them
+ // before Decimal/numeric probing; subclasses retain the generic
precedence below.
+ if let Ok(list) = obj.cast_exact::<pyo3::types::PyList>() {
+ let items = list
+ .iter()
+ .map(|item| py_to_value(&item))
+ .collect::<PyResult<Vec<_>>>()?;
+ return Ok(Value::List(items));
+ }
+ if let Ok(tuple) = obj.cast_exact::<pyo3::types::PyTuple>() {
+ let items = tuple
+ .iter()
+ .map(|item| py_to_value(&item))
+ .collect::<PyResult<Vec<_>>>()?;
+ return Ok(Value::List(items));
+ }
+
// Check for bool (before int because bool is a subclass of int in Python)
if let Ok(b) = obj.extract::<bool>() {
return Ok(Value::Bool(b));
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 8421806..3320ef2 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -4225,6 +4225,244 @@ def
test_local_input_native_numbers_inside_nested_values(mock_connect):
assert dict(actual["mapping"]) == {"x": 2**63 - 1}
+# Exact list/tuple conversion and separate-process input IPC regressions.
+from datetime import date as container_input_date, datetime as
container_input_datetime
+from decimal import Decimal as ContainerInputDecimal
+
+from pyspark.sql import Row as ContainerInputRow
+
+
[email protected]("container", [list, tuple])
+def test_container_input_values_and_errors(container_input_capture_input,
container):
+ array_schema = T.StructType([T.StructField("value",
T.ArrayType(T.LongType()))])
+ for value, expected in [(container([1, None, -3]), [1, None, -3]),
+ (container(), []),
+ (container([1, 2, 255]), [1, 2, 255])]:
+ table = container_input_capture_input([(value,)], array_schema)
+ assert table.column(0)[0].as_py() == expected
+ binary_schema = T.StructType([T.StructField("value", T.BinaryType())])
+ for value in (b"abc", bytearray(b"abc")):
+ table = container_input_capture_input([(value,)], binary_schema)
+ assert table.column(0)[0].as_py() == b"abc"
+ nested_schema = T.StructType([
+ T.StructField("value", T.ArrayType(T.ArrayType(T.LongType())))])
+ value = container([container([1, None]), container([2]), container()])
+ table = container_input_capture_input([(value,)], nested_schema)
+ assert table.column(0)[0].as_py() == [[1, None], [2], []]
+ with pytest.raises(TypeError, match="unsupported value type"):
+ container_input_capture_input([(container([object()]),)], array_schema)
+
+
[email protected]("container", [list, tuple])
+def
test_container_input_subclass_numeric_precedence(container_input_capture_input,
container):
+ trace = []
+
+ class Indexed(container):
+ def __index__(self):
+ trace.append("index")
+ return 42
+
+ class Floating(container):
+ def __float__(self):
+ trace.append("float")
+ return 1.25
+
+ class Iterated(container):
+ def __iter__(self):
+ raise AssertionError("existing built-in iterator must remain in
use")
+
+ cases = [(Indexed([1, 2]), T.LongType(), 42),
+ (Floating([1, 2]), T.DoubleType(), 1.25),
+ (Iterated([1, 2]), T.ArrayType(T.LongType()), [1, 2])]
+ for value, dtype, expected in cases:
+ schema = T.StructType([T.StructField("value", dtype)])
+ table = container_input_capture_input([(value,)], schema)
+ assert table.column(0)[0].as_py() == expected
+ assert trace == ["index", "float"]
+
+
+def
test_container_input_recursive_leaf_and_row_types(container_input_capture_input):
+ from datetime import timezone
+
+ record = ContainerInputRow(
+ flag=True, number=42, ratio=1.25, text="text", blob=b"abc",
+ decimal=ContainerInputDecimal("-12.340"),
+ date=container_input_date(2000, 2, 29),
+ timestamp=container_input_datetime(2020, 1, 1),
+ child=ContainerInputRow(x=1, y="a"), missing=None)
+ record_schema = T.StructType([
+ T.StructField("flag", T.BooleanType()), T.StructField("number",
T.LongType()),
+ T.StructField("ratio", T.DoubleType()), T.StructField("text",
T.StringType()),
+ T.StructField("blob", T.BinaryType()), T.StructField("decimal",
T.DecimalType(18, 3)),
+ T.StructField("date", T.DateType()), T.StructField("timestamp",
T.TimestampType()),
+ T.StructField("child", T.StructType([
+ T.StructField("x", T.LongType()), T.StructField("y",
T.StringType())])),
+ T.StructField("missing", T.LongType()),
+ ])
+ table = container_input_capture_input(
+ [(record,)], T.StructType([T.StructField("value", record_schema)]))
+ result = table.column(0)[0].as_py()
+ expected_timestamp = container_input_datetime.fromtimestamp(
+ record.timestamp.timestamp(), timezone.utc)
+ assert result == {
+ "flag": True, "number": 42, "ratio": 1.25, "text": "text", "blob":
b"abc",
+ "decimal": ContainerInputDecimal("-12.340"), "date":
container_input_date(2000, 2, 29),
+ "timestamp": expected_timestamp, "child": {"x": 1, "y": "a"},
"missing": None,
+ }
+ assert type(result["flag"]) is bool
+ assert type(result["decimal"]) is ContainerInputDecimal
+
+
+def _container_input_fields(data):
+ """Decode only the small protobuf envelope around a local Arrow
relation."""
+ def varint(pos):
+ value, shift = 0, 0
+ while True:
+ byte = data[pos]
+ pos += 1
+ value |= (byte & 127) << shift
+ if byte < 128:
+ return value, pos
+ shift += 7
+
+ result, pos = {}, 0
+ while pos < len(data):
+ tag, pos = varint(pos)
+ wire = tag & 7
+ if wire == 2:
+ length, pos = varint(pos)
+ result[tag >> 3] = data[pos:pos + length]
+ pos += length
+ elif wire == 0:
+ _, pos = varint(pos)
+ elif wire in (1, 5):
+ pos += 8 if wire == 1 else 4
+ else:
+ raise AssertionError(f"unexpected protobuf wire type: {wire}")
+ return result
+
+
+def _container_input_capture_server(connection):
+ from concurrent.futures import ThreadPoolExecutor
+ import grpc
+
+ captured = []
+
+ def execute(request, context):
+ # ExecutePlanRequest.plan=3 -> Plan.root=1. Names/pandas input
+ # uses Relation.to_df=18 -> ToDF.input=1 before local_relation=11.
+ relation = _container_input_fields(
+ _container_input_fields(_container_input_fields(request)[3])[1])
+ if 18 in relation:
+ relation =
_container_input_fields(_container_input_fields(relation[18])[1])
+ assert 11 in relation, relation
+ captured.append(_container_input_fields(relation[11])[1])
+ context.abort(grpc.StatusCode.INVALID_ARGUMENT, "captured container
input")
+ yield b"" # Make this a streaming handler.
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ server = grpc.server(executor)
+ server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler(
+ "spark.connect.SparkConnectService", {
+ "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute),
+ }),))
+ port = server.add_insecure_port("127.0.0.1:0")
+ server.start()
+ connection.send(port)
+ try:
+ while True:
+ command = connection.recv()
+ if command == "stop":
+ break
+ assert command == "get"
+ connection.send(captured.pop(0))
+ finally:
+ server.stop(0).wait()
+
+
[email protected]
+def container_input_capture_input():
+ import multiprocessing
+ import sys
+ from pathlib import Path
+
+ # importlib-mode pytest does not add the tests namespace to sys.path.
+ # Spawned children need it to import this server target; append it so the
+ # installed wheel still takes precedence over the source pyspark package.
+ test_container_input_root = str(Path(__file__).resolve().parents[1])
+ if test_container_input_root not in sys.path:
+ sys.path.append(test_container_input_root)
+ pytest.importorskip("grpc")
+ from pyspark.sql import SparkSession
+
+ context = multiprocessing.get_context("spawn")
+ parent, child = context.Pipe()
+ process = context.Process(target=_container_input_capture_server,
args=(child,))
+ process.start()
+ try:
+ assert parent.poll(20), "capture service failed to start"
+ spark =
SparkSession.builder.remote(f"sc://127.0.0.1:{parent.recv()}").getOrCreate()
+
+ def capture(data, schema):
+ import pyarrow as pa
+ pd = pytest.importorskip("pandas")
+ # The optimized converter remains the pandas/generic path. Local
+ # rows use infer.convert, with deliberately different container
+ # eligibility and numeric-protocol semantics.
+ if not isinstance(data, pd.DataFrame):
+ data = pd.DataFrame(data, dtype=object)
+ df = spark.createDataFrame(data, schema=schema)
+ with pytest.raises(Exception, match="captured container input"):
+ df.collect()
+ parent.send("get")
+ assert parent.poll(10), "capture service failed to return the IPC
payload"
+ return pa.ipc.open_stream(parent.recv()).read_all()
+
+ yield capture
+ finally:
+ if process.is_alive():
+ parent.send("stop")
+ process.join(10)
+ if process.is_alive():
+ process.terminate()
+ process.join(10)
+ parent.close()
+ child.close()
+
+
[email protected]("source", ["pandas_from_rows", "pandas"])
[email protected]("schema_kind", ["inferred", "names", "explicit",
"ddl"])
[email protected]("container", [list, tuple])
+def test_container_input_array_input_ipc(container_input_capture_input,
source, schema_kind, container):
+ import pyarrow as pa
+ from pyspark.sql import types as T
+
+ values = [container([1, None, -3]), container(), None, container([255,
256])]
+ schema = {"inferred": None, "names": ["items"], "ddl": "items
array<bigint>",
+ "explicit": T.StructType([T.StructField("items",
T.ArrayType(T.LongType()))])}[schema_kind]
+ data = [(value,) for value in values]
+ if source == "pandas":
+ pd = pytest.importorskip("pandas")
+ data = pd.DataFrame(data, columns=["items"])
+ table = container_input_capture_input(data, schema)
+ assert table.column(0).type == pa.list_(pa.int64())
+ assert table.column(0).to_pylist() == [[1, None, -3], [], None, [255, 256]]
+
+
+def
test_container_input_nested_array_struct_binary_ipc(container_input_capture_input):
+ from pyspark.sql import types as T
+
+ schema = T.StructType([
+ T.StructField("nested", T.ArrayType(T.ArrayType(T.LongType()))),
+ T.StructField("record", T.StructType([
+ T.StructField("x", T.LongType()), T.StructField("y",
T.StringType())])),
+ T.StructField("blob", T.BinaryType()),
+ ])
+ table = container_input_capture_input([([[1, None], (), None],
ContainerInputRow(x=7, y="row"), b"abc")], schema)
+ assert table.to_pylist() == [{"nested": [[1, None], [], None],
+ "record": {"x": 7, "y": "row"}, "blob":
b"abc"}]
+
+
# ---------------------------------------------------------------------------
schema cache
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]