This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 780fb20 [SPARK-60699][CONNECT] Build Arrow columns directly for exact
integer tuples
780fb20 is described below
commit 780fb20f9cd600f8dfeb8d482f1cf5917c866d27
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 17:12:54 2026 +0900
[SPARK-60699][CONNECT] Build Arrow columns directly for exact integer tuples
### What changes were proposed in this pull request?
`createDataFrame(data, schema)`, for a `list` of exact `tuple`s of exact
`int`s (and `None`) with a `StructType` schema of `LongType` fields only, now
builds the Arrow `Int64` columns directly from the Python values, instead of
converting each value to a `Value` row and encoding the rows. The new
`SparkSession::create_dataframe_from_columns` of the Rust client sends the
columns as the same inline local relation that `create_dataframe` sends for
rows; `rows_to_arrow_ipc` and it share t [...]
Everything else keeps the conversion of local rows, with its validation and
errors in the same order: other schemas (DDL strings, inferred ones, other
field types), other containers (lists, dicts, `Row`s, subclasses of `list` and
`tuple`), other values (`bool`, subclasses of `int`, floats, integers outside
the int64 range), rows of another width, and `None` in a non-nullable field.
### Why are the changes needed?
[SPARK-60699](https://issues.apache.org/jira/browse/SPARK-60699): integer
tuples are a common local input, and building a row of `Value`s for each tuple
was the main cost of `createDataFrame` for them. Measured on macOS against a
local Spark 4.3.0 server, as the client CPU time of `createDataFrame` (median
of 7, the same in two rounds):
| Input | Master | This change |
| --- | ---: | ---: |
| 200,000 tuples of 8 ints | 270 ms | 36 ms (-87%) |
| 20,000 tuples of 3 ints | 4.9 ms | 1.3 ms (-73%) |
### Does this PR introduce _any_ user-facing change?
No, other than speed: the same IPC reaches the server. For the Rust client,
`SparkSession::create_dataframe_from_columns` is new.
### How was this patch tested?
- A new offline fixture, `python/tests/integer_tuple_cases.py`, run by a
new offline test, checks against a mock Connect server the Arrow IPC of 1, 3
and 32 columns (duplicate, Unicode and NUL names, field metadata, nullable and
non-nullable, the int64 bounds), empty and all-null input, inputs that keep the
conversion (mixed containers, subclasses, pandas, other schemas, inference),
the conversion of floats, the ignored-argument warnings, and the errors (width,
`bool`, overflow, `None [...]
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (775 passed), the build of all tests
with `wasm-udf`, the offline Python suites (798 passed, 9 skipped), the live
drop-in and Arrow suites (78 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master, the differential
test and the measurement by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #135 from dbtsai/codex/spark-60699-direct-integer-tuples.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/session.rs | 76 ++++++++++
crates/spark-connect/src/session.rs | 36 ++++-
python/tests/integer_tuple_cases.py | 274 ++++++++++++++++++++++++++++++++++++
python/tests/test_dropin_offline.py | 15 ++
4 files changed, 397 insertions(+), 4 deletions(-)
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index 6d2ba32..e305851 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -814,6 +814,28 @@ impl PySparkSession {
if is_loaded_instance(py, data, "pandas", "DataFrame")? {
return self.create_from_pandas(py, data, spec);
}
+ // Preserve schema parsing, ignored-argument warnings, and
pandas/Arrow routing.
+ // Only a native StructType opts into this explicit integer
optimization;
+ // DDL, inferred schemas and unsupported values keep schema-driven
conversion.
+ if schema.is_some_and(|s|
s.extract::<PyRef<crate::types::PyStructType>>().is_ok()) {
+ if let Spec::Struct(fields) = &spec {
+ if let Ok(data) = data.cast::<PyList>() {
+ if let Some(columns) = exact_long_tuple_columns(data,
fields) {
+ let schema = DataType::Struct {
+ fields: fields.clone(),
+ };
+ let df = py
+ .detach(|| {
+ self.session
+ .create_dataframe_from_columns(columns,
schema)
+ .map_err(Box::new)
+ })
+ .to_pyerr()?;
+ return Ok(PyDataFrame::new(df));
+ }
+ }
+ }
+ }
self.create_from_local(py, data, spec)
}
@@ -1434,6 +1456,60 @@ fn value_to_datatype(v: &Value) -> DataType {
}
}
+/// Build columns only for exact tuples of exact signed integers and None.
+/// Unsupported containers, values, overflow, widths and non-nullable nulls use
+/// schema-driven local conversion, preserving its validation and error
ordering.
+#[inline(never)]
+fn exact_long_tuple_columns(
+ data: &Bound<'_, PyList>,
+ fields: &[StructField],
+) -> Option<Vec<arrow::array::ArrayRef>> {
+ use arrow::array::Int64Builder;
+ use pyo3::types::{PyInt, PyTuple};
+ use std::sync::Arc;
+
+ if data.cast_exact::<PyList>().is_err()
+ || data.is_empty()
+ || fields.is_empty()
+ || fields.iter().any(|field| field.data_type != DataType::Long)
+ {
+ return None;
+ }
+ let mut builders: Vec<_> = fields
+ .iter()
+ .map(|_| Int64Builder::with_capacity(data.len()))
+ .collect();
+ for item in data.iter() {
+ let Ok(tuple) = item.cast_exact::<PyTuple>() else {
+ return None;
+ };
+ if tuple.len() != builders.len() {
+ return None;
+ }
+ for ((value, builder), field) in tuple.iter().zip(&mut
builders).zip(fields) {
+ if value.is_none() {
+ if !field.nullable {
+ return None;
+ }
+ builder.append_null();
+ } else if let Ok(integer) = value.cast_exact::<PyInt>() {
+ let Ok(integer) = integer.extract::<i64>() else {
+ return None;
+ };
+ builder.append_value(integer);
+ } else {
+ return None;
+ }
+ }
+ }
+ Some(
+ builders
+ .iter_mut()
+ .map(|builder| Arc::new(builder.finish()) as
arrow::array::ArrayRef)
+ .collect(),
+ )
+}
+
/// Convert a Python value to a Rust Value.
pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) -> PyResult<Value> {
// Check for None
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 1b3601d..136fa55 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -394,6 +394,25 @@ impl SparkSession {
Ok(DataFrame::new(self.clone(), plan))
}
+ /// Create an inline local relation from columns with the same encoding as
local rows.
+ ///
+ /// Unlike imported Arrow input, this path does not read server
configuration or
+ /// upload artifacts. Callers must supply one column per struct field.
+ pub fn create_dataframe_from_columns(
+ &self,
+ columns: Vec<arrow::array::ArrayRef>,
+ schema: DataType,
+ ) -> Result<DataFrame> {
+ let arrow_schema =
arrow::datatypes::Schema::new(schema_to_arrow_fields(&schema)?);
+ let row_count = columns.first().map_or(0, |column| column.len());
+ let data = Some(columns_to_arrow_ipc(columns, arrow_schema,
row_count)?);
+ let plan = LogicalPlan::LocalRelation {
+ schema: Some(schema),
+ data,
+ };
+ Ok(DataFrame::new(self.clone(), plan))
+ }
+
/// Create a local DataFrame from Arrow batches without materializing rows.
///
/// `schema` optionally supplies the Spark schema applied by the server.
With
@@ -795,8 +814,6 @@ fn row_width_mismatch(row_index: usize, actual: usize,
expected: usize) -> Spark
/// Used by createDataFrame to package local data.
fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) -> Result<Vec<u8>> {
use arrow::datatypes::Schema as ArrowSchema;
- use arrow::ipc::writer::StreamWriter;
- use arrow::record_batch::RecordBatch;
if rows.is_empty() {
return Ok(vec![]);
@@ -828,8 +845,19 @@ fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) ->
Result<Vec<u8>> {
columns.push(values_to_arrow(&column, &field.data_type)?);
}
- // Create RecordBatch, with the number of rows, which a batch without
columns needs.
- let options =
arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(rows.len()));
+ columns_to_arrow_ipc(columns, arrow_schema, rows.len())
+}
+
+fn columns_to_arrow_ipc(
+ columns: Vec<arrow::array::ArrayRef>,
+ arrow_schema: arrow::datatypes::Schema,
+ row_count: usize,
+) -> Result<Vec<u8>> {
+ use arrow::ipc::writer::StreamWriter;
+ use arrow::record_batch::RecordBatch;
+
+ // Preserve the row count even for a batch without columns.
+ let options =
arrow::record_batch::RecordBatchOptions::new().with_row_count(Some(row_count));
let batch = RecordBatch::try_new_with_options(Arc::new(arrow_schema),
columns, &options)
.map_err(|e| SparkError::connect_msg(format!("Failed to create Arrow
batch: {}", e)))?;
diff --git a/python/tests/integer_tuple_cases.py
b/python/tests/integer_tuple_cases.py
new file mode 100644
index 0000000..184d4dc
--- /dev/null
+++ b/python/tests/integer_tuple_cases.py
@@ -0,0 +1,274 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Check integer tuple input and complete IPC values without a Spark server or
JVM."""
+
+from concurrent.futures import ThreadPoolExecutor
+import json
+import unittest
+
+try:
+ import grpc
+ import pandas as pd
+ import pyarrow as pa
+except ImportError:
+ grpc = pd = pa = None
+
+from pyspark.sql import Row, SparkSession, types as T
+
+
+def _varint(value):
+ out = bytearray()
+ while value > 127:
+ out.append((value & 127) | 128)
+ value >>= 7
+ out.append(value)
+ return bytes(out)
+
+
+def _bytes(number, value):
+ return _varint(number * 8 + 2) + _varint(len(value)) + value
+
+
+def _fields(data):
+ """Read the varint/bytes envelopes used by the fixture's protobuf
messages."""
+ def read(pos):
+ value = shift = 0
+ while True:
+ byte = data[pos]
+ pos += 1
+ value |= (byte & 127) << shift
+ if byte < 128:
+ return value, pos
+ shift += 7
+
+ pos = 0
+ result = {}
+ while pos < len(data):
+ tag, pos = read(pos)
+ wire = tag & 7
+ if wire == 2:
+ size, pos = read(pos)
+ result.setdefault(tag >> 3, []).append(data[pos:pos + size])
+ pos += size
+ elif wire == 0:
+ _, pos = read(pos)
+ else:
+ raise AssertionError(f"Unexpected protobuf wire type: {wire}")
+ return result
+
+
[email protected](grpc is None or pa is None, "grpcio, pandas and pyarrow are
required")
+class IntegerTupleTests(unittest.TestCase):
+ def setUp(self):
+ self.captured = []
+ self.config_calls = 0
+
+ def execute(raw, context):
+ # ExecutePlanRequest.plan(3) -> Plan.root(1) ->
Relation.local_relation(11).
+ request = _fields(raw)
+ root = _fields(_fields(request[3][0])[1][0])
+ while 18 in root: # Relation.to_df -> ToDF.input.
+ root = _fields(_fields(root[18][0])[1][0])
+ self.captured.append(_fields(root[11][0]))
+ yield (_bytes(1, request[1][0]) + _bytes(15, b"input-name-fixture")
+ + _bytes(12, request[6][0]) + _bytes(13, b"complete")
+ + _bytes(14, b""))
+
+ def config(raw, context):
+ self.config_calls += 1
+ request = _fields(raw)
+ operation = _fields(request[3][0])
+ if 2 in operation: # ConfigRequest.Get.keys(1).
+ keys = _fields(operation[2][0])[1]
+ else:
+ self.assertEqual(set(operation), {3}) #
GetWithDefault.pairs(1).
+ keys = [_fields(pair)[1][0] for pair in
_fields(operation[3][0])[1]]
+ values = {
+ b"spark.sql.session.localRelationCacheThreshold": str(1 <<
60).encode(),
+ b"spark.sql.connect.serverStacktrace.enabled": b"false",
+ b"spark.sql.pyspark.jvmStacktrace.enabled": b"false",
+ }
+ pairs = b"".join(_bytes(2, _bytes(1, key) + _bytes(2, values[key]))
+ for key in keys)
+ return _bytes(1, request[1][0]) + _bytes(4, b"input-name-fixture")
+ pairs
+
+ def release(raw, context):
+ return _bytes(1, _fields(raw)[1][0])
+
+ pool = ThreadPoolExecutor(max_workers=2)
+ self.addCleanup(pool.shutdown)
+ server = grpc.server(pool)
+ server.add_generic_rpc_handlers([grpc.method_handlers_generic_handler(
+ "spark.connect.SparkConnectService", {
+ "ExecutePlan": grpc.unary_stream_rpc_method_handler(execute),
+ "Config": grpc.unary_unary_rpc_method_handler(config),
+ "ReleaseExecute": grpc.unary_unary_rpc_method_handler(release),
+ "ReleaseSession": grpc.unary_unary_rpc_method_handler(release),
+ }
+ )])
+ port = server.add_insecure_port("127.0.0.1:0")
+ self.assertNotEqual(port, 0)
+ server.start()
+ self.addCleanup(lambda: server.stop(0).wait())
+ self.spark =
SparkSession.builder.remote(f"sc://127.0.0.1:{port}").create()
+ self.addCleanup(self.spark.stop)
+
+
+ def capture(self, data, schema):
+ before = len(self.captured)
+ config_before = self.config_calls
+ frame = self.spark.createDataFrame(data, schema)
+ self.assertEqual(len(self.captured), before) # construction remains
offline
+ self.assertEqual(self.config_calls, config_before)
+ frame.toArrow()
+ self.assertEqual(len(self.captured), before + 1)
+ self.assertEqual(self.config_calls, config_before)
+ local = self.captured[-1]
+ self.assertEqual(json.loads(local[2][0]), json.loads(schema.json()))
+ if 1 not in local:
+ self.assertEqual(len(data), 0)
+ return None
+ table = pa.ipc.open_stream(local[1][0]).read_all()
+ table.validate(full=True)
+ return table
+
+ def test_complete_columns_schema_and_metadata(self):
+ for width in (1, 3, 32):
+ for nullable in (False, True):
+ names = [f"雪{j}" for j in range(width)]
+ if width == 3:
+ names = ["duplicate", "duplicate", "embedded\x00nul"]
+ schema = T.StructType([T.StructField(name, T.LongType(),
nullable,
+ {"index": j, "nested": {"v": "raw"}}) for j, name in
enumerate(names)])
+ data = [tuple(None if nullable and (i+j) % 5 == 0 else
+ (-(1 << 63) if i == 0 else (1 << 63)-1 if i == 1 else
i*width+j)
+ for j in range(width)) for i in range(61)]
+ with self.subTest(width=width, nullable=nullable):
+ table = self.capture(data, schema)
+ expected = pa.Table.from_arrays([
+ pa.array([row[j] for row in data], type=pa.int64())
+ for j in range(width)], schema=pa.schema([
+ pa.field(name, pa.int64(), nullable=nullable) for
name in names]))
+ self.assertEqual(table.schema, expected.schema)
+ self.assertTrue(table.equals(expected))
+
+ def test_direct_columns_equal_the_schema_driven_conversion(self):
+ # A subclass of tuple keeps a row on the schema-driven conversion, so
the same data
+ # in exact tuples (the direct columns) and in that subclass (the
conversion of local
+ # rows) must send the same Arrow IPC bytes.
+ class TupleSubclass(tuple):
+ pass
+ for width, nullable in ((1, True), (3, False), (32, True)):
+ schema = T.StructType([T.StructField(f"c{j}", T.LongType(),
nullable,
+ {"index": j}) for j in
range(width)])
+ data = [tuple(None if nullable and (i + j) % 4 == 0 else
+ (-(1 << 63) if i == 0 else (1 << 63) - 1 if i == 1
else i - j)
+ for j in range(width)) for i in range(50)]
+ with self.subTest(width=width, nullable=nullable):
+ self.capture(data, schema)
+ direct = self.captured[-1][1][0]
+ self.capture([TupleSubclass(row) for row in data], schema)
+ converted = self.captured[-1][1][0]
+ self.assertEqual(direct, converted)
+
+ def test_empty_and_all_null(self):
+ schema = T.StructType([T.StructField("v", T.LongType())])
+ self.assertIsNone(self.capture([], schema))
+ self.assertEqual(self.capture([(None,), (None,)],
schema).column(0).to_pylist(),
+ [None, None])
+ # Empty and zero-column input retain current local-input semantics.
+ empty_schema = T.StructType([])
+ self.assertIsNone(self.capture([], empty_schema))
+ zero_columns = self.capture([(), (), ()], empty_schema)
+ self.assertEqual(zero_columns.num_columns, 0)
+ self.assertEqual(zero_columns.num_rows, 3)
+
+ def test_mixed_containers_and_subclasses_fall_back(self):
+ class IntSubclass(int):
+ pass
+ class TupleSubclass(tuple):
+ pass
+ class ListSubclass(list):
+ pass
+ schema = T.StructType([T.StructField("v", T.LongType())])
+ for data in ([(1,), [2], {"v": 3}, Row(v=4)],
+ [(1,), TupleSubclass((2,)), (IntSubclass(3),)],
+ ListSubclass([(1,), (2,), (3,)]),
+ pd.DataFrame({"v": [1, 2, 3]})):
+ with self.subTest(kind=type(data).__name__):
+ table = self.capture(data, schema)
+ self.assertEqual(table.column(0).to_pylist(), list(range(1,
len(data)+1)))
+
+ def test_other_schemas_and_inference(self):
+ cases = [(T.DoubleType(), [(1.5,), (None,)], pa.float64()),
+ (T.StringType(), [("a",), (None,)], pa.string()),
+ (T.BooleanType(), [(True,), (False,)], pa.bool_()),
+ (T.IntegerType(), [(1,), (None,)], pa.int32())]
+ for dtype, rows, arrow_type in cases:
+ schema = T.StructType([T.StructField("v", dtype)])
+ table = self.capture(rows, schema)
+ self.assertEqual(table.column(0).type, arrow_type)
+ self.assertEqual(table.column(0).to_pylist(), [r[0] for r in rows])
+ for schema in (None, ["v"]):
+ frame = self.spark.createDataFrame([(1,), (2,)], schema)
+ frame.toArrow()
+ table = pa.ipc.open_stream(self.captured[-1][1][0]).read_all()
+ self.assertEqual(table.column(0).to_pylist(), [1, 2])
+
+ def
test_float_coercion_and_ignored_arguments_use_current_local_behavior(self):
+ schema = T.StructType([T.StructField("v", T.LongType())])
+ table = self.capture([(1.9,), (-2.9,)], schema)
+ self.assertEqual(table.column(0).to_pylist(), [1, -2])
+ for name in ("samplingRatio", "verifySchema"):
+ with self.subTest(name=name), self.assertWarnsRegex(UserWarning,
f"'{name}' is ignored"):
+ self.spark.createDataFrame([(1,)], schema, **{name: True})
+
+ def test_existing_errors_and_row_indices(self):
+ schema = T.StructType([T.StructField("v", T.LongType())])
+ from pyspark.errors import PySparkValueError
+ for data, width in (([(1,), ()], "0"), ([(1,), (2, 3)], "2")):
+ with self.subTest(data=data), self.assertRaises(PySparkValueError)
as caught:
+ self.spark.createDataFrame(data, schema)
+ self.assertEqual(caught.exception.getCondition(),
"AXIS_LENGTH_MISMATCH")
+ self.assertEqual(caught.exception.getMessageParameters(),
+ {"expected_length": "1", "actual_length": width})
+ # Fallback uses current schema-driven conversion, which rejects bool,
+ # reports integer overflow, and enforces nullability before IPC
creation.
+ with self.assertRaisesRegex(TypeError, "Expected integer, got bool"):
+ self.spark.createDataFrame([(1,), (True,)], schema)
+ for value in (1 << 63, -(1 << 63)-1):
+ with self.subTest(value=value), self.assertRaises(OverflowError):
+ self.spark.createDataFrame([(1,), (value,)], schema)
+ required = T.StructType([T.StructField("v", T.LongType(), False)])
+ with self.assertRaisesRegex(PySparkValueError, "must not be None"):
+ self.spark.createDataFrame([(1,), (None,)], required)
+ # Every row's width is checked before cells are converted. A
speculative
+ # integer scan must defer these failures to the ordinary local path.
+ for data, selected_schema in (([(True,), ()], schema),
+ ([(None,), ()], required)):
+ with self.subTest(data=data), self.assertRaises(PySparkValueError)
as caught:
+ self.spark.createDataFrame(data, selected_schema)
+ self.assertEqual(caught.exception.getCondition(),
"AXIS_LENGTH_MISMATCH")
+ self.assertEqual(caught.exception.getMessageParameters(),
+ {"expected_length": "1", "actual_length": "0"})
+ self.assertEqual(self.captured, [])
+ self.assertEqual(self.config_calls, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index a8dd237..8ee2728 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -47,6 +47,21 @@ def test_create_dataframe_input_names_offline():
assert result.returncode == 0, result.stdout + result.stderr
+def test_create_dataframe_integer_tuples_offline():
+ # Isolate the mock fixture's active session from the rest of the offline
suite.
+ import subprocess
+ import sys
+ from pathlib import Path
+
+ for dependency in ("grpc", "pandas", "pyarrow"):
+ pytest.importorskip(dependency)
+ result = subprocess.run(
+ [sys.executable,
str(Path(__file__).with_name("integer_tuple_cases.py"))],
+ capture_output=True, text=True,
+ )
+ assert result.returncode == 0, result.stdout + result.stderr
+
+
# ---------------------------------------------------------------------------
types
ATOMIC = [
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]