This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 29d322d [SPARK-60640][CONNECT] Borrow local values during Arrow input
encoding
29d322d is described below
commit 29d322d52d46b2c5f44f0d392ea41594e1083346
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 18:26:51 2026 +0900
[SPARK-60640][CONNECT] Borrow local values during Arrow input encoding
### What changes were proposed in this pull request?
The Arrow encoder of local rows (`rows_to_arrow_ipc` and `values_to_arrow`
in the Rust client) now borrows the `Value`s of each column and of nested
children (`&[&Value]`), including null placeholders and the values that struct
fields look up by name or position, instead of cloning each column, list, map
and struct of the input rows before building the Arrow arrays.
Numeric conversions, duplicate-name lookup, nullability, schema validation,
the null map key check and the order of errors are unchanged. Arrow buffers and
the IPC serialization still allocate and copy; this is not zero-copy.
### Why are the changes needed?
[SPARK-60640](https://issues.apache.org/jira/browse/SPARK-60640): the
clones copied every string, binary value and nested container of the input once
more before encoding. Measured on macOS against a local Spark 4.3.0 server, as
the client CPU time of `createDataFrame` of local rows (median of 5, two
rounds):
| Input | Master | This change |
| --- | ---: | ---: |
| 200,000 rows of 4 strings | 146-154 ms | 115-119 ms (-22%) |
| 100,000 rows of an array, a map and a struct of strings | 203-206 ms |
138-139 ms (-33%) |
### Does this PR introduce _any_ user-facing change?
No, other than speed.
### How was this patch tested?
- New unit tests round-trip nested text and binary values (Unicode, NUL,
empty, null) through the IPC, and check the order of the errors of nested
arrays and maps.
- In the merge with master, master's check that a map key is not null is
kept in the borrowing map arm, and its test passes borrowed values.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (785 passed), the build of all tests
with `wasm-udf`, the offline Python suites (886 passed, 9 skipped), the live
drop-in and Arrow suites (81 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master and the
measurement by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #114 from dbtsai/codex/spark-60640-borrowed-arrow.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/session.rs | 190 ++++++++++++++++++++++++++++++------
1 file changed, 158 insertions(+), 32 deletions(-)
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 136fa55..761288f 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -838,10 +838,7 @@ fn rows_to_arrow_ipc(rows: &[Row], schema: &DataType) ->
Result<Vec<u8>> {
}
for (field_idx, field) in fields.iter().enumerate() {
- let column: Vec<Value> = rows
- .iter()
- .map(|row| row.values()[field_idx].clone())
- .collect();
+ let column: Vec<&Value> = rows.iter().map(|row|
&row.values()[field_idx]).collect();
columns.push(values_to_arrow(&column, &field.data_type)?);
}
@@ -974,8 +971,9 @@ fn datatype_to_arrow(dt: &DataType) ->
Result<arrow::datatypes::DataType> {
}
/// Recursively build an Arrow array from a column of `Value`s against a
declared
-/// `DataType`, covering scalars and the nested array/map/struct types.
-fn values_to_arrow(values: &[Value], dt: &DataType) -> Result<Arc<dyn
arrow::array::Array>> {
+/// `DataType`, covering scalars and the nested array/map/struct types.
Temporary
+/// columns borrow their values; building the Arrow output still copies
payloads.
+fn values_to_arrow(values: &[&Value], dt: &DataType) -> Result<Arc<dyn
arrow::array::Array>> {
use arrow::array::*;
use arrow::buffer::{NullBuffer, OffsetBuffer};
use arrow::datatypes::{Field, Fields};
@@ -984,7 +982,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
($arr:ty, $pat:path) => {{
let vals: Result<Vec<_>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
$pat(x) => Ok(Some(*x)),
_ => Err(SparkError::connect_msg("Type mismatch in row
data")),
@@ -1001,7 +999,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
($arr:ty, $t:ty) => {{
let vals: Result<Vec<_>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::Byte(x) => Ok(Some(*x as $t)),
Value::Short(x) => Ok(Some(*x as $t)),
@@ -1017,7 +1015,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
($arr:ty, $t:ty) => {{
let vals: Result<Vec<_>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::Byte(x) => Ok(Some(*x as $t)),
Value::Short(x) => Ok(Some(*x as $t)),
@@ -1044,7 +1042,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
DataType::String { .. } | DataType::Char { .. } | DataType::Varchar {
.. } => {
let vals: Result<Vec<Option<&str>>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::String(s) => Ok(Some(s.as_str())),
_ => Err(SparkError::connect_msg("Type mismatch in row
data")),
@@ -1055,7 +1053,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
DataType::Binary => {
let vals: Result<Vec<Option<&[u8]>>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::Binary(b) => Ok(Some(b.as_slice())),
_ => Err(SparkError::connect_msg("Type mismatch in row
data")),
@@ -1068,7 +1066,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
DataType::Timestamp | DataType::TimestampNtz => {
let vals: Result<Vec<Option<i64>>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::Timestamp(t) => Ok(Some(*t)),
_ => Err(SparkError::connect_msg("Type mismatch in row
data")),
@@ -1087,7 +1085,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
DataType::TimestampNtzNanos { .. } | DataType::TimestampLtzNanos { ..
} => {
let vals: Result<Vec<Option<i64>>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::Timestamp(t) =>
t.checked_mul(1_000).map(Some).ok_or_else(|| {
SparkError::connect_msg("Timestamp out of range for
nanosecond precision")
@@ -1107,7 +1105,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
let col_scale = *scale;
let vals: Result<Vec<Option<i128>>> = values
.iter()
- .map(|v| match v {
+ .map(|v| match *v {
Value::Null => Ok(None),
Value::Decimal { value, .. } => {
decimal_str_to_unscaled(value, col_scale).map(Some)
@@ -1126,18 +1124,18 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
element_type,
contains_null,
} => {
- let mut child: Vec<Value> = Vec::new();
+ let mut child: Vec<&Value> = Vec::new();
let mut offsets: Vec<i32> = vec![0];
let mut valid =
arrow::array::builder::BooleanBufferBuilder::new(values.len());
for v in values {
- match v {
+ match *v {
Value::Null => {
valid.append(false);
offsets.push(*offsets.last().unwrap());
}
Value::List(items) => {
valid.append(true);
- child.extend(items.iter().cloned());
+ child.extend(items.iter());
offsets.push(child.len() as i32);
}
_ => return Err(SparkError::connect_msg("Type mismatch:
expected array")),
@@ -1157,14 +1155,14 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
)))
}
DataType::Struct { fields } => {
- let mut cols: Vec<Vec<Value>> =
vec![Vec::with_capacity(values.len()); fields.len()];
+ let mut cols: Vec<Vec<&Value>> =
vec![Vec::with_capacity(values.len()); fields.len()];
let mut valid =
arrow::array::builder::BooleanBufferBuilder::new(values.len());
for v in values {
- match v {
+ match *v {
Value::Null => {
valid.append(false);
for c in cols.iter_mut() {
- c.push(Value::Null);
+ c.push(&Value::Null);
}
}
Value::Struct(sf) => {
@@ -1181,7 +1179,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
} else {
sf.get_by_name(&f.name).or_else(|| sf.get(i))
};
- cols[i].push(val.cloned().unwrap_or(Value::Null));
+ cols[i].push(val.unwrap_or(&Value::Null));
}
}
// A Python dict supplied for a struct field (e.g.
array<struct> data)
@@ -1190,10 +1188,10 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
valid.append(true);
for (i, f) in fields.iter().enumerate() {
let value = m.iter().find_map(|(key, value)| match
key {
- Value::String(key) if *key == f.name =>
Some(value.clone()),
+ Value::String(key) if *key == f.name =>
Some(value),
_ => None,
});
- cols[i].push(value.unwrap_or(Value::Null));
+ cols[i].push(value.unwrap_or(&Value::Null));
}
}
_ => return Err(SparkError::connect_msg("Type mismatch:
expected struct")),
@@ -1230,12 +1228,12 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
value_type,
value_contains_null,
} => {
- let mut keys: Vec<Value> = Vec::new();
- let mut vals: Vec<Value> = Vec::new();
+ let mut keys: Vec<&Value> = Vec::new();
+ let mut vals: Vec<&Value> = Vec::new();
let mut offsets: Vec<i32> = vec![0];
let mut valid =
arrow::array::builder::BooleanBufferBuilder::new(values.len());
for v in values {
- match v {
+ match *v {
Value::Null => {
valid.append(false);
offsets.push(*offsets.last().unwrap());
@@ -1246,8 +1244,8 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
if matches!(k, Value::Null) {
return Err(SparkError::connect_msg("A map key
cannot be null"));
}
- keys.push(k.clone());
- vals.push(val.clone());
+ keys.push(k);
+ vals.push(val);
}
offsets.push(keys.len() as i32);
}
@@ -1678,7 +1676,7 @@ mod tests {
Value::Null,
Value::Struct(Row::new(vec!["dup".to_string()],
vec![Value::Null]).into()),
];
- let array = values_to_arrow(&values, &target).unwrap();
+ let array = values_to_arrow(&values.iter().collect::<Vec<_>>(),
&target).unwrap();
let array = array.as_any().downcast_ref::<StructArray>().unwrap();
assert_eq!(array.len(), 4);
assert!(!array.is_null(0));
@@ -2256,6 +2254,131 @@ mod tests {
}
}
+ #[test]
+ fn input_ipc_round_trips_nested_text_and_binary() {
+ let list = |element_type| DataType::Array {
+ element_type: Box::new(element_type),
+ contains_null: true,
+ };
+ let map = |value_type| DataType::Map {
+ key_type: Box::new(text_type()),
+ value_type: Box::new(value_type),
+ value_contains_null: true,
+ };
+ let nested_type = DataType::Struct {
+ fields: vec![
+ struct_field("binary", list(DataType::Binary)),
+ struct_field("text", map(text_type())),
+ ],
+ };
+ let schema = schema(vec![list(text_type()), map(DataType::Binary),
nested_type]);
+ let make_rows = || {
+ vec![
+ row(vec![
+ Value::List(vec![
+ Value::String("é\0名前".into()),
+ Value::String(String::new()),
+ Value::Null,
+ ]),
+ Value::Map(
+ [
+ (
+ Value::String("é\0名前".into()),
+ Value::Binary(vec![0, 255, 1]),
+ ),
+ (Value::String(String::new()),
Value::Binary(vec![])),
+ (Value::String("null".into()), Value::Null),
+ ]
+ .into(),
+ ),
+ Value::Struct(
+ Row::new(
+ vec!["binary".into(), "text".into()],
+ vec![
+ Value::List(vec![
+ Value::Binary(vec![0, 255]),
+ Value::Binary(vec![]),
+ Value::Null,
+ ]),
+ Value::Map(
+ [
+ (
+ Value::String("key\0".into()),
+ Value::String("名前\0é".into()),
+ ),
+ (
+ Value::String(String::new()),
+ Value::String(String::new()),
+ ),
+ (Value::String("null".into()),
Value::Null),
+ ]
+ .into(),
+ ),
+ ],
+ )
+ .into(),
+ ),
+ ]),
+ row(vec![
+ Value::List(vec![]),
+ Value::Map(Default::default()),
+ Value::Null,
+ ]),
+ row(vec![Value::Null; 3]),
+ ]
+ };
+ let rows = make_rows();
+ let data = rows_to_arrow_ipc(&rows, &schema).unwrap();
+ drop(rows);
+ // The IPC is an owned buffer: decode it after the rows it was encoded
from are gone.
+ let response = proto::execute_plan_response::ArrowBatch {
+ data: data.into(),
+ ..Default::default()
+ };
+ let decoded = crate::dataframe::decode_arrow_batch(&response).unwrap();
+ // Build the expected rows anew rather than keeping the originals.
+ let expected = make_rows();
+ assert_eq!(decoded.len(), expected.len());
+ for (actual, expected) in decoded.iter().zip(expected) {
+ assert_eq!(actual.values(), expected.values());
+ }
+ }
+
+ #[test]
+ fn input_ipc_preserves_nested_error_order() {
+ let list = |element_type| DataType::Array {
+ element_type: Box::new(element_type),
+ contains_null: true,
+ };
+ // Validate all parent values before recursing into children.
+ let rows = [
+ row(vec![Value::List(vec![Value::String("bad integer".into())])]),
+ row(vec![Value::Long(1)]),
+ ];
+ let error = rows_to_arrow_ipc(&rows,
&schema(vec![list(DataType::Long)]))
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("Type mismatch: expected array"), "{error}");
+
+ // Map key validation precedes map value validation, including nested
schemas.
+ let map_type = DataType::Map {
+ key_type: Box::new(list(text_type())),
+ value_type: Box::new(DataType::Long),
+ value_contains_null: true,
+ };
+ let rows = [row(vec![Value::Map(
+ [(
+ Value::String("key".into()),
+ Value::String("bad integer".into()),
+ )]
+ .into(),
+ )])];
+ let error = rows_to_arrow_ipc(&rows, &schema(vec![map_type]))
+ .unwrap_err()
+ .to_string();
+ assert!(error.contains("Type mismatch: expected array"), "{error}");
+ }
+
#[test]
fn a_null_map_key_is_an_error() {
let map = DataType::Map {
@@ -2264,7 +2387,7 @@ mod tests {
value_contains_null: true,
};
let value = Value::Map(vec![(Value::Null, Value::Long(1))]);
- let error = values_to_arrow(&[value], &map).unwrap_err().to_string();
+ let error = values_to_arrow(&[&value], &map).unwrap_err().to_string();
assert!(error.contains("A map key cannot be null"), "{error}");
}
@@ -2277,7 +2400,7 @@ mod tests {
];
let names = vec!["a".to_string(), "a".to_string()];
let value = Value::Struct(Row::new(names, vec![Value::Long(1),
Value::Long(2)]).into());
- let array = values_to_arrow(&[value], &DataType::Struct { fields
}).unwrap();
+ let array = values_to_arrow(&[&value], &DataType::Struct { fields
}).unwrap();
let array = array.as_any().downcast_ref::<StructArray>().unwrap();
let column = |i: usize| {
let column = array
@@ -2292,7 +2415,10 @@ mod tests {
#[test]
fn struct_values_without_fields_keep_their_length() {
let empty = Value::Struct(Row::empty().into());
- let array = values_to_arrow(&[empty, Value::Null], &DataType::Struct {
fields: vec![] });
+ let array = values_to_arrow(
+ &[&empty, &Value::Null],
+ &DataType::Struct { fields: vec![] },
+ );
let array = array.unwrap();
assert_eq!(array.len(), 2);
assert!(array.is_valid(0));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]