This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 7109a84 [SPARK-60641][CONNECT] Accept Byte and Short inputs for
floating schemas
7109a84 is described below
commit 7109a84a02f5159cc22483d55547e93cb968dd29
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 17:11:57 2026 +0900
[SPARK-60641][CONNECT] Accept Byte and Short inputs for floating schemas
### What changes were proposed in this pull request?
The Arrow encoder of local rows (`create_dataframe` of the Rust client) now
takes `Value::Byte` and `Value::Short` for `FloatType` and `DoubleType` fields,
converting them as it already converts `Value::Integer` and `Value::Long`. This
covers nested leaves too: array elements, map values and struct fields.
### Why are the changes needed?
[SPARK-60641](https://issues.apache.org/jira/browse/SPARK-60641): Rows
collected from `TINYINT` or `SMALLINT` columns hold `Value::Byte` and
`Value::Short`, and creating a DataFrame of them with a `FloatType` or
`DoubleType` schema failed with `Type mismatch in row data`, though the wider
integers work.
### Does this PR introduce _any_ user-facing change?
Yes, for the Rust client: `create_dataframe` takes `Value::Byte` and
`Value::Short` for floating fields, nested ones included. The Python client's
conversion of local data is unchanged.
### How was this patch tested?
- A unit test encodes `Byte` and `Short` minima, maxima, negatives, zero
and nulls into `Float` and `Double` scalars, array elements, map values and
struct fields, reads the IPC back with Arrow, and checks that booleans, strings
and binary values are still rejected.
- A unit test checks the struct field resolution of the encoder, with
duplicate names, a positional field, nested fields and a missing field, and the
integer types it encodes.
- A new e2e test collects `TINYINT` and `SMALLINT` rows from a Spark server
and creates `FloatType` and `DoubleType` DataFrames of them.
- Against the current master, merged into this branch, and a local Spark
4.3.0 server: the lib and every integration test of `apache-spark-connect` and
`apache-spark-connect-core`, e2e included (777 passed), the build of all tests
with `wasm-udf`, the offline Python suites (733 passed, 9 skipped), the live
drop-in and Arrow suites (77 passed) and the live cases (10 passed). `cargo fmt
--all --check` passes.
Not in this change: the encoder narrows an integer that does not fit its
field type with `as` (`Long(257)` becomes `1` in a `ByteType` field), where
PySpark raises `ArrowInvalid`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6); the merge with master and the review
follow-ups by Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #115 from dbtsai/codex/spark-60641.
Lead-authored-by: DB Tsai <[email protected]>
Co-authored-by: Hyukjin Kwon <[email protected]>
Co-authored-by: DB Tsai <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/spark-connect/src/session.rs | 313 +++++++++++++++++++++++++++++
crates/spark-connect/tests/e2e_coverage.rs | 62 ++++++
2 files changed, 375 insertions(+)
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 3b30c55..1b3601d 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -991,6 +991,8 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
.iter()
.map(|v| match v {
Value::Null => Ok(None),
+ Value::Byte(x) => Ok(Some(*x as $t)),
+ Value::Short(x) => Ok(Some(*x as $t)),
Value::Float(x) => Ok(Some(*x as $t)),
Value::Double(x) => Ok(Some(*x as $t)),
Value::Integer(x) => Ok(Some(*x as $t)),
@@ -1310,6 +1312,317 @@ mod tests {
}
}
+ #[test]
+ fn small_integer_rows_encode_as_floating_ipc() {
+ use arrow::array::{Array, Float32Array, Float64Array, ListArray,
MapArray, StructArray};
+ use arrow::ipc::reader::StreamReader;
+ use std::io::Cursor;
+
+ let inputs = vec![
+ Value::Byte(i8::MIN),
+ Value::Byte(-1),
+ Value::Byte(0),
+ Value::Byte(i8::MAX),
+ Value::Short(i16::MIN),
+ Value::Short(-1),
+ Value::Short(0),
+ Value::Short(i16::MAX),
+ Value::Null,
+ ];
+ let expected = vec![
+ Some(-128.0),
+ Some(-1.0),
+ Some(0.0),
+ Some(127.0),
+ Some(-32768.0),
+ Some(-1.0),
+ Some(0.0),
+ Some(32767.0),
+ None,
+ ];
+ for target in [DataType::Float, DataType::Double] {
+ let schema = DataType::Struct {
+ fields: vec![
+ struct_field("scalar", target.clone()),
+ struct_field(
+ "array",
+ DataType::Array {
+ element_type: Box::new(target.clone()),
+ contains_null: true,
+ },
+ ),
+ struct_field(
+ "map",
+ DataType::Map {
+ key_type: Box::new(DataType::String {
+ collation: "UTF8_BINARY".into(),
+ }),
+ value_type: Box::new(target.clone()),
+ value_contains_null: true,
+ },
+ ),
+ struct_field(
+ "struct",
+ DataType::Struct {
+ fields: vec![struct_field("n", target.clone())],
+ },
+ ),
+ ],
+ };
+ let rows: Vec<_> = inputs
+ .iter()
+ .map(|value| {
+ Row::new(
+ vec![
+ "scalar".into(),
+ "array".into(),
+ "map".into(),
+ "struct".into(),
+ ],
+ vec![
+ value.clone(),
+ Value::List(vec![value.clone()]),
+ Value::Map(vec![(Value::String("k".to_string()),
value.clone())]),
+ Value::Struct(Row::new(vec!["n".into()],
vec![value.clone()]).into()),
+ ],
+ )
+ })
+ .collect();
+ let bytes = rows_to_arrow_ipc(&rows, &schema).unwrap();
+ let mut reader = StreamReader::try_new(Cursor::new(bytes),
None).unwrap();
+ let batch = reader.next().unwrap().unwrap();
+ assert_eq!(batch.num_rows(), inputs.len());
+ let array = batch
+ .column(1)
+ .as_any()
+ .downcast_ref::<ListArray>()
+ .unwrap();
+ let map =
batch.column(2).as_any().downcast_ref::<MapArray>().unwrap();
+ let nested = batch
+ .column(3)
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .unwrap();
+ for column in [
+ batch.column(0),
+ array.values(),
+ map.values(),
+ nested.column(0),
+ ] {
+ match target {
+ DataType::Float => assert_eq!(
+
column.as_any().downcast_ref::<Float32Array>().unwrap(),
+ &Float32Array::from(
+ expected
+ .iter()
+ .map(|v| v.map(|n| n as f32))
+ .collect::<Vec<_>>()
+ ),
+ ),
+ DataType::Double => assert_eq!(
+
column.as_any().downcast_ref::<Float64Array>().unwrap(),
+ &Float64Array::from(expected.clone()),
+ ),
+ _ => unreachable!(),
+ }
+ assert_eq!(column.null_count(), 1);
+ }
+ assert!(reader.next().is_none());
+ for invalid in [
+ Value::Bool(true),
+ Value::String("7".into()),
+ Value::Binary(vec![7]),
+ ] {
+ for (value, data_type) in [
+ (invalid.clone(), target.clone()),
+ (
+ Value::List(vec![invalid.clone()]),
+ DataType::Array {
+ element_type: Box::new(target.clone()),
+ contains_null: true,
+ },
+ ),
+ (
+ Value::Map(vec![(Value::String("k".into()),
invalid.clone())]),
+ DataType::Map {
+ key_type: Box::new(text_type()),
+ value_type: Box::new(target.clone()),
+ value_contains_null: true,
+ },
+ ),
+ (
+ Value::Struct(Row::new(vec!["n".into()],
vec![invalid.clone()]).into()),
+ DataType::Struct {
+ fields: vec![struct_field("n", target.clone())],
+ },
+ ),
+ ] {
+ let rows = vec![Row::new(vec!["n".into()], vec![value])];
+ let schema = DataType::Struct {
+ fields: vec![struct_field("n", data_type)],
+ };
+ assert!(rows_to_arrow_ipc(&rows, &schema)
+ .unwrap_err()
+ .to_string()
+ .contains("Type mismatch in row data"));
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn nested_struct_ipc_preserves_first_name_position_and_numeric_types() {
+ use arrow::array::{
+ Array, Int16Array, Int64Array, Int8Array, ListArray, MapArray,
StringArray, StructArray,
+ };
+
+ let source = Value::Struct(
+ Row::with_shared_fields(
+ vec!["dup", "dup", "other", "nested", "items", "map"]
+ .into_iter()
+ .map(str::to_string)
+ .collect::<Arc<[String]>>(),
+ vec![
+ Value::Long(25),
+ Value::Long(999),
+ Value::Long(-2),
+ Value::Struct(Row::new(vec!["n".to_string()],
vec![Value::Long(30)]).into()),
+ Value::List(vec![Value::Long(655), Value::Null]),
+ Value::Map(vec![(Value::String("k".to_string()),
Value::Long(656))]),
+ ],
+ )
+ .into(),
+ );
+ let target = DataType::Struct {
+ fields: vec![
+ struct_field("dup", DataType::Byte),
+ struct_field("positional", DataType::Short),
+ struct_field("dup", DataType::Long),
+ struct_field(
+ "nested",
+ DataType::Struct {
+ fields: vec![struct_field("n", DataType::Byte)],
+ },
+ ),
+ struct_field(
+ "items",
+ DataType::Array {
+ element_type: Box::new(DataType::Short),
+ contains_null: true,
+ },
+ ),
+ struct_field(
+ "map",
+ DataType::Map {
+ key_type: Box::new(DataType::String {
+ collation: "UTF8_BINARY".to_string(),
+ }),
+ value_type: Box::new(DataType::Short),
+ value_contains_null: true,
+ },
+ ),
+ struct_field("missing", DataType::Integer),
+ ],
+ };
+ let rows = [
+ Row::new(vec!["payload".into()], vec![source]),
+ Row::new(vec!["payload".into()], vec![Value::Null]),
+ Row::new(
+ vec!["payload".into()],
+ vec![Value::Struct(Row::new(vec![], vec![]).into())],
+ ),
+ ];
+ let schema = DataType::Struct {
+ fields: vec![struct_field("payload", target)],
+ };
+ // The values fit their declared types: an out-of-range value is not
part of what this
+ // checks (the encoder narrows it with `as`, where PySpark raises
ArrowInvalid).
+ // Read the actual IPC with Arrow rather than the client's decoder, so
+ // symmetric encoding/decoding errors cannot hide a coercion
regression.
+ let bytes = rows_to_arrow_ipc(&rows, &schema).unwrap();
+ let mut reader = StreamReader::try_new(Cursor::new(bytes),
None).unwrap();
+ let batch = reader.next().unwrap().unwrap();
+ assert!(reader.next().is_none());
+ let actual = batch
+ .column(0)
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .unwrap();
+ assert_eq!(actual.len(), 3);
+ assert!(!actual.is_null(0));
+ assert!(actual.is_null(1));
+ assert!(!actual.is_null(2));
+ assert_eq!(
+ actual
+ .column(0)
+ .as_any()
+ .downcast_ref::<Int8Array>()
+ .unwrap(),
+ &Int8Array::from(vec![Some(25), None, None]),
+ );
+ assert_eq!(
+ actual
+ .column(1)
+ .as_any()
+ .downcast_ref::<Int16Array>()
+ .unwrap(),
+ &Int16Array::from(vec![Some(999), None, None]),
+ );
+ assert_eq!(
+ actual
+ .column(2)
+ .as_any()
+ .downcast_ref::<Int64Array>()
+ .unwrap(),
+ &Int64Array::from(vec![Some(25), None, None]),
+ );
+ let nested = actual
+ .column(3)
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .unwrap();
+ assert_eq!(nested.null_count(), 2);
+ assert_eq!(
+ nested
+ .column(0)
+ .as_any()
+ .downcast_ref::<Int8Array>()
+ .unwrap(),
+ &Int8Array::from(vec![Some(30), None, None]),
+ );
+ let items = actual
+ .column(4)
+ .as_any()
+ .downcast_ref::<ListArray>()
+ .unwrap();
+ assert_eq!(items.null_count(), 2);
+ assert_eq!(items.value_offsets(), &[0, 2, 2, 2]);
+ assert_eq!(
+ items
+ .values()
+ .as_any()
+ .downcast_ref::<Int16Array>()
+ .unwrap(),
+ &Int16Array::from(vec![Some(655), None]),
+ );
+ let map = actual
+ .column(5)
+ .as_any()
+ .downcast_ref::<MapArray>()
+ .unwrap();
+ assert_eq!(map.null_count(), 2);
+ assert_eq!(map.value_offsets(), &[0, 1, 1, 1]);
+ assert_eq!(
+ map.keys().as_any().downcast_ref::<StringArray>().unwrap(),
+ &StringArray::from(vec!["k"]),
+ );
+ assert_eq!(
+ map.values().as_any().downcast_ref::<Int16Array>().unwrap(),
+ &Int16Array::from(vec![Some(656)]),
+ );
+ assert_eq!(actual.column(6).null_count(), 3);
+ }
+
#[test]
fn struct_arrow_encoding_preserves_lookup_map_and_null_semantics() {
use arrow::array::{Array, Int16Array, Int32Array, Int64Array,
StructArray};
diff --git a/crates/spark-connect/tests/e2e_coverage.rs
b/crates/spark-connect/tests/e2e_coverage.rs
index fd4a2be..b76db10 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -1155,6 +1155,68 @@ fn ml_estimators_and_transformers() {
}
}
+#[test]
+fn collected_small_integer_rows_accept_floating_schemas() {
+ if !should_run() {
+ return;
+ }
+ use spark_connect::row::Value;
+ use spark_connect::types::{DataType, StructField};
+
+ let s = session();
+ for (sql_type, min, max) in [("TINYINT", -128, 127), ("SMALLINT", -32768,
32767)] {
+ let rows = s.sql(&format!(
+ "SELECT CAST(n AS {sql_type}) AS n FROM VALUES ({min}), (-1), (0),
({max}), (NULL) AS t(n)"
+ )).unwrap().collect().unwrap();
+ for row in &rows {
+ assert!(
+ matches!(
+ (sql_type, row.get(0)),
+ ("TINYINT", Some(Value::Byte(_)))
+ | ("SMALLINT", Some(Value::Short(_)))
+ | (_, Some(Value::Null))
+ ),
+ "unexpected {sql_type} source row: {row:?}"
+ );
+ }
+ for target in [DataType::Float, DataType::Double] {
+ let schema = DataType::Struct {
+ fields: vec![StructField {
+ name: "n".into(),
+ data_type: target.clone(),
+ nullable: true,
+ metadata: Default::default(),
+ }],
+ };
+ let actual = s
+ .create_dataframe(rows.clone(), schema)
+ .unwrap()
+ .collect()
+ .unwrap();
+ let mut actual: Vec<_> = actual
+ .iter()
+ .map(|row| match (&target, row.get(0).unwrap()) {
+ (DataType::Float, Value::Float(n)) => Some(*n as f64),
+ (DataType::Double, Value::Double(n)) => Some(*n),
+ (_, Value::Null) => None,
+ other => panic!("unexpected floating result: {other:?}"),
+ })
+ .collect();
+ actual.sort_by(|a, b| a.partial_cmp(b).unwrap());
+ assert_eq!(
+ actual,
+ vec![
+ None,
+ Some(min as f64),
+ Some(-1.0),
+ Some(0.0),
+ Some(max as f64)
+ ]
+ );
+ }
+ }
+}
+
#[test]
fn create_dataframe_typed_schemas() {
if !should_run() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]