This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 7109a84  [SPARK-60641][CONNECT] Accept Byte and Short inputs for 
floating schemas
7109a84 is described below

commit 7109a84a02f5159cc22483d55547e93cb968dd29
Author: DB Tsai <[email protected]>
AuthorDate: Fri Nov 6 17:11:57 2026 +0900

    [SPARK-60641][CONNECT] Accept Byte and Short inputs for floating schemas
    
    ### What changes were proposed in this pull request?
    
    The Arrow encoder of local rows (`create_dataframe` of the Rust client) now 
takes `Value::Byte` and `Value::Short` for `FloatType` and `DoubleType` fields, 
converting them as it already converts `Value::Integer` and `Value::Long`. This 
covers nested leaves too: array elements, map values and struct fields.
    
    ### Why are the changes needed?
    
    [SPARK-60641](https://issues.apache.org/jira/browse/SPARK-60641): Rows 
collected from `TINYINT` or `SMALLINT` columns hold `Value::Byte` and 
`Value::Short`, and creating a DataFrame of them with a `FloatType` or 
`DoubleType` schema failed with `Type mismatch in row data`, though the wider 
integers work.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, for the Rust client: `create_dataframe` takes `Value::Byte` and 
`Value::Short` for floating fields, nested ones included. The Python client's 
conversion of local data is unchanged.
    
    ### How was this patch tested?
    
    - A unit test encodes `Byte` and `Short` minima, maxima, negatives, zero 
and nulls into `Float` and `Double` scalars, array elements, map values and 
struct fields, reads the IPC back with Arrow, and checks that booleans, strings 
and binary values are still rejected.
    - A unit test checks the struct field resolution of the encoder, with 
duplicate names, a positional field, nested fields and a missing field, and the 
integer types it encodes.
    - A new e2e test collects `TINYINT` and `SMALLINT` rows from a Spark server 
and creates `FloatType` and `DoubleType` DataFrames of them.
    - Against the current master, merged into this branch, and a local Spark 
4.3.0 server: the lib and every integration test of `apache-spark-connect` and 
`apache-spark-connect-core`, e2e included (777 passed), the build of all tests 
with `wasm-udf`, the offline Python suites (733 passed, 9 skipped), the live 
drop-in and Arrow suites (77 passed) and the live cases (10 passed). `cargo fmt 
--all --check` passes.
    
    Not in this change: the encoder narrows an integer that does not fit its 
field type with `as` (`Long(257)` becomes `1` in a `ByteType` field), where 
PySpark raises `ArrowInvalid`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6); the merge with master and the review 
follow-ups by Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #115 from dbtsai/codex/spark-60641.
    
    Lead-authored-by: DB Tsai <[email protected]>
    Co-authored-by: Hyukjin Kwon <[email protected]>
    Co-authored-by: DB Tsai <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/spark-connect/src/session.rs        | 313 +++++++++++++++++++++++++++++
 crates/spark-connect/tests/e2e_coverage.rs |  62 ++++++
 2 files changed, 375 insertions(+)

diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index 3b30c55..1b3601d 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -991,6 +991,8 @@ fn values_to_arrow(values: &[Value], dt: &DataType) -> 
Result<Arc<dyn arrow::arr
                 .iter()
                 .map(|v| match v {
                     Value::Null => Ok(None),
+                    Value::Byte(x) => Ok(Some(*x as $t)),
+                    Value::Short(x) => Ok(Some(*x as $t)),
                     Value::Float(x) => Ok(Some(*x as $t)),
                     Value::Double(x) => Ok(Some(*x as $t)),
                     Value::Integer(x) => Ok(Some(*x as $t)),
@@ -1310,6 +1312,317 @@ mod tests {
         }
     }
 
+    #[test]
+    fn small_integer_rows_encode_as_floating_ipc() {
+        use arrow::array::{Array, Float32Array, Float64Array, ListArray, 
MapArray, StructArray};
+        use arrow::ipc::reader::StreamReader;
+        use std::io::Cursor;
+
+        let inputs = vec![
+            Value::Byte(i8::MIN),
+            Value::Byte(-1),
+            Value::Byte(0),
+            Value::Byte(i8::MAX),
+            Value::Short(i16::MIN),
+            Value::Short(-1),
+            Value::Short(0),
+            Value::Short(i16::MAX),
+            Value::Null,
+        ];
+        let expected = vec![
+            Some(-128.0),
+            Some(-1.0),
+            Some(0.0),
+            Some(127.0),
+            Some(-32768.0),
+            Some(-1.0),
+            Some(0.0),
+            Some(32767.0),
+            None,
+        ];
+        for target in [DataType::Float, DataType::Double] {
+            let schema = DataType::Struct {
+                fields: vec![
+                    struct_field("scalar", target.clone()),
+                    struct_field(
+                        "array",
+                        DataType::Array {
+                            element_type: Box::new(target.clone()),
+                            contains_null: true,
+                        },
+                    ),
+                    struct_field(
+                        "map",
+                        DataType::Map {
+                            key_type: Box::new(DataType::String {
+                                collation: "UTF8_BINARY".into(),
+                            }),
+                            value_type: Box::new(target.clone()),
+                            value_contains_null: true,
+                        },
+                    ),
+                    struct_field(
+                        "struct",
+                        DataType::Struct {
+                            fields: vec![struct_field("n", target.clone())],
+                        },
+                    ),
+                ],
+            };
+            let rows: Vec<_> = inputs
+                .iter()
+                .map(|value| {
+                    Row::new(
+                        vec![
+                            "scalar".into(),
+                            "array".into(),
+                            "map".into(),
+                            "struct".into(),
+                        ],
+                        vec![
+                            value.clone(),
+                            Value::List(vec![value.clone()]),
+                            Value::Map(vec![(Value::String("k".to_string()), 
value.clone())]),
+                            Value::Struct(Row::new(vec!["n".into()], 
vec![value.clone()]).into()),
+                        ],
+                    )
+                })
+                .collect();
+            let bytes = rows_to_arrow_ipc(&rows, &schema).unwrap();
+            let mut reader = StreamReader::try_new(Cursor::new(bytes), 
None).unwrap();
+            let batch = reader.next().unwrap().unwrap();
+            assert_eq!(batch.num_rows(), inputs.len());
+            let array = batch
+                .column(1)
+                .as_any()
+                .downcast_ref::<ListArray>()
+                .unwrap();
+            let map = 
batch.column(2).as_any().downcast_ref::<MapArray>().unwrap();
+            let nested = batch
+                .column(3)
+                .as_any()
+                .downcast_ref::<StructArray>()
+                .unwrap();
+            for column in [
+                batch.column(0),
+                array.values(),
+                map.values(),
+                nested.column(0),
+            ] {
+                match target {
+                    DataType::Float => assert_eq!(
+                        
column.as_any().downcast_ref::<Float32Array>().unwrap(),
+                        &Float32Array::from(
+                            expected
+                                .iter()
+                                .map(|v| v.map(|n| n as f32))
+                                .collect::<Vec<_>>()
+                        ),
+                    ),
+                    DataType::Double => assert_eq!(
+                        
column.as_any().downcast_ref::<Float64Array>().unwrap(),
+                        &Float64Array::from(expected.clone()),
+                    ),
+                    _ => unreachable!(),
+                }
+                assert_eq!(column.null_count(), 1);
+            }
+            assert!(reader.next().is_none());
+            for invalid in [
+                Value::Bool(true),
+                Value::String("7".into()),
+                Value::Binary(vec![7]),
+            ] {
+                for (value, data_type) in [
+                    (invalid.clone(), target.clone()),
+                    (
+                        Value::List(vec![invalid.clone()]),
+                        DataType::Array {
+                            element_type: Box::new(target.clone()),
+                            contains_null: true,
+                        },
+                    ),
+                    (
+                        Value::Map(vec![(Value::String("k".into()), 
invalid.clone())]),
+                        DataType::Map {
+                            key_type: Box::new(text_type()),
+                            value_type: Box::new(target.clone()),
+                            value_contains_null: true,
+                        },
+                    ),
+                    (
+                        Value::Struct(Row::new(vec!["n".into()], 
vec![invalid.clone()]).into()),
+                        DataType::Struct {
+                            fields: vec![struct_field("n", target.clone())],
+                        },
+                    ),
+                ] {
+                    let rows = vec![Row::new(vec!["n".into()], vec![value])];
+                    let schema = DataType::Struct {
+                        fields: vec![struct_field("n", data_type)],
+                    };
+                    assert!(rows_to_arrow_ipc(&rows, &schema)
+                        .unwrap_err()
+                        .to_string()
+                        .contains("Type mismatch in row data"));
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn nested_struct_ipc_preserves_first_name_position_and_numeric_types() {
+        use arrow::array::{
+            Array, Int16Array, Int64Array, Int8Array, ListArray, MapArray, 
StringArray, StructArray,
+        };
+
+        let source = Value::Struct(
+            Row::with_shared_fields(
+                vec!["dup", "dup", "other", "nested", "items", "map"]
+                    .into_iter()
+                    .map(str::to_string)
+                    .collect::<Arc<[String]>>(),
+                vec![
+                    Value::Long(25),
+                    Value::Long(999),
+                    Value::Long(-2),
+                    Value::Struct(Row::new(vec!["n".to_string()], 
vec![Value::Long(30)]).into()),
+                    Value::List(vec![Value::Long(655), Value::Null]),
+                    Value::Map(vec![(Value::String("k".to_string()), 
Value::Long(656))]),
+                ],
+            )
+            .into(),
+        );
+        let target = DataType::Struct {
+            fields: vec![
+                struct_field("dup", DataType::Byte),
+                struct_field("positional", DataType::Short),
+                struct_field("dup", DataType::Long),
+                struct_field(
+                    "nested",
+                    DataType::Struct {
+                        fields: vec![struct_field("n", DataType::Byte)],
+                    },
+                ),
+                struct_field(
+                    "items",
+                    DataType::Array {
+                        element_type: Box::new(DataType::Short),
+                        contains_null: true,
+                    },
+                ),
+                struct_field(
+                    "map",
+                    DataType::Map {
+                        key_type: Box::new(DataType::String {
+                            collation: "UTF8_BINARY".to_string(),
+                        }),
+                        value_type: Box::new(DataType::Short),
+                        value_contains_null: true,
+                    },
+                ),
+                struct_field("missing", DataType::Integer),
+            ],
+        };
+        let rows = [
+            Row::new(vec!["payload".into()], vec![source]),
+            Row::new(vec!["payload".into()], vec![Value::Null]),
+            Row::new(
+                vec!["payload".into()],
+                vec![Value::Struct(Row::new(vec![], vec![]).into())],
+            ),
+        ];
+        let schema = DataType::Struct {
+            fields: vec![struct_field("payload", target)],
+        };
+        // The values fit their declared types: an out-of-range value is not 
part of what this
+        // checks (the encoder narrows it with `as`, where PySpark raises 
ArrowInvalid).
+        // Read the actual IPC with Arrow rather than the client's decoder, so
+        // symmetric encoding/decoding errors cannot hide a coercion 
regression.
+        let bytes = rows_to_arrow_ipc(&rows, &schema).unwrap();
+        let mut reader = StreamReader::try_new(Cursor::new(bytes), 
None).unwrap();
+        let batch = reader.next().unwrap().unwrap();
+        assert!(reader.next().is_none());
+        let actual = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .unwrap();
+        assert_eq!(actual.len(), 3);
+        assert!(!actual.is_null(0));
+        assert!(actual.is_null(1));
+        assert!(!actual.is_null(2));
+        assert_eq!(
+            actual
+                .column(0)
+                .as_any()
+                .downcast_ref::<Int8Array>()
+                .unwrap(),
+            &Int8Array::from(vec![Some(25), None, None]),
+        );
+        assert_eq!(
+            actual
+                .column(1)
+                .as_any()
+                .downcast_ref::<Int16Array>()
+                .unwrap(),
+            &Int16Array::from(vec![Some(999), None, None]),
+        );
+        assert_eq!(
+            actual
+                .column(2)
+                .as_any()
+                .downcast_ref::<Int64Array>()
+                .unwrap(),
+            &Int64Array::from(vec![Some(25), None, None]),
+        );
+        let nested = actual
+            .column(3)
+            .as_any()
+            .downcast_ref::<StructArray>()
+            .unwrap();
+        assert_eq!(nested.null_count(), 2);
+        assert_eq!(
+            nested
+                .column(0)
+                .as_any()
+                .downcast_ref::<Int8Array>()
+                .unwrap(),
+            &Int8Array::from(vec![Some(30), None, None]),
+        );
+        let items = actual
+            .column(4)
+            .as_any()
+            .downcast_ref::<ListArray>()
+            .unwrap();
+        assert_eq!(items.null_count(), 2);
+        assert_eq!(items.value_offsets(), &[0, 2, 2, 2]);
+        assert_eq!(
+            items
+                .values()
+                .as_any()
+                .downcast_ref::<Int16Array>()
+                .unwrap(),
+            &Int16Array::from(vec![Some(655), None]),
+        );
+        let map = actual
+            .column(5)
+            .as_any()
+            .downcast_ref::<MapArray>()
+            .unwrap();
+        assert_eq!(map.null_count(), 2);
+        assert_eq!(map.value_offsets(), &[0, 1, 1, 1]);
+        assert_eq!(
+            map.keys().as_any().downcast_ref::<StringArray>().unwrap(),
+            &StringArray::from(vec!["k"]),
+        );
+        assert_eq!(
+            map.values().as_any().downcast_ref::<Int16Array>().unwrap(),
+            &Int16Array::from(vec![Some(656)]),
+        );
+        assert_eq!(actual.column(6).null_count(), 3);
+    }
+
     #[test]
     fn struct_arrow_encoding_preserves_lookup_map_and_null_semantics() {
         use arrow::array::{Array, Int16Array, Int32Array, Int64Array, 
StructArray};
diff --git a/crates/spark-connect/tests/e2e_coverage.rs 
b/crates/spark-connect/tests/e2e_coverage.rs
index fd4a2be..b76db10 100644
--- a/crates/spark-connect/tests/e2e_coverage.rs
+++ b/crates/spark-connect/tests/e2e_coverage.rs
@@ -1155,6 +1155,68 @@ fn ml_estimators_and_transformers() {
     }
 }
 
+#[test]
+fn collected_small_integer_rows_accept_floating_schemas() {
+    if !should_run() {
+        return;
+    }
+    use spark_connect::row::Value;
+    use spark_connect::types::{DataType, StructField};
+
+    let s = session();
+    for (sql_type, min, max) in [("TINYINT", -128, 127), ("SMALLINT", -32768, 
32767)] {
+        let rows = s.sql(&format!(
+            "SELECT CAST(n AS {sql_type}) AS n FROM VALUES ({min}), (-1), (0), 
({max}), (NULL) AS t(n)"
+        )).unwrap().collect().unwrap();
+        for row in &rows {
+            assert!(
+                matches!(
+                    (sql_type, row.get(0)),
+                    ("TINYINT", Some(Value::Byte(_)))
+                        | ("SMALLINT", Some(Value::Short(_)))
+                        | (_, Some(Value::Null))
+                ),
+                "unexpected {sql_type} source row: {row:?}"
+            );
+        }
+        for target in [DataType::Float, DataType::Double] {
+            let schema = DataType::Struct {
+                fields: vec![StructField {
+                    name: "n".into(),
+                    data_type: target.clone(),
+                    nullable: true,
+                    metadata: Default::default(),
+                }],
+            };
+            let actual = s
+                .create_dataframe(rows.clone(), schema)
+                .unwrap()
+                .collect()
+                .unwrap();
+            let mut actual: Vec<_> = actual
+                .iter()
+                .map(|row| match (&target, row.get(0).unwrap()) {
+                    (DataType::Float, Value::Float(n)) => Some(*n as f64),
+                    (DataType::Double, Value::Double(n)) => Some(*n),
+                    (_, Value::Null) => None,
+                    other => panic!("unexpected floating result: {other:?}"),
+                })
+                .collect();
+            actual.sort_by(|a, b| a.partial_cmp(b).unwrap());
+            assert_eq!(
+                actual,
+                vec![
+                    None,
+                    Some(min as f64),
+                    Some(-1.0),
+                    Some(0.0),
+                    Some(max as f64)
+                ]
+            );
+        }
+    }
+}
+
 #[test]
 fn create_dataframe_typed_schemas() {
     if !should_run() {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to