This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 9ead246  [SPARK-60927][CONNECT] Keep the keys of map values typed and 
in order
9ead246 is described below

commit 9ead246b71d0257f2a56baf868ad3baec8bf4497
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 07:01:05 2026 +0900

    [SPARK-60927][CONNECT] Keep the keys of map values typed and in order
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60927](https://issues.apache.org/jira/browse/SPARK-60927): map 
values keep their entries in order, with keys of their own type, as PySpark 
reads a map into a dict.
    
    - `Value::Map` is now a list of key-value pairs, in the order that the 
server sent them, rather than a sorted map of string keys. Decoding keeps each 
key as the value of its own type, and encoding writes the keys of a map type as 
given.
    - In the Python client, a collected map is a dict with those keys, in 
order. A key that Python cannot hash, such as an array, raises `TypeError`, as 
in PySpark.
    - A dict passed to `createDataFrame` keeps its keys too, so that integer 
keys infer `map<bigint,...>` rather than `map<string,...>`.
    
    This changes the Rust API of `Value::Map` from `BTreeMap<String, Value>` to 
`Vec<(Value, Value)>`.
    
    ### Why are the changes needed?
    
    Every map key became a string, and the keys were sorted:
    
    - `spark.sql("SELECT map(1, 'a', 2, 'b') AS m").first().m` had the keys 
`'1'` and `'2'`, a boolean, double or date key also became a string, a date as 
its day number, and an array key became its Rust debug form.
    - `map('b', 1, 'a', 2)` came back with `'a'` first.
    - `createDataFrame` of a dict with integer keys sent `map<string,string>`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, maps now come back and go out as in PySpark Connect 4.3.0. One 
difference stays: PySpark raises the `TypeError` for an unhashable key at 
`collect()`, and this client when the value is read, as it turns a row's values 
into Python ones then.
    
    ### How was this patch tested?
    
    - Against a local Spark 4.3.0 server, a probe of sixteen cases, maps with 
integer, string, boolean, double, date, array and struct keys, nested and in 
arrays, null and empty, and `createDataFrame` of dicts with integer, string and 
date keys, gives output identical to official PySpark 4.3.0.
    - A new unit test checks that a decoded map keeps its entries in order with 
keys of their own type. A new live test checks the collected maps and 
`createDataFrame`; it also passes with official PySpark 4.3.0.
    - The lib tests and every integration test, e2e included, against the local 
server (635 passed), the build of all tests with `wasm-udf`, the offline Python 
suites (514 passed, 8 live cases skipped), and the live drop-in and Arrow 
suites (52 passed) and live cases (10 passed). Clippy reports no diagnostics in 
the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #161 from HyukjinKwon/SPARK-60927-map-keys.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/row.rs                  |   4 +-
 crates/pyspark-rs/src/session.rs              |  23 +++---
 crates/spark-connect/src/dataframe.rs         | 103 ++++++++++----------------
 crates/spark-connect/src/row.rs               |   9 +--
 crates/spark-connect/src/session.rs           |  21 +++---
 crates/spark-connect/tests/e2e_integration.rs |   7 +-
 python/tests/test_dropin_e2e.py               |  20 +++++
 7 files changed, 94 insertions(+), 93 deletions(-)

diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 480fdc9..fe43303 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -185,9 +185,11 @@ pub fn value_to_py<'py>(py: Python<'py>, v: &Value) -> 
PyResult<Bound<'py, PyAny
             list.into_any()
         }
         Value::Map(m) => {
+            // Keys of their own type, in order; an unhashable key, such as an 
array, raises
+            // TypeError, as in PySpark.
             let dict = PyDict::new(py);
             for (k, val) in m {
-                dict.set_item(k, value_to_py(py, val)?)?;
+                dict.set_item(value_to_py(py, k)?, value_to_py(py, val)?)?;
             }
             dict.into_any()
         }
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index aae3795..8807b8c 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1183,14 +1183,17 @@ fn value_to_datatype(v: &Value) -> DataType {
                 .collect(),
         },
         Value::Map(m) => {
-            let value_type = m
-                .values()
-                .find(|x| !matches!(x, Value::Null))
-                .map(value_to_datatype)
-                .unwrap_or_else(utf8);
+            // The type of the first non-null key or value, as the other 
fields take theirs.
+            let first_type = |values: Vec<&Value>| {
+                values
+                    .into_iter()
+                    .find(|x| !matches!(x, Value::Null))
+                    .map(value_to_datatype)
+                    .unwrap_or_else(utf8)
+            };
             DataType::Map {
-                key_type: Box::new(utf8()),
-                value_type: Box::new(value_type),
+                key_type: Box::new(first_type(m.iter().map(|(k, _)| 
k).collect())),
+                value_type: Box::new(first_type(m.iter().map(|(_, v)| 
v).collect())),
                 value_contains_null: true,
             }
         }
@@ -1271,11 +1274,11 @@ pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) -> 
PyResult<Value> {
         return Ok(Value::Struct(pyrow.to_row(obj.py())?.into()));
     }
 
-    // dict -> map (keys coerced to their string form, values recursively 
converted)
+    // dict -> map, its entries in order with keys and values of their own 
types
     if let Ok(dict) = obj.cast::<pyo3::types::PyDict>() {
-        let mut map = std::collections::BTreeMap::new();
+        let mut map = Vec::with_capacity(dict.len());
         for (k, v) in dict.iter() {
-            map.insert(k.str()?.to_string(), py_to_value(&v)?);
+            map.push((py_to_value(&k)?, py_to_value(&v)?));
         }
         return Ok(Value::Map(map));
     }
diff --git a/crates/spark-connect/src/dataframe.rs 
b/crates/spark-connect/src/dataframe.rs
index a83b0e6..8aa9db0 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -2100,29 +2100,6 @@ fn record_batches_to_polars(
         .map_err(|e| SparkError::connect_msg(format!("Failed to create Polars 
DataFrame: {e}")))
 }
 
-/// Render a decoded map key ([`Value`]) as its natural scalar string, since
-/// `Value::Map` keys are `String`. Numeric/boolean keys use their value (`1`,
-/// `true`), decimals their preserved digits; non-scalar keys (rare) fall back 
to
-/// Debug. This must never use the enum's Debug form for scalars - a 
`map<int,…>`
-/// key has to be "1", not "Integer(1)".
-fn map_key_to_string(v: Value) -> String {
-    match v {
-        Value::String(s) => s,
-        Value::Bool(b) => b.to_string(),
-        Value::Byte(x) => x.to_string(),
-        Value::Short(x) => x.to_string(),
-        Value::Integer(x) => x.to_string(),
-        Value::Long(x) => x.to_string(),
-        Value::Float(x) => x.to_string(),
-        Value::Double(x) => x.to_string(),
-        Value::Date(d) => d.to_string(),
-        Value::Timestamp(t) => t.to_string(),
-        interval @ Value::DayTimeInterval(_) => interval.to_string(),
-        Value::Decimal { value, .. } => value,
-        other => format!("{other:?}"),
-    }
-}
-
 /// Immutable decoding metadata prepared once for each column in a record 
batch.
 /// Container layouts follow their children so struct names are also shared 
inside
 /// lists and maps, without retaining the Arrow arrays after decoding.
@@ -2342,16 +2319,13 @@ fn arrow_value_at_with_layout(
         let entries = arr.value(index);
         let keys = entries.column(0);
         let vals = entries.column(1);
-        let mut map = std::collections::BTreeMap::new();
+        // The entries in their order, with keys of their own type, as PySpark 
reads them.
+        let mut map = Vec::with_capacity(entries.len());
         for i in 0..entries.len() {
-            // A map key stringifies to its natural scalar form (e.g. `1`, 
`true`,
-            // `1.5`), not the Rust enum's Debug output - a `map<int,string>` 
key must
-            // be "1", never "Integer(1)".
-            let k = 
map_key_to_string(arrow_value_at_with_layout(keys.as_ref(), i, &children.0)?);
-            map.insert(
-                k,
+            map.push((
+                arrow_value_at_with_layout(keys.as_ref(), i, &children.0)?,
                 arrow_value_at_with_layout(vals.as_ref(), i, &children.1)?,
-            );
+            ));
         }
         return Ok(Value::Map(map));
     }
@@ -2549,7 +2523,7 @@ mod nested_decode_tests {
                 let Value::Map(map) = value else {
                     panic!("expected map");
                 };
-                leaves.push(as_struct(map.values().next().unwrap()));
+                leaves.push(as_struct(&map[0].1));
             }
         }
         assert_eq!(leaves.len(), 4);
@@ -2614,13 +2588,10 @@ mod nested_decode_tests {
         );
         assert_eq!(
             rows[0].get(1),
-            Some(&Value::Map(
-                [
-                    ("k1".to_string(), value(1)),
-                    ("k2".to_string(), Value::Null)
-                ]
-                .into()
-            ))
+            Some(&Value::Map(vec![
+                (Value::String("k1".to_string()), value(1)),
+                (Value::String("k2".to_string()), Value::Null)
+            ]))
         );
         assert_eq!(rows[1].get(0), Some(&Value::List(vec![])));
         assert_eq!(rows[1].get(1), Some(&Value::Map(Default::default())));
@@ -2629,9 +2600,10 @@ mod nested_decode_tests {
         assert_eq!(rows[3].get(0), Some(&Value::List(vec![value(4), 
value(5)])));
         assert_eq!(
             rows[3].get(1),
-            Some(&Value::Map(
-                [("k4".to_string(), value(4)), ("k5".to_string(), 
value(5))].into()
-            ))
+            Some(&Value::Map(vec![
+                (Value::String("k4".to_string()), value(4)),
+                (Value::String("k5".to_string()), value(5))
+            ]))
         );
         let (Value::List(first), Value::List(last)) =
             (rows[0].get(0).unwrap(), rows[3].get(0).unwrap())
@@ -2648,8 +2620,8 @@ mod nested_decode_tests {
             panic!("expected maps");
         };
         assert!(std::ptr::eq(
-            as_struct(&first["k1"]).fields(),
-            as_struct(&last["k4"]).fields()
+            as_struct(&first[0].1).fields(),
+            as_struct(&last[0].1).fields()
         ));
     }
 
@@ -3922,28 +3894,31 @@ mod arrow_value_tests {
     }
 
     #[test]
-    fn map_key_to_string_covers_scalar_arms() {
-        assert_eq!(map_key_to_string(Value::String("x".to_string())), "x");
-        assert_eq!(map_key_to_string(Value::Bool(true)), "true");
-        assert_eq!(map_key_to_string(Value::Byte(1)), "1");
-        assert_eq!(map_key_to_string(Value::Short(2)), "2");
-        assert_eq!(map_key_to_string(Value::Integer(3)), "3");
-        assert_eq!(map_key_to_string(Value::Long(4)), "4");
-        assert_eq!(map_key_to_string(Value::Float(1.5)), "1.5");
-        assert_eq!(map_key_to_string(Value::Double(2.5)), "2.5");
-        assert_eq!(map_key_to_string(Value::Date(5)), "5");
-        assert_eq!(map_key_to_string(Value::Timestamp(6)), "6");
-        assert_eq!(map_key_to_string(Value::DayTimeInterval(-1)), "-1");
+    fn a_map_keeps_its_entries_in_order_with_keys_of_their_type() {
+        let entries = StructArray::from(vec![
+            (
+                Arc::new(Field::new("key", arrow::datatypes::DataType::Int32, 
false)),
+                Arc::new(Int32Array::from(vec![2, 1])) as ArrayRef,
+            ),
+            (
+                Arc::new(Field::new("value", arrow::datatypes::DataType::Utf8, 
true)),
+                Arc::new(StringArray::from(vec!["b", "a"])) as ArrayRef,
+            ),
+        ]);
+        let map = MapArray::new(
+            Arc::new(Field::new("entries", entries.data_type().clone(), 
false)),
+            arrow::buffer::OffsetBuffer::new(vec![0, 2].into()),
+            entries,
+            None,
+            false,
+        );
         assert_eq!(
-            map_key_to_string(Value::Decimal {
-                value: "7.5".to_string(),
-                precision: None,
-                scale: None,
-            }),
-            "7.5"
+            arrow_value_at(&map, 0).unwrap(),
+            Value::Map(vec![
+                (Value::Integer(2), Value::String("b".to_string())),
+                (Value::Integer(1), Value::String("a".to_string())),
+            ])
         );
-        // Non-scalar key falls back to the Debug form.
-        let _ = map_key_to_string(Value::List(vec![]));
     }
 
     #[test]
diff --git a/crates/spark-connect/src/row.rs b/crates/spark-connect/src/row.rs
index a817b04..38dab85 100644
--- a/crates/spark-connect/src/row.rs
+++ b/crates/spark-connect/src/row.rs
@@ -3,7 +3,6 @@
 //! A Row represents a single record: an ordered collection of (field_name, 
value) pairs.
 //! Values are accessed by index or by field name. Supports all Spark SQL 
value types.
 
-use std::collections::BTreeMap;
 use std::fmt;
 use std::sync::Arc;
 
@@ -46,8 +45,9 @@ pub enum Value {
     },
     /// Array of values
     List(Vec<Value>),
-    /// Map of key-value pairs
-    Map(BTreeMap<String, Value>),
+    /// Map entries, in their order, with keys and values of their own types, 
as PySpark
+    /// reads a map into a dict.
+    Map(Vec<(Value, Value)>),
     /// Struct (nested Row), with ordered names and values.
     /// Construct with `Value::Struct(Row::new(fields, values).into())`.
     Struct(StructValue),
@@ -756,8 +756,7 @@ mod tests {
             Value::List(vec![Value::Integer(1), 
Value::Integer(2)]).to_string(),
             "[1, 2]"
         );
-        let mut m = std::collections::BTreeMap::new();
-        m.insert("k".to_string(), Value::Integer(9));
+        let m = vec![(Value::String("k".to_string()), Value::Integer(9))];
         assert_eq!(Value::Map(m).to_string(), "{k: 9}");
         assert_eq!(
             Value::Struct(Row::new(vec!["a".to_string()], 
vec![Value::Integer(1)]).into())
diff --git a/crates/spark-connect/src/session.rs 
b/crates/spark-connect/src/session.rs
index eeff006..a66366f 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -1046,7 +1046,11 @@ fn values_to_arrow(values: &[Value], dt: &DataType) -> 
Result<Arc<dyn arrow::arr
                     Value::Map(m) => {
                         valid.append(true);
                         for (i, f) in fields.iter().enumerate() {
-                            
cols[i].push(m.get(&f.name).cloned().unwrap_or(Value::Null));
+                            let value = m.iter().find_map(|(key, value)| match 
key {
+                                Value::String(key) if *key == f.name => 
Some(value.clone()),
+                                _ => None,
+                            });
+                            cols[i].push(value.unwrap_or(Value::Null));
                         }
                     }
                     _ => return Err(SparkError::connect_msg("Type mismatch: 
expected struct")),
@@ -1092,7 +1096,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) -> 
Result<Arc<dyn arrow::arr
                     Value::Map(m) => {
                         valid.append(true);
                         for (k, val) in m {
-                            keys.push(Value::String(k.clone()));
+                            keys.push(k.clone());
                             vals.push(val.clone());
                         }
                         offsets.push(keys.len() as i32);
@@ -1203,13 +1207,10 @@ mod tests {
                 )
                 .into(),
             ),
-            Value::Map(
-                [
-                    ("dup".to_string(), Value::Long(5)),
-                    ("other".to_string(), Value::Long(9)),
-                ]
-                .into(),
-            ),
+            Value::Map(vec![
+                (Value::String("dup".to_string()), Value::Long(5)),
+                (Value::String("other".to_string()), Value::Long(9)),
+            ]),
             Value::Null,
             Value::Struct(Row::new(vec!["dup".to_string()], 
vec![Value::Null]).into()),
         ];
@@ -1285,7 +1286,7 @@ mod tests {
                 vec!["items".to_string(), "lookup".to_string()],
                 vec![
                     Value::List(vec![leaf_value(), Value::Null]),
-                    Value::Map([("k".to_string(), leaf_value())].into()),
+                    Value::Map(vec![(Value::String("k".to_string()), 
leaf_value())]),
                 ],
             )
             .into(),
diff --git a/crates/spark-connect/tests/e2e_integration.rs 
b/crates/spark-connect/tests/e2e_integration.rs
index 9b1e048..0a81f1b 100644
--- a/crates/spark-connect/tests/e2e_integration.rs
+++ b/crates/spark-connect/tests/e2e_integration.rs
@@ -657,9 +657,10 @@ fn test_day_time_interval_real_server() {
     );
     assert_eq!(
         row.get(5),
-        Some(&Value::Map(
-            [("k".to_string(), Value::DayTimeInterval(micros))].into()
-        ))
+        Some(&Value::Map(vec![(
+            Value::String("k".to_string()),
+            Value::DayTimeInterval(micros)
+        )]))
     );
     for prefetch in [false, true] {
         let streamed: Vec<_> = frame
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 594dc12..ceb4569 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -361,3 +361,23 @@ def test_sql_runs_statements_eagerly(session):
 def test_sql_fails_at_a_statement_that_does_not_parse(session):
     with pytest.raises(AnalysisException, match="PARSE_SYNTAX_ERROR"):
         session.sql("SELEC 1")
+
+
+def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
+    row = session.range(1).selectExpr(
+        "map(2, 'b', 1, 'a') AS ints",
+        "map('b', 1, 'a', 2) AS strs",
+        "map(date'2020-01-02', 1) AS dates",
+        "map(named_struct('a', 1), 'x') AS structs",
+    ).first()
+    assert list(row.ints.items()) == [(2, "b"), (1, "a")]
+    assert list(row.strs.items()) == [("b", 1), ("a", 2)]
+    assert row.dates == {date(2020, 1, 2): 1}
+    assert row.structs == {Row(a=1): "x"}
+    # A list key is unhashable, as in PySpark. (PySpark raises at collect(); 
this client
+    # turns a row's values into Python ones when they are read.)
+    with pytest.raises(TypeError, match="unhashable"):
+        session.range(1).selectExpr("map(array(1), 'x') AS m").first().m
+    df = session.createDataFrame([({2: "b", 1: "a"},)])
+    assert df.schema.simpleString() == "struct<_1:map<bigint,string>>"
+    assert list(df.first()[0].items()) == [(2, "b"), (1, "a")]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to