This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 9ead246 [SPARK-60927][CONNECT] Keep the keys of map values typed and
in order
9ead246 is described below
commit 9ead246b71d0257f2a56baf868ad3baec8bf4497
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Fri Nov 6 07:01:05 2026 +0900
[SPARK-60927][CONNECT] Keep the keys of map values typed and in order
### What changes were proposed in this pull request?
[SPARK-60927](https://issues.apache.org/jira/browse/SPARK-60927): map
values keep their entries in order, with keys of their own type, as PySpark
reads a map into a dict.
- `Value::Map` is now a list of key-value pairs, in the order that the
server sent them, rather than a sorted map of string keys. Decoding keeps each
key as the value of its own type, and encoding writes the keys of a map type as
given.
- In the Python client, a collected map is a dict with those keys, in
order. A key that Python cannot hash, such as an array, raises `TypeError`, as
in PySpark.
- A dict passed to `createDataFrame` keeps its keys too, so that integer
keys infer `map<bigint,...>` rather than `map<string,...>`.
This changes the Rust API of `Value::Map` from `BTreeMap<String, Value>` to
`Vec<(Value, Value)>`.
### Why are the changes needed?
Every map key became a string, and the keys were sorted:
- `spark.sql("SELECT map(1, 'a', 2, 'b') AS m").first().m` had the keys
`'1'` and `'2'`, a boolean, double or date key also became a string, a date as
its day number, and an array key became its Rust debug form.
- `map('b', 1, 'a', 2)` came back with `'a'` first.
- `createDataFrame` of a dict with integer keys sent `map<string,string>`.
### Does this PR introduce _any_ user-facing change?
Yes, maps now come back and go out as in PySpark Connect 4.3.0. One
difference stays: PySpark raises the `TypeError` for an unhashable key at
`collect()`, and this client when the value is read, as it turns a row's values
into Python ones then.
### How was this patch tested?
- Against a local Spark 4.3.0 server, a probe of sixteen cases, maps with
integer, string, boolean, double, date, array and struct keys, nested and in
arrays, null and empty, and `createDataFrame` of dicts with integer, string and
date keys, gives output identical to official PySpark 4.3.0.
- A new unit test checks that a decoded map keeps its entries in order with
keys of their own type. A new live test checks the collected maps and
`createDataFrame`; it also passes with official PySpark 4.3.0.
- The lib tests and every integration test, e2e included, against the local
server (635 passed), the build of all tests with `wasm-udf`, the offline Python
suites (514 passed, 8 live cases skipped), and the live drop-in and Arrow
suites (52 passed) and live cases (10 passed). Clippy reports no diagnostics in
the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #161 from HyukjinKwon/SPARK-60927-map-keys.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/row.rs | 4 +-
crates/pyspark-rs/src/session.rs | 23 +++---
crates/spark-connect/src/dataframe.rs | 103 ++++++++++----------------
crates/spark-connect/src/row.rs | 9 +--
crates/spark-connect/src/session.rs | 21 +++---
crates/spark-connect/tests/e2e_integration.rs | 7 +-
python/tests/test_dropin_e2e.py | 20 +++++
7 files changed, 94 insertions(+), 93 deletions(-)
diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 480fdc9..fe43303 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -185,9 +185,11 @@ pub fn value_to_py<'py>(py: Python<'py>, v: &Value) ->
PyResult<Bound<'py, PyAny
list.into_any()
}
Value::Map(m) => {
+ // Keys of their own type, in order; an unhashable key, such as an
array, raises
+ // TypeError, as in PySpark.
let dict = PyDict::new(py);
for (k, val) in m {
- dict.set_item(k, value_to_py(py, val)?)?;
+ dict.set_item(value_to_py(py, k)?, value_to_py(py, val)?)?;
}
dict.into_any()
}
diff --git a/crates/pyspark-rs/src/session.rs b/crates/pyspark-rs/src/session.rs
index aae3795..8807b8c 100644
--- a/crates/pyspark-rs/src/session.rs
+++ b/crates/pyspark-rs/src/session.rs
@@ -1183,14 +1183,17 @@ fn value_to_datatype(v: &Value) -> DataType {
.collect(),
},
Value::Map(m) => {
- let value_type = m
- .values()
- .find(|x| !matches!(x, Value::Null))
- .map(value_to_datatype)
- .unwrap_or_else(utf8);
+ // The type of the first non-null key or value, as the other
fields take theirs.
+ let first_type = |values: Vec<&Value>| {
+ values
+ .into_iter()
+ .find(|x| !matches!(x, Value::Null))
+ .map(value_to_datatype)
+ .unwrap_or_else(utf8)
+ };
DataType::Map {
- key_type: Box::new(utf8()),
- value_type: Box::new(value_type),
+ key_type: Box::new(first_type(m.iter().map(|(k, _)|
k).collect())),
+ value_type: Box::new(first_type(m.iter().map(|(_, v)|
v).collect())),
value_contains_null: true,
}
}
@@ -1271,11 +1274,11 @@ pub(crate) fn py_to_value(obj: &Bound<'_, PyAny>) ->
PyResult<Value> {
return Ok(Value::Struct(pyrow.to_row(obj.py())?.into()));
}
- // dict -> map (keys coerced to their string form, values recursively
converted)
+ // dict -> map, its entries in order with keys and values of their own
types
if let Ok(dict) = obj.cast::<pyo3::types::PyDict>() {
- let mut map = std::collections::BTreeMap::new();
+ let mut map = Vec::with_capacity(dict.len());
for (k, v) in dict.iter() {
- map.insert(k.str()?.to_string(), py_to_value(&v)?);
+ map.push((py_to_value(&k)?, py_to_value(&v)?));
}
return Ok(Value::Map(map));
}
diff --git a/crates/spark-connect/src/dataframe.rs
b/crates/spark-connect/src/dataframe.rs
index a83b0e6..8aa9db0 100644
--- a/crates/spark-connect/src/dataframe.rs
+++ b/crates/spark-connect/src/dataframe.rs
@@ -2100,29 +2100,6 @@ fn record_batches_to_polars(
.map_err(|e| SparkError::connect_msg(format!("Failed to create Polars
DataFrame: {e}")))
}
-/// Render a decoded map key ([`Value`]) as its natural scalar string, since
-/// `Value::Map` keys are `String`. Numeric/boolean keys use their value (`1`,
-/// `true`), decimals their preserved digits; non-scalar keys (rare) fall back
to
-/// Debug. This must never use the enum's Debug form for scalars - a
`map<int,…>`
-/// key has to be "1", not "Integer(1)".
-fn map_key_to_string(v: Value) -> String {
- match v {
- Value::String(s) => s,
- Value::Bool(b) => b.to_string(),
- Value::Byte(x) => x.to_string(),
- Value::Short(x) => x.to_string(),
- Value::Integer(x) => x.to_string(),
- Value::Long(x) => x.to_string(),
- Value::Float(x) => x.to_string(),
- Value::Double(x) => x.to_string(),
- Value::Date(d) => d.to_string(),
- Value::Timestamp(t) => t.to_string(),
- interval @ Value::DayTimeInterval(_) => interval.to_string(),
- Value::Decimal { value, .. } => value,
- other => format!("{other:?}"),
- }
-}
-
/// Immutable decoding metadata prepared once for each column in a record
batch.
/// Container layouts follow their children so struct names are also shared
inside
/// lists and maps, without retaining the Arrow arrays after decoding.
@@ -2342,16 +2319,13 @@ fn arrow_value_at_with_layout(
let entries = arr.value(index);
let keys = entries.column(0);
let vals = entries.column(1);
- let mut map = std::collections::BTreeMap::new();
+ // The entries in their order, with keys of their own type, as PySpark
reads them.
+ let mut map = Vec::with_capacity(entries.len());
for i in 0..entries.len() {
- // A map key stringifies to its natural scalar form (e.g. `1`,
`true`,
- // `1.5`), not the Rust enum's Debug output - a `map<int,string>`
key must
- // be "1", never "Integer(1)".
- let k =
map_key_to_string(arrow_value_at_with_layout(keys.as_ref(), i, &children.0)?);
- map.insert(
- k,
+ map.push((
+ arrow_value_at_with_layout(keys.as_ref(), i, &children.0)?,
arrow_value_at_with_layout(vals.as_ref(), i, &children.1)?,
- );
+ ));
}
return Ok(Value::Map(map));
}
@@ -2549,7 +2523,7 @@ mod nested_decode_tests {
let Value::Map(map) = value else {
panic!("expected map");
};
- leaves.push(as_struct(map.values().next().unwrap()));
+ leaves.push(as_struct(&map[0].1));
}
}
assert_eq!(leaves.len(), 4);
@@ -2614,13 +2588,10 @@ mod nested_decode_tests {
);
assert_eq!(
rows[0].get(1),
- Some(&Value::Map(
- [
- ("k1".to_string(), value(1)),
- ("k2".to_string(), Value::Null)
- ]
- .into()
- ))
+ Some(&Value::Map(vec![
+ (Value::String("k1".to_string()), value(1)),
+ (Value::String("k2".to_string()), Value::Null)
+ ]))
);
assert_eq!(rows[1].get(0), Some(&Value::List(vec![])));
assert_eq!(rows[1].get(1), Some(&Value::Map(Default::default())));
@@ -2629,9 +2600,10 @@ mod nested_decode_tests {
assert_eq!(rows[3].get(0), Some(&Value::List(vec![value(4),
value(5)])));
assert_eq!(
rows[3].get(1),
- Some(&Value::Map(
- [("k4".to_string(), value(4)), ("k5".to_string(),
value(5))].into()
- ))
+ Some(&Value::Map(vec![
+ (Value::String("k4".to_string()), value(4)),
+ (Value::String("k5".to_string()), value(5))
+ ]))
);
let (Value::List(first), Value::List(last)) =
(rows[0].get(0).unwrap(), rows[3].get(0).unwrap())
@@ -2648,8 +2620,8 @@ mod nested_decode_tests {
panic!("expected maps");
};
assert!(std::ptr::eq(
- as_struct(&first["k1"]).fields(),
- as_struct(&last["k4"]).fields()
+ as_struct(&first[0].1).fields(),
+ as_struct(&last[0].1).fields()
));
}
@@ -3922,28 +3894,31 @@ mod arrow_value_tests {
}
#[test]
- fn map_key_to_string_covers_scalar_arms() {
- assert_eq!(map_key_to_string(Value::String("x".to_string())), "x");
- assert_eq!(map_key_to_string(Value::Bool(true)), "true");
- assert_eq!(map_key_to_string(Value::Byte(1)), "1");
- assert_eq!(map_key_to_string(Value::Short(2)), "2");
- assert_eq!(map_key_to_string(Value::Integer(3)), "3");
- assert_eq!(map_key_to_string(Value::Long(4)), "4");
- assert_eq!(map_key_to_string(Value::Float(1.5)), "1.5");
- assert_eq!(map_key_to_string(Value::Double(2.5)), "2.5");
- assert_eq!(map_key_to_string(Value::Date(5)), "5");
- assert_eq!(map_key_to_string(Value::Timestamp(6)), "6");
- assert_eq!(map_key_to_string(Value::DayTimeInterval(-1)), "-1");
+ fn a_map_keeps_its_entries_in_order_with_keys_of_their_type() {
+ let entries = StructArray::from(vec![
+ (
+ Arc::new(Field::new("key", arrow::datatypes::DataType::Int32,
false)),
+ Arc::new(Int32Array::from(vec![2, 1])) as ArrayRef,
+ ),
+ (
+ Arc::new(Field::new("value", arrow::datatypes::DataType::Utf8,
true)),
+ Arc::new(StringArray::from(vec!["b", "a"])) as ArrayRef,
+ ),
+ ]);
+ let map = MapArray::new(
+ Arc::new(Field::new("entries", entries.data_type().clone(),
false)),
+ arrow::buffer::OffsetBuffer::new(vec![0, 2].into()),
+ entries,
+ None,
+ false,
+ );
assert_eq!(
- map_key_to_string(Value::Decimal {
- value: "7.5".to_string(),
- precision: None,
- scale: None,
- }),
- "7.5"
+ arrow_value_at(&map, 0).unwrap(),
+ Value::Map(vec![
+ (Value::Integer(2), Value::String("b".to_string())),
+ (Value::Integer(1), Value::String("a".to_string())),
+ ])
);
- // Non-scalar key falls back to the Debug form.
- let _ = map_key_to_string(Value::List(vec![]));
}
#[test]
diff --git a/crates/spark-connect/src/row.rs b/crates/spark-connect/src/row.rs
index a817b04..38dab85 100644
--- a/crates/spark-connect/src/row.rs
+++ b/crates/spark-connect/src/row.rs
@@ -3,7 +3,6 @@
//! A Row represents a single record: an ordered collection of (field_name,
value) pairs.
//! Values are accessed by index or by field name. Supports all Spark SQL
value types.
-use std::collections::BTreeMap;
use std::fmt;
use std::sync::Arc;
@@ -46,8 +45,9 @@ pub enum Value {
},
/// Array of values
List(Vec<Value>),
- /// Map of key-value pairs
- Map(BTreeMap<String, Value>),
+ /// Map entries, in their order, with keys and values of their own types,
as PySpark
+ /// reads a map into a dict.
+ Map(Vec<(Value, Value)>),
/// Struct (nested Row), with ordered names and values.
/// Construct with `Value::Struct(Row::new(fields, values).into())`.
Struct(StructValue),
@@ -756,8 +756,7 @@ mod tests {
Value::List(vec![Value::Integer(1),
Value::Integer(2)]).to_string(),
"[1, 2]"
);
- let mut m = std::collections::BTreeMap::new();
- m.insert("k".to_string(), Value::Integer(9));
+ let m = vec![(Value::String("k".to_string()), Value::Integer(9))];
assert_eq!(Value::Map(m).to_string(), "{k: 9}");
assert_eq!(
Value::Struct(Row::new(vec!["a".to_string()],
vec![Value::Integer(1)]).into())
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index eeff006..a66366f 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -1046,7 +1046,11 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
Value::Map(m) => {
valid.append(true);
for (i, f) in fields.iter().enumerate() {
-
cols[i].push(m.get(&f.name).cloned().unwrap_or(Value::Null));
+ let value = m.iter().find_map(|(key, value)| match
key {
+ Value::String(key) if *key == f.name =>
Some(value.clone()),
+ _ => None,
+ });
+ cols[i].push(value.unwrap_or(Value::Null));
}
}
_ => return Err(SparkError::connect_msg("Type mismatch:
expected struct")),
@@ -1092,7 +1096,7 @@ fn values_to_arrow(values: &[Value], dt: &DataType) ->
Result<Arc<dyn arrow::arr
Value::Map(m) => {
valid.append(true);
for (k, val) in m {
- keys.push(Value::String(k.clone()));
+ keys.push(k.clone());
vals.push(val.clone());
}
offsets.push(keys.len() as i32);
@@ -1203,13 +1207,10 @@ mod tests {
)
.into(),
),
- Value::Map(
- [
- ("dup".to_string(), Value::Long(5)),
- ("other".to_string(), Value::Long(9)),
- ]
- .into(),
- ),
+ Value::Map(vec![
+ (Value::String("dup".to_string()), Value::Long(5)),
+ (Value::String("other".to_string()), Value::Long(9)),
+ ]),
Value::Null,
Value::Struct(Row::new(vec!["dup".to_string()],
vec![Value::Null]).into()),
];
@@ -1285,7 +1286,7 @@ mod tests {
vec!["items".to_string(), "lookup".to_string()],
vec![
Value::List(vec![leaf_value(), Value::Null]),
- Value::Map([("k".to_string(), leaf_value())].into()),
+ Value::Map(vec![(Value::String("k".to_string()),
leaf_value())]),
],
)
.into(),
diff --git a/crates/spark-connect/tests/e2e_integration.rs
b/crates/spark-connect/tests/e2e_integration.rs
index 9b1e048..0a81f1b 100644
--- a/crates/spark-connect/tests/e2e_integration.rs
+++ b/crates/spark-connect/tests/e2e_integration.rs
@@ -657,9 +657,10 @@ fn test_day_time_interval_real_server() {
);
assert_eq!(
row.get(5),
- Some(&Value::Map(
- [("k".to_string(), Value::DayTimeInterval(micros))].into()
- ))
+ Some(&Value::Map(vec![(
+ Value::String("k".to_string()),
+ Value::DayTimeInterval(micros)
+ )]))
);
for prefetch in [false, true] {
let streamed: Vec<_> = frame
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 594dc12..ceb4569 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -361,3 +361,23 @@ def test_sql_runs_statements_eagerly(session):
def test_sql_fails_at_a_statement_that_does_not_parse(session):
with pytest.raises(AnalysisException, match="PARSE_SYNTAX_ERROR"):
session.sql("SELEC 1")
+
+
+def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
+ row = session.range(1).selectExpr(
+ "map(2, 'b', 1, 'a') AS ints",
+ "map('b', 1, 'a', 2) AS strs",
+ "map(date'2020-01-02', 1) AS dates",
+ "map(named_struct('a', 1), 'x') AS structs",
+ ).first()
+ assert list(row.ints.items()) == [(2, "b"), (1, "a")]
+ assert list(row.strs.items()) == [("b", 1), ("a", 2)]
+ assert row.dates == {date(2020, 1, 2): 1}
+ assert row.structs == {Row(a=1): "x"}
+ # A list key is unhashable, as in PySpark. (PySpark raises at collect();
this client
+ # turns a row's values into Python ones when they are read.)
+ with pytest.raises(TypeError, match="unhashable"):
+ session.range(1).selectExpr("map(array(1), 'x') AS m").first().m
+ df = session.createDataFrame([({2: "b", 1: "a"},)])
+ assert df.schema.simpleString() == "struct<_1:map<bigint,string>>"
+ assert list(df.first()[0].items()) == [(2, "b"), (1, "a")]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]