This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 3a887869 [SPARK-61225][CONNECT][PYTHON] Collect maps whose keys have a
user-defined type with their deserialized keys, as PySpark does
3a887869 is described below
commit 3a887869ace0c8c91b23400ee655de5791b3ed81
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Mon Nov 9 17:31:02 2026 +0900
[SPARK-61225][CONNECT][PYTHON] Collect maps whose keys have a user-defined
type with their deserialized keys, as PySpark does
### What changes were proposed in this pull request?
[SPARK-61225](https://issues.apache.org/jira/browse/SPARK-61225): Collect
maps whose keys have a user-defined type with their deserialized keys, as
PySpark does.
- The Rust client built the Python dict of a map from the raw storage key
before the UDT converter ran. For a key of a user-defined type the storage key
is a list, which cannot be a dict key, so `collect()`, `toLocalIterator()`,
`head()`, `first()` and `take()` raised `TypeError: unhashable type: 'list'`.
- A map whose key type has a user-defined type (at any depth, also in
arrays, structs and map values) is now passed to the converter as its (key,
value) pairs. `pyspark.sql.types._udt_converter` deserializes each key and
builds the dict, as PySpark does, so `{(1.0, 2.0): 1}` is returned.
- Maps without a user-defined key are unchanged, and a plain array-keyed
map still raises the same `TypeError` as PySpark.
### Why are the changes needed?
A DataFrame whose map key is a user-defined type cannot be collected. For
example, with `MapType(PointUDT(), IntegerType())` where `PointUDT` serializes
to a list and deserializes to a tuple, `df.collect()` raises `TypeError:
unhashable type: 'list'`. PySpark returns `[Row(m={(1.0, 2.0): 1})]`. Map
values of a user-defined type already work.
### Does this PR introduce _any_ user-facing change?
Yes. Collecting a map whose keys have a user-defined type returns the
deserialized keys, as PySpark does, instead of raising `TypeError`.
### How was this patch tested?
- `test_map_with_user_defined_keys_is_built_from_the_deserialized_keys`
(offline): the converter of a map with user-defined keys builds the dict from
the (key, value) pairs with the keys deserialized, and handles None and empty
maps.
-
`test_maps_with_user_defined_keys_collect_their_deserialized_keys_as_pyspark`
(live): a map with user-defined keys at the top level, in a struct, in an array
and in a map value, through `collect`, `toLocalIterator`, `head`, `first` and
`take`, and a plain array-keyed map still raising `TypeError`. The same results
come from official PySpark 4.3.0.
- The lib tests of both crates (658 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2704 passed, 8 skipped), and the
live drop-in and Arrow suites (225 passed) and live cases (12 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #407 from HyukjinKwon/SPARK-61225-fu-p5.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 4 +++
crates/pyspark-rs/src/row.rs | 69 ++++++++++++++++++++++++++++++++++++-
python/pyspark/sql/types.py | 11 +++++-
python/tests/test_dropin_e2e.py | 55 +++++++++++++++++++++++++++++
python/tests/test_dropin_offline.py | 17 +++++++++
5 files changed, 154 insertions(+), 2 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index b7244bd1..426538a5 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -4320,6 +4320,9 @@ fn rows_to_py(
};
let fields = PyList::new(py, first.fields())?;
let mut nested_fields = NestedFieldNames::default();
+ if let Some(schema) = schema {
+ nested_fields.note_udt_key_maps(schema);
+ }
rows.iter()
.map(|row| {
crate::row::row_values_to_py(
@@ -4397,6 +4400,7 @@ impl PyLocalRowIterator {
// The result is known once its first message arrives,
possibly with no rows.
check_collectable(py, Some(&schema))?;
self.converters = Some(udt_field_converters(py,
Some(&schema))?);
+ self.nested_fields.note_udt_key_maps(&schema);
}
}
match next {
diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 0fbf771a..1c942cc1 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -1,6 +1,6 @@
//! Conversion of native Spark values into tuple-backed Python Rows.
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use pyo3::prelude::*;
@@ -41,6 +41,8 @@ enum NestedFieldPath {
pub(crate) struct NestedFieldNames {
path: Vec<NestedFieldPath>,
fields: HashMap<Vec<NestedFieldPath>, Py<PyList>>,
+ /// The schema paths of the maps whose keys have a user-defined type.
+ udt_key_maps: HashSet<Vec<NestedFieldPath>>,
}
impl NestedFieldNames {
@@ -48,6 +50,14 @@ impl NestedFieldNames {
pub(crate) fn clear(&mut self) {
self.path.clear();
self.fields.clear();
+ self.udt_key_maps.clear();
+ }
+
+ /// Record the maps of `schema` whose keys have a user-defined type. Their
keys are
+ /// deserialized by the converters, so such a map is passed on as its
(key, value) pairs.
+ pub(crate) fn note_udt_key_maps(&mut self, schema:
&spark_connect::types::DataType) {
+ let mut path = Vec::new();
+ collect_udt_key_maps(schema, &mut path, &mut self.udt_key_maps);
}
/// Convert one schema step below this path, restoring it even if
conversion fails.
@@ -80,6 +90,50 @@ impl NestedFieldNames {
}
}
+/// Collect the paths of the maps in `data_type` whose keys have a
user-defined type, along
+/// the same steps that `value_to_py` takes through the values.
+fn collect_udt_key_maps(
+ data_type: &spark_connect::types::DataType,
+ path: &mut Vec<NestedFieldPath>,
+ out: &mut HashSet<Vec<NestedFieldPath>>,
+) {
+ use spark_connect::types::DataType;
+ match data_type {
+ DataType::Struct { fields } => {
+ for (index, field) in fields.iter().enumerate() {
+ path.push(NestedFieldPath::Field(index));
+ collect_udt_key_maps(&field.data_type, path, out);
+ path.pop();
+ }
+ }
+ DataType::Array { element_type, .. } => {
+ path.push(NestedFieldPath::ArrayElement);
+ collect_udt_key_maps(element_type, path, out);
+ path.pop();
+ }
+ DataType::Map {
+ key_type,
+ value_type,
+ ..
+ } => {
+ if crate::types::has_udt(key_type) {
+ out.insert(path.clone());
+ }
+ path.push(NestedFieldPath::MapKey);
+ collect_udt_key_maps(key_type, path, out);
+ path.pop();
+ path.push(NestedFieldPath::MapValue);
+ collect_udt_key_maps(value_type, path, out);
+ path.pop();
+ }
+ DataType::Udt {
+ sql_type: Some(sql_type),
+ ..
+ } => collect_udt_key_maps(sql_type, path, out),
+ _ => {}
+ }
+}
+
/// Build a Row from named values, deserializing user-defined fields with
`converters`.
/// The caller shares `fields` across result rows and nested names across the
execution.
pub(crate) fn row_values_to_py(
@@ -248,6 +302,19 @@ fn value_to_py<'py>(
}
list.into_any()
}
+ Value::Map(m) if
nested_fields.udt_key_maps.contains(&nested_fields.path) => {
+ // The keys are deserialized from their user-defined type by the
converter of the
+ // field, which builds the dict, so the entries are passed on as
pairs.
+ let pairs = PyList::empty(py);
+ for (k, val) in m {
+ let key =
+ nested_fields.value_at(py, k, binary_as_bytes,
NestedFieldPath::MapKey)?;
+ let val =
+ nested_fields.value_at(py, val, binary_as_bytes,
NestedFieldPath::MapValue)?;
+ pairs.append(PyTuple::new(py, [key, val])?)?;
+ }
+ pairs.into_any()
+ }
Value::Map(m) => {
// Keys of their own type, in order; an unhashable key, such as an
array, raises
// TypeError, as in PySpark.
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 09f1df76..d671e80b 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -610,7 +610,16 @@ def _udt_converter(dataType):
return None
key = key or (lambda v: v)
val = val or (lambda v: v)
- return lambda value: None if value is None else {key(k): val(v) for k,
v in value.items()}
+
+ def convert_map(value):
+ if value is None:
+ return None
+ # The Rust client passes a map whose keys have a user-defined type
as its pairs,
+ # so that the keys are deserialized before the dict is built.
+ entries = value.items() if isinstance(value, dict) else value
+ return {key(k): val(v) for k, v in entries}
+
+ return convert_map
if isinstance(dataType, StructType):
fields = _udt_field_converters(dataType)
if fields is None:
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d68e38ea..9cf910e7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -37,6 +37,7 @@ from pyspark.sql.types import (
DoubleType,
IntegerType,
LongType,
+ MapType,
StringType,
StructField,
StructType,
@@ -3118,6 +3119,60 @@ def
test_pandas_frames_of_duplicate_nested_field_names_are_created_as_pyspark(se
+def
test_maps_with_user_defined_keys_collect_their_deserialized_keys_as_pyspark(session):
+ # As PySpark: the keys of a map of a user-defined type are deserialized,
at the top level,
+ # in a struct, in an array and in the value of a map, for collect,
toLocalIterator, head,
+ # first and take.
+ from pyspark.sql.types import UserDefinedType
+
+ class PairUDT(UserDefinedType):
+ @classmethod
+ def sqlType(cls):
+ return ArrayType(DoubleType(), False)
+
+ @classmethod
+ def module(cls):
+ return __name__
+
+ def serialize(self, obj):
+ return [obj[0], obj[1]]
+
+ def deserialize(self, datum):
+ return (datum[0], datum[1])
+
+ def __eq__(self, other):
+ return isinstance(other, PairUDT)
+
+ def __hash__(self):
+ return 1
+
+ pair = PairUDT()
+ schema = StructType([
+ StructField("m", MapType(pair, IntegerType())),
+ StructField("s", StructType([StructField("k", MapType(pair,
StringType()))])),
+ StructField("a", ArrayType(MapType(pair, IntegerType()))),
+ StructField("v", MapType(StringType(), MapType(pair, IntegerType()))),
+ ])
+ expected = Row(
+ m={(1.0, 2.0): 1, (3.0, 4.0): None},
+ s=Row(k={(1.0, 2.0): "x"}),
+ a=[{(5.0, 6.0): 7}],
+ v={"a": {(1.0, 2.0): 3}},
+ )
+ df = session.createDataFrame(
+ [({(1.0, 2.0): 1, (3.0, 4.0): None}, {"k": {(1.0, 2.0): "x"}}, [{(5.0,
6.0): 7}],
+ {"a": {(1.0, 2.0): 3}})],
+ schema,
+ )
+ assert df.collect() == [expected]
+ assert list(df.toLocalIterator()) == [expected]
+ assert df.head() == df.first() == expected
+ assert df.take(1) == [expected]
+ # A list key is unhashable, as in PySpark, also in a map with user-defined
keys.
+ with pytest.raises(TypeError, match="unhashable"):
+ session.sql("SELECT map(array(1), 'x') AS m").collect()
+
+
def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
row = session.range(1).selectExpr(
"map(2, 'b', 1, 'a') AS ints",
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 388993a8..dbd7f181 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -12323,6 +12323,23 @@ def
test_numpy_scalars_of_other_types_are_unsupported_as_pyspark(mock_connect):
assert error.value.getMessageParameters() == {"dtype": "uint8"}
+def test_map_with_user_defined_keys_is_built_from_the_deserialized_keys():
+ # The Rust client passes a map whose keys have a user-defined type as its
(key, value) pairs,
+ # so that the converter deserializes the keys before it builds the dict,
as PySpark does.
+ from pyspark.sql.types import _udt_field_converters
+
+ schema = T.StructType([
+ T.StructField("m", T.MapType(_DayPointUDT(), T.IntegerType())),
+ T.StructField("n", T.MapType(T.StringType(), _DayPointUDT())),
+ ])
+ m, n = _udt_field_converters(schema)
+ assert m([([1.0, 2.0], 1), ([3.0, 4.0], None)]) == {(1.0, 2.0): 1, (3.0,
4.0): None}
+ assert m(None) is None
+ assert m([]) == {}
+ assert n({"x": [1.0, 2.0]}) == {"x": (1.0, 2.0)}
+ assert n({}) == {}
+
+
def test_numpy_longdouble_scalar_matches_dtype_and_class_checks_as_pyspark():
np = pytest.importorskip("numpy")
from pyspark.errors import PySparkTypeError
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]