This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 3a887869 [SPARK-61225][CONNECT][PYTHON] Collect maps whose keys have a 
user-defined type with their deserialized keys, as PySpark does
3a887869 is described below

commit 3a887869ace0c8c91b23400ee655de5791b3ed81
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Mon Nov 9 17:31:02 2026 +0900

    [SPARK-61225][CONNECT][PYTHON] Collect maps whose keys have a user-defined 
type with their deserialized keys, as PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61225](https://issues.apache.org/jira/browse/SPARK-61225): Collect 
maps whose keys have a user-defined type with their deserialized keys, as 
PySpark does.
    
    - The Rust client built the Python dict of a map from the raw storage key 
before the UDT converter ran. For a key of a user-defined type the storage key 
is a list, which cannot be a dict key, so `collect()`, `toLocalIterator()`, 
`head()`, `first()` and `take()` raised `TypeError: unhashable type: 'list'`.
    - A map whose key type has a user-defined type (at any depth, also in 
arrays, structs and map values) is now passed to the converter as its (key, 
value) pairs. `pyspark.sql.types._udt_converter` deserializes each key and 
builds the dict, as PySpark does, so `{(1.0, 2.0): 1}` is returned.
    - Maps without a user-defined key are unchanged, and a plain array-keyed 
map still raises the same `TypeError` as PySpark.
    
    ### Why are the changes needed?
    
    A DataFrame whose map key is a user-defined type cannot be collected. For 
example, with `MapType(PointUDT(), IntegerType())` where `PointUDT` serializes 
to a list and deserializes to a tuple, `df.collect()` raises `TypeError: 
unhashable type: 'list'`. PySpark returns `[Row(m={(1.0, 2.0): 1})]`. Map 
values of a user-defined type already work.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Collecting a map whose keys have a user-defined type returns the 
deserialized keys, as PySpark does, instead of raising `TypeError`.
    
    ### How was this patch tested?
    
    - `test_map_with_user_defined_keys_is_built_from_the_deserialized_keys` 
(offline): the converter of a map with user-defined keys builds the dict from 
the (key, value) pairs with the keys deserialized, and handles None and empty 
maps.
    - 
`test_maps_with_user_defined_keys_collect_their_deserialized_keys_as_pyspark` 
(live): a map with user-defined keys at the top level, in a struct, in an array 
and in a map value, through `collect`, `toLocalIterator`, `head`, `first` and 
`take`, and a plain array-keyed map still raising `TypeError`. The same results 
come from official PySpark 4.3.0.
    - The lib tests of both crates (658 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2704 passed, 8 skipped), and the 
live drop-in and Arrow suites (225 passed) and live cases (12 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #407 from HyukjinKwon/SPARK-61225-fu-p5.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/dataframe.rs  |  4 +++
 crates/pyspark-rs/src/row.rs        | 69 ++++++++++++++++++++++++++++++++++++-
 python/pyspark/sql/types.py         | 11 +++++-
 python/tests/test_dropin_e2e.py     | 55 +++++++++++++++++++++++++++++
 python/tests/test_dropin_offline.py | 17 +++++++++
 5 files changed, 154 insertions(+), 2 deletions(-)

diff --git a/crates/pyspark-rs/src/dataframe.rs 
b/crates/pyspark-rs/src/dataframe.rs
index b7244bd1..426538a5 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -4320,6 +4320,9 @@ fn rows_to_py(
     };
     let fields = PyList::new(py, first.fields())?;
     let mut nested_fields = NestedFieldNames::default();
+    if let Some(schema) = schema {
+        nested_fields.note_udt_key_maps(schema);
+    }
     rows.iter()
         .map(|row| {
             crate::row::row_values_to_py(
@@ -4397,6 +4400,7 @@ impl PyLocalRowIterator {
                 // The result is known once its first message arrives, 
possibly with no rows.
                 check_collectable(py, Some(&schema))?;
                 self.converters = Some(udt_field_converters(py, 
Some(&schema))?);
+                self.nested_fields.note_udt_key_maps(&schema);
             }
         }
         match next {
diff --git a/crates/pyspark-rs/src/row.rs b/crates/pyspark-rs/src/row.rs
index 0fbf771a..1c942cc1 100644
--- a/crates/pyspark-rs/src/row.rs
+++ b/crates/pyspark-rs/src/row.rs
@@ -1,6 +1,6 @@
 //! Conversion of native Spark values into tuple-backed Python Rows.
 
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
 
 use pyo3::prelude::*;
@@ -41,6 +41,8 @@ enum NestedFieldPath {
 pub(crate) struct NestedFieldNames {
     path: Vec<NestedFieldPath>,
     fields: HashMap<Vec<NestedFieldPath>, Py<PyList>>,
+    /// The schema paths of the maps whose keys have a user-defined type.
+    udt_key_maps: HashSet<Vec<NestedFieldPath>>,
 }
 
 impl NestedFieldNames {
@@ -48,6 +50,14 @@ impl NestedFieldNames {
     pub(crate) fn clear(&mut self) {
         self.path.clear();
         self.fields.clear();
+        self.udt_key_maps.clear();
+    }
+
+    /// Record the maps of `schema` whose keys have a user-defined type. Their 
keys are
+    /// deserialized by the converters, so such a map is passed on as its 
(key, value) pairs.
+    pub(crate) fn note_udt_key_maps(&mut self, schema: 
&spark_connect::types::DataType) {
+        let mut path = Vec::new();
+        collect_udt_key_maps(schema, &mut path, &mut self.udt_key_maps);
     }
 
     /// Convert one schema step below this path, restoring it even if 
conversion fails.
@@ -80,6 +90,50 @@ impl NestedFieldNames {
     }
 }
 
+/// Collect the paths of the maps in `data_type` whose keys have a 
user-defined type, along
+/// the same steps that `value_to_py` takes through the values.
+fn collect_udt_key_maps(
+    data_type: &spark_connect::types::DataType,
+    path: &mut Vec<NestedFieldPath>,
+    out: &mut HashSet<Vec<NestedFieldPath>>,
+) {
+    use spark_connect::types::DataType;
+    match data_type {
+        DataType::Struct { fields } => {
+            for (index, field) in fields.iter().enumerate() {
+                path.push(NestedFieldPath::Field(index));
+                collect_udt_key_maps(&field.data_type, path, out);
+                path.pop();
+            }
+        }
+        DataType::Array { element_type, .. } => {
+            path.push(NestedFieldPath::ArrayElement);
+            collect_udt_key_maps(element_type, path, out);
+            path.pop();
+        }
+        DataType::Map {
+            key_type,
+            value_type,
+            ..
+        } => {
+            if crate::types::has_udt(key_type) {
+                out.insert(path.clone());
+            }
+            path.push(NestedFieldPath::MapKey);
+            collect_udt_key_maps(key_type, path, out);
+            path.pop();
+            path.push(NestedFieldPath::MapValue);
+            collect_udt_key_maps(value_type, path, out);
+            path.pop();
+        }
+        DataType::Udt {
+            sql_type: Some(sql_type),
+            ..
+        } => collect_udt_key_maps(sql_type, path, out),
+        _ => {}
+    }
+}
+
 /// Build a Row from named values, deserializing user-defined fields with 
`converters`.
 /// The caller shares `fields` across result rows and nested names across the 
execution.
 pub(crate) fn row_values_to_py(
@@ -248,6 +302,19 @@ fn value_to_py<'py>(
             }
             list.into_any()
         }
+        Value::Map(m) if 
nested_fields.udt_key_maps.contains(&nested_fields.path) => {
+            // The keys are deserialized from their user-defined type by the 
converter of the
+            // field, which builds the dict, so the entries are passed on as 
pairs.
+            let pairs = PyList::empty(py);
+            for (k, val) in m {
+                let key =
+                    nested_fields.value_at(py, k, binary_as_bytes, 
NestedFieldPath::MapKey)?;
+                let val =
+                    nested_fields.value_at(py, val, binary_as_bytes, 
NestedFieldPath::MapValue)?;
+                pairs.append(PyTuple::new(py, [key, val])?)?;
+            }
+            pairs.into_any()
+        }
         Value::Map(m) => {
             // Keys of their own type, in order; an unhashable key, such as an 
array, raises
             // TypeError, as in PySpark.
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 09f1df76..d671e80b 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -610,7 +610,16 @@ def _udt_converter(dataType):
             return None
         key = key or (lambda v: v)
         val = val or (lambda v: v)
-        return lambda value: None if value is None else {key(k): val(v) for k, 
v in value.items()}
+
+        def convert_map(value):
+            if value is None:
+                return None
+            # The Rust client passes a map whose keys have a user-defined type 
as its pairs,
+            # so that the keys are deserialized before the dict is built.
+            entries = value.items() if isinstance(value, dict) else value
+            return {key(k): val(v) for k, v in entries}
+
+        return convert_map
     if isinstance(dataType, StructType):
         fields = _udt_field_converters(dataType)
         if fields is None:
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index d68e38ea..9cf910e7 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -37,6 +37,7 @@ from pyspark.sql.types import (
     DoubleType,
     IntegerType,
     LongType,
+    MapType,
     StringType,
     StructField,
     StructType,
@@ -3118,6 +3119,60 @@ def 
test_pandas_frames_of_duplicate_nested_field_names_are_created_as_pyspark(se
 
 
 
+def 
test_maps_with_user_defined_keys_collect_their_deserialized_keys_as_pyspark(session):
+    # As PySpark: the keys of a map of a user-defined type are deserialized, 
at the top level,
+    # in a struct, in an array and in the value of a map, for collect, 
toLocalIterator, head,
+    # first and take.
+    from pyspark.sql.types import UserDefinedType
+
+    class PairUDT(UserDefinedType):
+        @classmethod
+        def sqlType(cls):
+            return ArrayType(DoubleType(), False)
+
+        @classmethod
+        def module(cls):
+            return __name__
+
+        def serialize(self, obj):
+            return [obj[0], obj[1]]
+
+        def deserialize(self, datum):
+            return (datum[0], datum[1])
+
+        def __eq__(self, other):
+            return isinstance(other, PairUDT)
+
+        def __hash__(self):
+            return 1
+
+    pair = PairUDT()
+    schema = StructType([
+        StructField("m", MapType(pair, IntegerType())),
+        StructField("s", StructType([StructField("k", MapType(pair, 
StringType()))])),
+        StructField("a", ArrayType(MapType(pair, IntegerType()))),
+        StructField("v", MapType(StringType(), MapType(pair, IntegerType()))),
+    ])
+    expected = Row(
+        m={(1.0, 2.0): 1, (3.0, 4.0): None},
+        s=Row(k={(1.0, 2.0): "x"}),
+        a=[{(5.0, 6.0): 7}],
+        v={"a": {(1.0, 2.0): 3}},
+    )
+    df = session.createDataFrame(
+        [({(1.0, 2.0): 1, (3.0, 4.0): None}, {"k": {(1.0, 2.0): "x"}}, [{(5.0, 
6.0): 7}],
+          {"a": {(1.0, 2.0): 3}})],
+        schema,
+    )
+    assert df.collect() == [expected]
+    assert list(df.toLocalIterator()) == [expected]
+    assert df.head() == df.first() == expected
+    assert df.take(1) == [expected]
+    # A list key is unhashable, as in PySpark, also in a map with user-defined 
keys.
+    with pytest.raises(TypeError, match="unhashable"):
+        session.sql("SELECT map(array(1), 'x') AS m").collect()
+
+
 def test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
     row = session.range(1).selectExpr(
         "map(2, 'b', 1, 'a') AS ints",
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 388993a8..dbd7f181 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -12323,6 +12323,23 @@ def 
test_numpy_scalars_of_other_types_are_unsupported_as_pyspark(mock_connect):
     assert error.value.getMessageParameters() == {"dtype": "uint8"}
 
 
+def test_map_with_user_defined_keys_is_built_from_the_deserialized_keys():
+    # The Rust client passes a map whose keys have a user-defined type as its 
(key, value) pairs,
+    # so that the converter deserializes the keys before it builds the dict, 
as PySpark does.
+    from pyspark.sql.types import _udt_field_converters
+
+    schema = T.StructType([
+        T.StructField("m", T.MapType(_DayPointUDT(), T.IntegerType())),
+        T.StructField("n", T.MapType(T.StringType(), _DayPointUDT())),
+    ])
+    m, n = _udt_field_converters(schema)
+    assert m([([1.0, 2.0], 1), ([3.0, 4.0], None)]) == {(1.0, 2.0): 1, (3.0, 
4.0): None}
+    assert m(None) is None
+    assert m([]) == {}
+    assert n({"x": [1.0, 2.0]}) == {"x": (1.0, 2.0)}
+    assert n({}) == {}
+
+
 def test_numpy_longdouble_scalar_matches_dtype_and_class_checks_as_pyspark():
     np = pytest.importorskip("numpy")
     from pyspark.errors import PySparkTypeError


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to