This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new d08f303  [SPARK-60933][CONNECT] Make every DataType pickle round-trip 
in the Rust drop-in
d08f303 is described below

commit d08f30347c2df2c66edd08802ee77e9cce7a4285
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Fri Nov 6 07:06:26 2026 +0900

    [SPARK-60933][CONNECT] Make every DataType pickle round-trip in the Rust 
drop-in
    
    ### What changes were proposed in this pull request?
    
    In the Rust-backed PySpark drop-in, some data types did not survive 
`pickle.loads(pickle.dumps(t))`, although every `DataType` must be picklable so 
that it reaches an official-pyspark UDF worker. This PR fixes them:
    
    - `pyspark.sql.types._parse_datatype_json_value` now parses the JSON string 
forms (atomics, `decimal(p,s)`, `string collate X`, intervals, 
`geometry(<crs>)`, `geography(<crs>, <alg>)`, ...) with the Rust core's 
`DataType::from_json`, through a new `_pyspark.parse_datatype_json`. Before, it 
kept a second grammar in Python that knew neither collations nor the spatial 
forms. The nested struct/array/map objects still recurse in Python, because a 
`udt` field has to rebuild its Python class.
    - `GeometryType` and `GeographyType` get `__reduce__`, as the other type 
classes have.
    - The base `DataType` gets a `__reduce__` fallback, built from 
`self.json()`, for a bare unparsed `DataType` and a pure-Python 
`UserDefinedType`.
    - The core JSON parser reads `"unparsed(x)"` back as `x`; it used to read 
it as `unparsed(unparsed(x))`. It also accepts `"decimal"`, `"null"` and 
`"calendarinterval"`, which the Python parser accepted before.
    - `data_type_to_py` returns a bare `DataType` for an unparsed type instead 
of calling back into the Python parser, which now sends JSON strings back to 
Rust and would loop.
    
    ### Why are the changes needed?
    
    To match PySpark. Before this change:
    
    ```python
    >>> import pickle
    >>> from pyspark.sql.types import *
    >>> pickle.dumps(GeometryType(4326))
    TypeError: cannot pickle 'pyspark.sql.types.GeometryType' object
    >>> pickle.loads(pickle.dumps(StringType("UTF8_LCASE")))
    ValueError: cannot parse datatype json: 'string collate UTF8_LCASE'
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Spatial types, collated strings and every type nested in a 
`StructType`, `ArrayType` or `MapType` now pickle, as in PySpark, so they can 
be a UDF's return type or appear in a schema sent to a worker.
    
    ### How was this patch tested?
    
    - New offline tests in `python/tests/test_dropin_offline.py`: every data 
type variant (atomic, parameterized, intervals, collated strings, each 
supported spatial SRID, nested array/map) round-trips through pickle with the 
same class, `==`, `hash` and JSON, alone and nested in a `StructType` with 
field metadata. Further tests cover parsing the spatial and collated JSON 
forms, an unparsed type, and a UDT nested next to a spatial type.
    - New Rust unit tests for the parser aliases and the `unparsed(x)` round 
trip.
    - `cargo fmt --all --check`, `cargo test -p apache-spark-connect --release 
--lib`, and `cargo test -p apache-spark-connect --release [--features wasm-udf] 
--no-run` pass, and clippy reports nothing in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes.
    
    Closes #162 from viirya/SPARK-60933.
    
    Authored-by: Liang-Chi Hsieh <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/lib.rs        |  3 ++
 crates/pyspark-rs/src/types.rs      | 33 ++++++++++++-
 crates/spark-connect/src/types.rs   | 38 ++++++++++++--
 python/pyspark/sql/types.py         | 55 +++++----------------
 python/tests/test_dropin_offline.py | 98 +++++++++++++++++++++++++++++++++++++
 5 files changed, 179 insertions(+), 48 deletions(-)

diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index 29584de..8f2ab6b 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -125,6 +125,9 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) -> 
PyResult<()> {
     m.add_class::<PyStatFunctions>()?;
     m.add_class::<storagelevel::PyStorageLevel>()?;
 
+    // The core DataType JSON grammar, which `pyspark.sql.types` parses type 
JSON with.
+    m.add_function(wrap_pyfunction!(types::parse_datatype_json, m)?)?;
+
     // Spark Declarative Pipelines (SDP) command execution
     m.add_class::<pipelines::PyPipelineRunStream>()?;
     m.add_function(wrap_pyfunction!(
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 17f4586..4eab163 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -146,6 +146,14 @@ impl PyDataType {
         }
     }
 
+    /// The fallback for the classes without their own `__reduce__`: a bare 
`DataType`
+    /// (an unparsed DDL type) and a pure-Python `UserDefinedType`, whose 
`json()` names
+    /// its Python class.
+    fn __reduce__(slf: &Bound<'_, Self>) -> PyResult<(Py<PyAny>, (String,))> {
+        let json: String = slf.call_method0("json")?.extract()?;
+        type_reduce(slf.py(), &json)
+    }
+
     // --- DataType object-model methods (v4.2.0 parity) ---
     #[pyo3(name = "json")]
     fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2447,6 +2455,9 @@ impl PyGeometryType {
     fn srid(&self) -> i32 {
         self.srid
     }
+    fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
+        type_reduce(py, &DataType::Geometry { srid: self.srid }.json())
+    }
     fn __repr__(&self) -> String {
         match self.srid {
             -1 => "GeometryType(ANY)".to_string(),
@@ -2496,6 +2507,9 @@ impl PyGeographyType {
     fn srid(&self) -> i32 {
         self.srid
     }
+    fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
+        type_reduce(py, &DataType::Geography { srid: self.srid }.json())
+    }
     fn __repr__(&self) -> String {
         match self.srid {
             -1 => "GeographyType(ANY)".to_string(),
@@ -2544,6 +2558,18 @@ fn spatial_type_srid(srid: &Bound<'_, PyAny>, 
geographic: bool) -> PyResult<i32>
     ))
 }
 
+/// Parse a non-UDT DataType's Spark JSON (e.g. `"string collate UTF8_LCASE"`,
+/// `"geography(OGC:CRS84, SPHERICAL)"`, or a nested struct/array/map object) 
with the core
+/// grammar, into its concrete Python type object. 
`pyspark.sql.types._parse_datatype_json_value`
+/// delegates its atomic and parameterized string forms here, so the drop-in 
keeps one JSON
+/// grammar (the core's) instead of a second parser in Python.
+#[pyfunction]
+pub fn parse_datatype_json(py: Python<'_>, json: &str) -> PyResult<Py<PyAny>> {
+    let dt = DataType::from_json_str(json)
+        .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))?;
+    data_type_to_py(py, &dt)
+}
+
 /// Convert a Python DataType object (any of the type classes) or a DDL string 
into
 /// the core `DataType`. Mirrors pyspark accepting `Union[DataType, str]` 
everywhere.
 /// Materialize a core [`DataType`] into its concrete Python type object — a 
real
@@ -2781,9 +2807,12 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt: 
&DataType) -> PyResult<Py<PyAn
                 value
             ))
         }
-        DataType::Udt { .. } | DataType::Unparsed { .. } => {
+        // An unparsed DDL string has no concrete Python class; keep it as a 
bare `DataType`.
+        // (The Python parser delegates JSON strings back here, so routing it 
there would loop.)
+        DataType::Unparsed { .. } => Py::new(py, PyDataType { inner: 
dt.clone() })?.into_any(),
+        DataType::Udt { .. } => {
             // A UDT reconstructs by re-importing its Python class; delegate 
to the Python
-            // parser (which also handles the unparsed-DDL fallback).
+            // parser.
             py.import("pyspark.sql.types")?
                 .getattr("_parse_datatype_json_string")?
                 .call1((dt.json(),))?
diff --git a/crates/spark-connect/src/types.rs 
b/crates/spark-connect/src/types.rs
index fb026fd..fd2894b 100644
--- a/crates/spark-connect/src/types.rs
+++ b/crates/spark-connect/src/types.rs
@@ -1112,7 +1112,9 @@ fn parse_json_value(value: &serde_json::Value, 
_field_name: Option<&str>) -> Res
 
 fn parse_json_type_string(s: &str) -> Result<DataType> {
     match s {
-        "void" => Ok(DataType::Null),
+        // `null` and `calendarinterval` are lenient aliases that the 
drop-in's Python parser
+        // accepted before it delegated here.
+        "void" | "null" => Ok(DataType::Null),
         "boolean" => Ok(DataType::Boolean),
         "byte" => Ok(DataType::Byte),
         "short" => Ok(DataType::Short),
@@ -1124,8 +1126,13 @@ fn parse_json_type_string(s: &str) -> Result<DataType> {
         "date" => Ok(DataType::Date),
         "timestamp" => Ok(DataType::Timestamp),
         "timestamp_ntz" => Ok(DataType::TimestampNtz),
-        "interval" => Ok(DataType::CalendarInterval),
+        "interval" | "calendarinterval" => Ok(DataType::CalendarInterval),
         "variant" => Ok(DataType::Variant),
+        // A bare `decimal`, as `DecimalType().typeName()` names it, is the 
default decimal(10,0).
+        "decimal" => Ok(DataType::Decimal {
+            precision: 10,
+            scale: 0,
+        }),
         s if s.starts_with("decimal(") => parse_decimal(s),
         s if s.starts_with("char(") => parse_char(s),
         s if s.starts_with("varchar(") => parse_varchar(s),
@@ -1143,8 +1150,9 @@ fn parse_json_type_string(s: &str) -> Result<DataType> {
         }),
         s if s.starts_with("geometry(") => parse_geometry(s),
         s if s.starts_with("geography(") => parse_geography(s),
-        s if s.starts_with("unparsed(") => Ok(DataType::Unparsed {
-            data_type_string: s.to_string(),
+        // The inverse of `json_value`, which writes 
`unparsed(<data_type_string>)`.
+        s if s.starts_with("unparsed(") && s.ends_with(')') => 
Ok(DataType::Unparsed {
+            data_type_string: s["unparsed(".len()..s.len() - 1].to_string(),
         }),
         _ => Err(SparkError::value(
             "INVALID_DATATYPE_STRING",
@@ -3065,6 +3073,28 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_from_json_unparsed_roundtrip() {
+        let dt = DataType::Unparsed {
+            data_type_string: "int".to_string(),
+        };
+        assert_eq!(DataType::from_json(&dt.json_value()).unwrap(), dt);
+    }
+
+    #[test]
+    fn test_from_json_lenient_aliases() {
+        let parse = |s: &str| 
DataType::from_json(&serde_json::json!(s)).unwrap();
+        assert_eq!(
+            parse("decimal"),
+            DataType::Decimal {
+                precision: 10,
+                scale: 0
+            }
+        );
+        assert_eq!(parse("null"), DataType::Null);
+        assert_eq!(parse("calendarinterval"), DataType::CalendarInterval);
+    }
+
     #[test]
     fn test_from_json_geometry() {
         let json_geom = serde_json::json!("geometry(any)");
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 6fb1d17..c615d23 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -289,50 +289,21 @@ def _parse_datatype_json_string(json_string):
 
 def _parse_datatype_json_value(v):
     """Recursively reconstruct a DataType from a parsed Spark-JSON value (a 
string for
-    atomic/parameterized types, or a dict for the nested struct/array/map 
forms)."""
-    import re as _re
+    atomic/parameterized types, or a dict for the nested struct/array/map 
forms).
+
+    The string forms (atomics, ``decimal(p,s)``, ``string collate X``, 
intervals,
+    ``geometry(<crs>)``, ``geography(<crs>, <alg>)``, ...) are parsed by the 
Rust core's
+    JSON grammar, so the drop-in keeps one grammar. The nested object forms 
recurse here,
+    because a ``udt`` leaf has to reconstruct its Python class."""
+    import json as _json
 
     if isinstance(v, str):
-        atomic = {
-            "void": NullType, "null": NullType, "boolean": BooleanType, 
"byte": ByteType,
-            "short": ShortType, "integer": IntegerType, "long": LongType, 
"float": FloatType,
-            "double": DoubleType, "string": StringType, "binary": BinaryType, 
"date": DateType,
-            "timestamp": TimestampType, "timestamp_ntz": TimestampNTZType, 
"variant": VariantType,
-        }
-        if v in atomic:
-            return atomic[v]()
-        m = _re.match(r"decimal\((\d+),\s*(\d+)\)$", v)
-        if m:
-            return DecimalType(int(m.group(1)), int(m.group(2)))
-        if v == "decimal":
-            return DecimalType()
-        m = _re.match(r"char\((\d+)\)$", v)
-        if m:
-            return CharType(int(m.group(1)))
-        m = _re.match(r"varchar\((\d+)\)$", v)
-        if m:
-            return VarcharType(int(m.group(1)))
-        m = _re.match(r"time\((\d+)\)$", v)
-        if m:
-            return TimeType(int(m.group(1)))
-        m = _re.match(r"timestamp_(ntz|ltz)\(\s*(\d+)\s*\)$", v)
-        if m:
-            cls = TimestampNTZNanosType if m.group(1) == "ntz" else 
TimestampLTZNanosType
-            return cls(int(m.group(2)))
-        if v == "calendarinterval":
-            return CalendarIntervalType()
-        _ym = {"year": 0, "month": 1}
-        _dt = {"day": 0, "hour": 1, "minute": 2, "second": 3}
-        m = _re.match(r"interval (\w+)(?: to (\w+))?$", v)
-        if m:
-            start, end = m.group(1), m.group(2)
-            if start in _ym:
-                s = _ym[start]
-                return YearMonthIntervalType(s, _ym.get(end, s) if end else s)
-            if start in _dt:
-                s = _dt[start]
-                return DayTimeIntervalType(s, _dt.get(end, s) if end else s)
-        raise ValueError(f"cannot parse datatype json: {v!r}")
+        from pyspark._pyspark import parse_datatype_json
+
+        try:
+            return parse_datatype_json(_json.dumps(v))
+        except ValueError:
+            raise ValueError(f"cannot parse datatype json: {v!r}") from None
     elif isinstance(v, dict):
         t = v.get("type")
         if t == "struct":
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 381ee9d..0ff1437 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -104,6 +104,104 @@ def test_parse_datatype_json_string_rejects_garbage():
         T._parse_datatype_json_string(json.dumps("not_a_type"))
 
 
+# One instance of every DataType variant the drop-in has, including the 
parameterized
+# forms whose JSON needs more than a bare type name (collations, spatial CRS 
names).
+EVERY_TYPE = [
+    T.NullType(), T.BooleanType(), T.ByteType(), T.ShortType(), 
T.IntegerType(),
+    T.LongType(), T.FloatType(), T.DoubleType(), T.DecimalType(), 
T.DecimalType(38, 18),
+    T.StringType(), T.StringType("UTF8_LCASE"), T.StringType("UNICODE_CI"),
+    T.CharType(5), T.VarcharType(9), T.BinaryType(), T.DateType(), 
T.TimestampType(),
+    T.TimestampNTZType(), T.TimestampNTZNanosType(9), 
T.TimestampLTZNanosType(7),
+    T.TimeType(3), T.CalendarIntervalType(), T.YearMonthIntervalType(0, 1),
+    T.YearMonthIntervalType(1, 1), T.DayTimeIntervalType(0, 3),
+    T.DayTimeIntervalType(2, 2), T.VariantType(),
+    T.GeometryType(0), T.GeometryType(3857), T.GeometryType(4326), 
T.GeometryType("ANY"),
+    T.GeographyType(4326), T.GeographyType("ANY"),
+    T.ArrayType(T.GeometryType(3857), False),
+    T.MapType(T.StringType("UTF8_LCASE"), T.GeographyType("ANY"), False),
+]
+
+
[email protected]("t", EVERY_TYPE, ids=repr)
+def test_every_datatype_pickle_roundtrip(t):
+    r = pickle.loads(pickle.dumps(t))
+    assert type(r) is type(t)
+    assert r == t and hash(r) == hash(t)
+    assert r.json() == t.json()
+
+
+def test_every_datatype_pickle_roundtrip_nested_in_struct():
+    st = T.StructType([
+        T.StructField(f"f{i}", t, i % 2 == 0, {"i": i, "tag": "v"})
+        for i, t in enumerate(EVERY_TYPE)
+    ])
+    nested = T.StructType([
+        T.StructField("s", st),
+        T.StructField("a", T.ArrayType(st)),
+        T.StructField("m", T.MapType(T.StringType("UNICODE"), st)),
+    ])
+    for t in (st, nested):
+        r = pickle.loads(pickle.dumps(t))
+        assert r == t
+        assert r.json() == t.json()
+    r = pickle.loads(pickle.dumps(st))
+    for got, want in zip(r.fields, st.fields):
+        assert type(got.dataType) is type(want.dataType)
+        assert got.dataType == want.dataType
+        assert got.metadata == want.metadata
+
+
+def test_spatial_and_collated_json_forms_parse():
+    import json
+    cases = {
+        "geometry(SRID:0)": T.GeometryType(0),
+        "geometry(EPSG:3857)": T.GeometryType(3857),
+        "geometry(OGC:CRS84)": T.GeometryType(4326),
+        "geometry(SRID:ANY)": T.GeometryType("ANY"),
+        "geography(OGC:CRS84, SPHERICAL)": T.GeographyType(4326),
+        "geography(SRID:ANY, SPHERICAL)": T.GeographyType("ANY"),
+        "string collate UTF8_LCASE": T.StringType("UTF8_LCASE"),
+    }
+    for j, want in cases.items():
+        assert T._parse_datatype_json_string(json.dumps(j)) == want
+        assert want.json() == json.dumps(j)
+
+
+def test_unparsed_datatype_pickle_roundtrip():
+    t = T._parse_datatype_json_value("unparsed(some_ddl)")
+    assert t.json() == '"unparsed(some_ddl)"'
+    r = pickle.loads(pickle.dumps(t))
+    assert r == t and r.json() == t.json()
+
+
+def test_user_defined_type_pickle_roundtrip():
+    import sys
+    import types as _pytypes
+
+    mod = _pytypes.ModuleType("_udt_pickle_mod")
+    sys.modules["_udt_pickle_mod"] = mod
+    src = (
+        "from pyspark.sql.types import UserDefinedType, ArrayType, 
DoubleType\n"
+        "class PointUDT(UserDefinedType):\n"
+        "    @classmethod\n"
+        "    def sqlType(cls): return ArrayType(DoubleType(), False)\n"
+        "    @classmethod\n"
+        "    def module(cls): return '_udt_pickle_mod'\n"
+    )
+    exec(compile(src, "_udt_pickle_mod", "exec"), mod.__dict__)
+    try:
+        u = mod.PointUDT()
+        assert isinstance(pickle.loads(pickle.dumps(u)), mod.PointUDT)
+        st = T.StructType([
+            T.StructField("p", u), T.StructField("g", T.GeometryType(4326)),
+        ])
+        r = pickle.loads(pickle.dumps(st))
+        assert isinstance(r.fields[0].dataType, mod.PointUDT)
+        assert r.fields[1].dataType == T.GeometryType(4326)
+    finally:
+        del sys.modules["_udt_pickle_mod"]
+
+
 def test_structfield_construction():
     f = T.StructField("n", T.IntegerType(), True)
     st = T.StructType([f])


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to