This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new d08f303 [SPARK-60933][CONNECT] Make every DataType pickle round-trip
in the Rust drop-in
d08f303 is described below
commit d08f30347c2df2c66edd08802ee77e9cce7a4285
Author: Liang-Chi Hsieh <[email protected]>
AuthorDate: Fri Nov 6 07:06:26 2026 +0900
[SPARK-60933][CONNECT] Make every DataType pickle round-trip in the Rust
drop-in
### What changes were proposed in this pull request?
In the Rust-backed PySpark drop-in, some data types did not survive
`pickle.loads(pickle.dumps(t))`, although every `DataType` must be picklable so
that it reaches an official-pyspark UDF worker. This PR fixes them:
- `pyspark.sql.types._parse_datatype_json_value` now parses the JSON string
forms (atomics, `decimal(p,s)`, `string collate X`, intervals,
`geometry(<crs>)`, `geography(<crs>, <alg>)`, ...) with the Rust core's
`DataType::from_json`, through a new `_pyspark.parse_datatype_json`. Before, it
kept a second grammar in Python that knew neither collations nor the spatial
forms. The nested struct/array/map objects still recurse in Python, because a
`udt` field has to rebuild its Python class.
- `GeometryType` and `GeographyType` get `__reduce__`, as the other type
classes have.
- The base `DataType` gets a `__reduce__` fallback, built from
`self.json()`, for a bare unparsed `DataType` and a pure-Python
`UserDefinedType`.
- The core JSON parser reads `"unparsed(x)"` back as `x`; it used to read
it as `unparsed(unparsed(x))`. It also accepts `"decimal"`, `"null"` and
`"calendarinterval"`, which the Python parser accepted before.
- `data_type_to_py` returns a bare `DataType` for an unparsed type instead
of calling back into the Python parser, which now sends JSON strings back to
Rust and would loop.
### Why are the changes needed?
To match PySpark. Before this change:
```python
>>> import pickle
>>> from pyspark.sql.types import *
>>> pickle.dumps(GeometryType(4326))
TypeError: cannot pickle 'pyspark.sql.types.GeometryType' object
>>> pickle.loads(pickle.dumps(StringType("UTF8_LCASE")))
ValueError: cannot parse datatype json: 'string collate UTF8_LCASE'
```
### Does this PR introduce _any_ user-facing change?
Yes. Spatial types, collated strings and every type nested in a
`StructType`, `ArrayType` or `MapType` now pickle, as in PySpark, so they can
be a UDF's return type or appear in a schema sent to a worker.
### How was this patch tested?
- New offline tests in `python/tests/test_dropin_offline.py`: every data
type variant (atomic, parameterized, intervals, collated strings, each
supported spatial SRID, nested array/map) round-trips through pickle with the
same class, `==`, `hash` and JSON, alone and nested in a `StructType` with
field metadata. Further tests cover parsing the spatial and collated JSON
forms, an unparsed type, and a UDT nested next to a spatial type.
- New Rust unit tests for the parser aliases and the `unparsed(x)` round
trip.
- `cargo fmt --all --check`, `cargo test -p apache-spark-connect --release
--lib`, and `cargo test -p apache-spark-connect --release [--features wasm-udf]
--no-run` pass, and clippy reports nothing in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Yes.
Closes #162 from viirya/SPARK-60933.
Authored-by: Liang-Chi Hsieh <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/lib.rs | 3 ++
crates/pyspark-rs/src/types.rs | 33 ++++++++++++-
crates/spark-connect/src/types.rs | 38 ++++++++++++--
python/pyspark/sql/types.py | 55 +++++----------------
python/tests/test_dropin_offline.py | 98 +++++++++++++++++++++++++++++++++++++
5 files changed, 179 insertions(+), 48 deletions(-)
diff --git a/crates/pyspark-rs/src/lib.rs b/crates/pyspark-rs/src/lib.rs
index 29584de..8f2ab6b 100644
--- a/crates/pyspark-rs/src/lib.rs
+++ b/crates/pyspark-rs/src/lib.rs
@@ -125,6 +125,9 @@ fn _pyspark(_py: Python<'_>, m: &Bound<'_, PyModule>) ->
PyResult<()> {
m.add_class::<PyStatFunctions>()?;
m.add_class::<storagelevel::PyStorageLevel>()?;
+ // The core DataType JSON grammar, which `pyspark.sql.types` parses type
JSON with.
+ m.add_function(wrap_pyfunction!(types::parse_datatype_json, m)?)?;
+
// Spark Declarative Pipelines (SDP) command execution
m.add_class::<pipelines::PyPipelineRunStream>()?;
m.add_function(wrap_pyfunction!(
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 17f4586..4eab163 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -146,6 +146,14 @@ impl PyDataType {
}
}
+ /// The fallback for the classes without their own `__reduce__`: a bare
`DataType`
+ /// (an unparsed DDL type) and a pure-Python `UserDefinedType`, whose
`json()` names
+ /// its Python class.
+ fn __reduce__(slf: &Bound<'_, Self>) -> PyResult<(Py<PyAny>, (String,))> {
+ let json: String = slf.call_method0("json")?.extract()?;
+ type_reduce(slf.py(), &json)
+ }
+
// --- DataType object-model methods (v4.2.0 parity) ---
#[pyo3(name = "json")]
fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2447,6 +2455,9 @@ impl PyGeometryType {
fn srid(&self) -> i32 {
self.srid
}
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
+ type_reduce(py, &DataType::Geometry { srid: self.srid }.json())
+ }
fn __repr__(&self) -> String {
match self.srid {
-1 => "GeometryType(ANY)".to_string(),
@@ -2496,6 +2507,9 @@ impl PyGeographyType {
fn srid(&self) -> i32 {
self.srid
}
+ fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
+ type_reduce(py, &DataType::Geography { srid: self.srid }.json())
+ }
fn __repr__(&self) -> String {
match self.srid {
-1 => "GeographyType(ANY)".to_string(),
@@ -2544,6 +2558,18 @@ fn spatial_type_srid(srid: &Bound<'_, PyAny>,
geographic: bool) -> PyResult<i32>
))
}
+/// Parse a non-UDT DataType's Spark JSON (e.g. `"string collate UTF8_LCASE"`,
+/// `"geography(OGC:CRS84, SPHERICAL)"`, or a nested struct/array/map object)
with the core
+/// grammar, into its concrete Python type object.
`pyspark.sql.types._parse_datatype_json_value`
+/// delegates its atomic and parameterized string forms here, so the drop-in
keeps one JSON
+/// grammar (the core's) instead of a second parser in Python.
+#[pyfunction]
+pub fn parse_datatype_json(py: Python<'_>, json: &str) -> PyResult<Py<PyAny>> {
+ let dt = DataType::from_json_str(json)
+ .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError,
_>(e.to_string()))?;
+ data_type_to_py(py, &dt)
+}
+
/// Convert a Python DataType object (any of the type classes) or a DDL string
into
/// the core `DataType`. Mirrors pyspark accepting `Union[DataType, str]`
everywhere.
/// Materialize a core [`DataType`] into its concrete Python type object — a
real
@@ -2781,9 +2807,12 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt:
&DataType) -> PyResult<Py<PyAn
value
))
}
- DataType::Udt { .. } | DataType::Unparsed { .. } => {
+ // An unparsed DDL string has no concrete Python class; keep it as a
bare `DataType`.
+ // (The Python parser delegates JSON strings back here, so routing it
there would loop.)
+ DataType::Unparsed { .. } => Py::new(py, PyDataType { inner:
dt.clone() })?.into_any(),
+ DataType::Udt { .. } => {
// A UDT reconstructs by re-importing its Python class; delegate
to the Python
- // parser (which also handles the unparsed-DDL fallback).
+ // parser.
py.import("pyspark.sql.types")?
.getattr("_parse_datatype_json_string")?
.call1((dt.json(),))?
diff --git a/crates/spark-connect/src/types.rs
b/crates/spark-connect/src/types.rs
index fb026fd..fd2894b 100644
--- a/crates/spark-connect/src/types.rs
+++ b/crates/spark-connect/src/types.rs
@@ -1112,7 +1112,9 @@ fn parse_json_value(value: &serde_json::Value,
_field_name: Option<&str>) -> Res
fn parse_json_type_string(s: &str) -> Result<DataType> {
match s {
- "void" => Ok(DataType::Null),
+ // `null` and `calendarinterval` are lenient aliases that the
drop-in's Python parser
+ // accepted before it delegated here.
+ "void" | "null" => Ok(DataType::Null),
"boolean" => Ok(DataType::Boolean),
"byte" => Ok(DataType::Byte),
"short" => Ok(DataType::Short),
@@ -1124,8 +1126,13 @@ fn parse_json_type_string(s: &str) -> Result<DataType> {
"date" => Ok(DataType::Date),
"timestamp" => Ok(DataType::Timestamp),
"timestamp_ntz" => Ok(DataType::TimestampNtz),
- "interval" => Ok(DataType::CalendarInterval),
+ "interval" | "calendarinterval" => Ok(DataType::CalendarInterval),
"variant" => Ok(DataType::Variant),
+ // A bare `decimal`, as `DecimalType().typeName()` names it, is the
default decimal(10,0).
+ "decimal" => Ok(DataType::Decimal {
+ precision: 10,
+ scale: 0,
+ }),
s if s.starts_with("decimal(") => parse_decimal(s),
s if s.starts_with("char(") => parse_char(s),
s if s.starts_with("varchar(") => parse_varchar(s),
@@ -1143,8 +1150,9 @@ fn parse_json_type_string(s: &str) -> Result<DataType> {
}),
s if s.starts_with("geometry(") => parse_geometry(s),
s if s.starts_with("geography(") => parse_geography(s),
- s if s.starts_with("unparsed(") => Ok(DataType::Unparsed {
- data_type_string: s.to_string(),
+ // The inverse of `json_value`, which writes
`unparsed(<data_type_string>)`.
+ s if s.starts_with("unparsed(") && s.ends_with(')') =>
Ok(DataType::Unparsed {
+ data_type_string: s["unparsed(".len()..s.len() - 1].to_string(),
}),
_ => Err(SparkError::value(
"INVALID_DATATYPE_STRING",
@@ -3065,6 +3073,28 @@ mod tests {
}
}
+ #[test]
+ fn test_from_json_unparsed_roundtrip() {
+ let dt = DataType::Unparsed {
+ data_type_string: "int".to_string(),
+ };
+ assert_eq!(DataType::from_json(&dt.json_value()).unwrap(), dt);
+ }
+
+ #[test]
+ fn test_from_json_lenient_aliases() {
+ let parse = |s: &str|
DataType::from_json(&serde_json::json!(s)).unwrap();
+ assert_eq!(
+ parse("decimal"),
+ DataType::Decimal {
+ precision: 10,
+ scale: 0
+ }
+ );
+ assert_eq!(parse("null"), DataType::Null);
+ assert_eq!(parse("calendarinterval"), DataType::CalendarInterval);
+ }
+
#[test]
fn test_from_json_geometry() {
let json_geom = serde_json::json!("geometry(any)");
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 6fb1d17..c615d23 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -289,50 +289,21 @@ def _parse_datatype_json_string(json_string):
def _parse_datatype_json_value(v):
"""Recursively reconstruct a DataType from a parsed Spark-JSON value (a
string for
- atomic/parameterized types, or a dict for the nested struct/array/map
forms)."""
- import re as _re
+ atomic/parameterized types, or a dict for the nested struct/array/map
forms).
+
+ The string forms (atomics, ``decimal(p,s)``, ``string collate X``,
intervals,
+ ``geometry(<crs>)``, ``geography(<crs>, <alg>)``, ...) are parsed by the
Rust core's
+ JSON grammar, so the drop-in keeps one grammar. The nested object forms
recurse here,
+ because a ``udt`` leaf has to reconstruct its Python class."""
+ import json as _json
if isinstance(v, str):
- atomic = {
- "void": NullType, "null": NullType, "boolean": BooleanType,
"byte": ByteType,
- "short": ShortType, "integer": IntegerType, "long": LongType,
"float": FloatType,
- "double": DoubleType, "string": StringType, "binary": BinaryType,
"date": DateType,
- "timestamp": TimestampType, "timestamp_ntz": TimestampNTZType,
"variant": VariantType,
- }
- if v in atomic:
- return atomic[v]()
- m = _re.match(r"decimal\((\d+),\s*(\d+)\)$", v)
- if m:
- return DecimalType(int(m.group(1)), int(m.group(2)))
- if v == "decimal":
- return DecimalType()
- m = _re.match(r"char\((\d+)\)$", v)
- if m:
- return CharType(int(m.group(1)))
- m = _re.match(r"varchar\((\d+)\)$", v)
- if m:
- return VarcharType(int(m.group(1)))
- m = _re.match(r"time\((\d+)\)$", v)
- if m:
- return TimeType(int(m.group(1)))
- m = _re.match(r"timestamp_(ntz|ltz)\(\s*(\d+)\s*\)$", v)
- if m:
- cls = TimestampNTZNanosType if m.group(1) == "ntz" else
TimestampLTZNanosType
- return cls(int(m.group(2)))
- if v == "calendarinterval":
- return CalendarIntervalType()
- _ym = {"year": 0, "month": 1}
- _dt = {"day": 0, "hour": 1, "minute": 2, "second": 3}
- m = _re.match(r"interval (\w+)(?: to (\w+))?$", v)
- if m:
- start, end = m.group(1), m.group(2)
- if start in _ym:
- s = _ym[start]
- return YearMonthIntervalType(s, _ym.get(end, s) if end else s)
- if start in _dt:
- s = _dt[start]
- return DayTimeIntervalType(s, _dt.get(end, s) if end else s)
- raise ValueError(f"cannot parse datatype json: {v!r}")
+ from pyspark._pyspark import parse_datatype_json
+
+ try:
+ return parse_datatype_json(_json.dumps(v))
+ except ValueError:
+ raise ValueError(f"cannot parse datatype json: {v!r}") from None
elif isinstance(v, dict):
t = v.get("type")
if t == "struct":
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 381ee9d..0ff1437 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -104,6 +104,104 @@ def test_parse_datatype_json_string_rejects_garbage():
T._parse_datatype_json_string(json.dumps("not_a_type"))
+# One instance of every DataType variant the drop-in has, including the
parameterized
+# forms whose JSON needs more than a bare type name (collations, spatial CRS
names).
+EVERY_TYPE = [
+ T.NullType(), T.BooleanType(), T.ByteType(), T.ShortType(),
T.IntegerType(),
+ T.LongType(), T.FloatType(), T.DoubleType(), T.DecimalType(),
T.DecimalType(38, 18),
+ T.StringType(), T.StringType("UTF8_LCASE"), T.StringType("UNICODE_CI"),
+ T.CharType(5), T.VarcharType(9), T.BinaryType(), T.DateType(),
T.TimestampType(),
+ T.TimestampNTZType(), T.TimestampNTZNanosType(9),
T.TimestampLTZNanosType(7),
+ T.TimeType(3), T.CalendarIntervalType(), T.YearMonthIntervalType(0, 1),
+ T.YearMonthIntervalType(1, 1), T.DayTimeIntervalType(0, 3),
+ T.DayTimeIntervalType(2, 2), T.VariantType(),
+ T.GeometryType(0), T.GeometryType(3857), T.GeometryType(4326),
T.GeometryType("ANY"),
+ T.GeographyType(4326), T.GeographyType("ANY"),
+ T.ArrayType(T.GeometryType(3857), False),
+ T.MapType(T.StringType("UTF8_LCASE"), T.GeographyType("ANY"), False),
+]
+
+
[email protected]("t", EVERY_TYPE, ids=repr)
+def test_every_datatype_pickle_roundtrip(t):
+ r = pickle.loads(pickle.dumps(t))
+ assert type(r) is type(t)
+ assert r == t and hash(r) == hash(t)
+ assert r.json() == t.json()
+
+
+def test_every_datatype_pickle_roundtrip_nested_in_struct():
+ st = T.StructType([
+ T.StructField(f"f{i}", t, i % 2 == 0, {"i": i, "tag": "v"})
+ for i, t in enumerate(EVERY_TYPE)
+ ])
+ nested = T.StructType([
+ T.StructField("s", st),
+ T.StructField("a", T.ArrayType(st)),
+ T.StructField("m", T.MapType(T.StringType("UNICODE"), st)),
+ ])
+ for t in (st, nested):
+ r = pickle.loads(pickle.dumps(t))
+ assert r == t
+ assert r.json() == t.json()
+ r = pickle.loads(pickle.dumps(st))
+ for got, want in zip(r.fields, st.fields):
+ assert type(got.dataType) is type(want.dataType)
+ assert got.dataType == want.dataType
+ assert got.metadata == want.metadata
+
+
+def test_spatial_and_collated_json_forms_parse():
+ import json
+ cases = {
+ "geometry(SRID:0)": T.GeometryType(0),
+ "geometry(EPSG:3857)": T.GeometryType(3857),
+ "geometry(OGC:CRS84)": T.GeometryType(4326),
+ "geometry(SRID:ANY)": T.GeometryType("ANY"),
+ "geography(OGC:CRS84, SPHERICAL)": T.GeographyType(4326),
+ "geography(SRID:ANY, SPHERICAL)": T.GeographyType("ANY"),
+ "string collate UTF8_LCASE": T.StringType("UTF8_LCASE"),
+ }
+ for j, want in cases.items():
+ assert T._parse_datatype_json_string(json.dumps(j)) == want
+ assert want.json() == json.dumps(j)
+
+
+def test_unparsed_datatype_pickle_roundtrip():
+ t = T._parse_datatype_json_value("unparsed(some_ddl)")
+ assert t.json() == '"unparsed(some_ddl)"'
+ r = pickle.loads(pickle.dumps(t))
+ assert r == t and r.json() == t.json()
+
+
+def test_user_defined_type_pickle_roundtrip():
+ import sys
+ import types as _pytypes
+
+ mod = _pytypes.ModuleType("_udt_pickle_mod")
+ sys.modules["_udt_pickle_mod"] = mod
+ src = (
+ "from pyspark.sql.types import UserDefinedType, ArrayType,
DoubleType\n"
+ "class PointUDT(UserDefinedType):\n"
+ " @classmethod\n"
+ " def sqlType(cls): return ArrayType(DoubleType(), False)\n"
+ " @classmethod\n"
+ " def module(cls): return '_udt_pickle_mod'\n"
+ )
+ exec(compile(src, "_udt_pickle_mod", "exec"), mod.__dict__)
+ try:
+ u = mod.PointUDT()
+ assert isinstance(pickle.loads(pickle.dumps(u)), mod.PointUDT)
+ st = T.StructType([
+ T.StructField("p", u), T.StructField("g", T.GeometryType(4326)),
+ ])
+ r = pickle.loads(pickle.dumps(st))
+ assert isinstance(r.fields[0].dataType, mod.PointUDT)
+ assert r.fields[1].dataType == T.GeometryType(4326)
+ finally:
+ del sys.modules["_udt_pickle_mod"]
+
+
def test_structfield_construction():
f = T.StructField("n", T.IntegerType(), True)
st = T.StructType([f])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]