This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 66fcf88  [SPARK-60906][CONNECT] Make str(), repr() and fromDDL of data 
types match PySpark
66fcf88 is described below

commit 66fcf88cb5f92eef2aa80f10b7d7fa3772156dbd
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Thu Nov 5 18:48:16 2026 +0900

    [SPARK-60906][CONNECT] Make str(), repr() and fromDDL of data types match 
PySpark
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60906](https://issues.apache.org/jira/browse/SPARK-60906): `str()`, 
`repr()`, and `fromDDL` of the Rust-backed Python client's data types now 
behave as in PySpark Connect 4.3.0.
    
    - The base `DataType` no longer defines `__str__`, which returned the 
simple string, so `str()` of a data type is its `repr()`, as in PySpark, where 
data types define only `__repr__`.
    - The base `DataType.__repr__` is the class name, as in PySpark. It applies 
to a bare `DataType()` and to pure-Python subclasses, such as a 
`UserDefinedType`, that do not define their own. The concrete types keep their 
`__repr__`, which already matched PySpark.
    - `DataType.fromDDL`, and the same classmethod on every type, returns the 
concrete type (for example a `StructType` or a `DecimalType`) instead of a bare 
`DataType`.
    
    ### Why are the changes needed?
    
    `str(LongType())` was `bigint` rather than `LongType()`, so `str()` of 
every data type differed from PySpark, as did error messages that format a type 
with `str()`, such as those of `UNSUPPORTED_DATA_TYPE_FOR_ARROW` and the UDF 
return-type checks. `repr()` of a bare `DataType()` or of a `UserDefinedType` 
was `void`, and `fromDDL` returned an object whose type was the base class.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, these now match PySpark Connect 4.3.0:
    
    - `str(LongType())` is `LongType()` (was `bigint`), and `str()` of a 
`StructType` is its `repr()`, such as `StructType([StructField('a', LongType(), 
True)])` (was `struct<a:bigint>`). `simpleString()` is unchanged.
    - `repr(DataType())` is `DataType()` and `repr(ExamplePointUDT())` is 
`ExamplePointUDT()` (both were `void`).
    - `DataType.fromDDL("b string, a int")` returns a `StructType`, and 
`DataType.fromDDL("decimal(10,10)")` a `DecimalType(10,10)`, as in PySpark's 
examples.
    
    ### How was this patch tested?
    
    - `repr()` and `str()` of 30 data types, covering every atomic, interval, 
decimal, char, varchar, array, map and struct type, a `StructField`, a bare 
`DataType()` and a `UserDefinedType`, are identical to official PySpark 4.3.0.
    - New offline tests check `str()` and `repr()` against PySpark's strings, 
the `UserDefinedType` repr, and `fromDDL` against the examples in PySpark's 
docstring. The `str()` and `repr()` tests also pass with official PySpark 
4.3.0. The existing `StructType.add` regression now reads the base type through 
`DataType.simpleString`, which is what it checked through `str()` before.
    - The offline Python suites (437 passed, 8 live cases skipped), and the 
live drop-in and Arrow suites (21 passed) and live cases (9 passed) against a 
local Spark 4.3.0 server. The Rust layer never formats a data type with 
`str()`; clippy reports no diagnostics in `types.rs`, before or after.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #150 from HyukjinKwon/SPARK-60906-datatype-str-repr.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/types.rs      | 203 +++---------------------------------
 python/pyspark/sql/udf.py           |   9 +-
 python/tests/test_dropin_offline.py |  57 +++++++++-
 3 files changed, 73 insertions(+), 196 deletions(-)

diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 983c141..1fb7f1a 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -61,12 +61,6 @@ macro_rules! init_chain {
     };
 }
 
-impl PyDataType {
-    pub fn new(inner: DataType) -> Self {
-        PyDataType { inner }
-    }
-}
-
 /// Construct a `StructType` Python object (with its full base chain) from 
fields.
 fn py_new_struct(py: Python<'_>, fields: Vec<StructField>) -> 
PyResult<Py<PyStructType>> {
     Py::new(
@@ -179,22 +173,24 @@ impl PyDataType {
     }
     #[classmethod]
     #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
+    fn __obj_from_ddl(cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<Py<PyAny>> {
+        let dt = DataType::from_ddl(ddl)
+            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))?;
+        // The concrete type, as pyspark returns (a StructType for a list of 
fields). The
+        // subclasses inherit this classmethod, so that any of them parses DDL 
the same way.
+        data_type_to_py(cls.py(), &dt)
     }
     #[pyo3(name = "typeName")]
     fn __dt_type_name(&self) -> String {
         self.inner.type_name()
     }
 
-    fn __repr__(&self) -> String {
-        self.inner.simple_string()
-    }
-
-    fn __str__(&self) -> String {
-        self.inner.simple_string()
+    /// As pyspark's `DataType.__repr__`: the class name, for the base class 
and for
+    /// pure-Python subclasses (such as a `UserDefinedType`) that do not 
define their own;
+    /// the concrete types override it. There is no `__str__`, so `str()` is 
`repr()`, as
+    /// in pyspark.
+    fn __repr__(slf: &Bound<'_, Self>) -> PyResult<String> {
+        Ok(format!("{}()", slf.get_type().name()?))
     }
 
     fn simpleString(&self) -> String {
@@ -291,13 +287,6 @@ impl PyNullType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"void\"")
     }
@@ -348,13 +337,6 @@ impl PyBooleanType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"boolean\"")
     }
@@ -405,13 +387,6 @@ impl PyByteType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"byte\"")
     }
@@ -466,13 +441,6 @@ impl PyShortType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"short\"")
     }
@@ -527,13 +495,6 @@ impl PyIntegerType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"integer\"")
     }
@@ -588,13 +549,6 @@ impl PyLongType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"long\"")
     }
@@ -649,13 +603,6 @@ impl PyFloatType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"float\"")
     }
@@ -710,13 +657,6 @@ impl PyDoubleType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"double\"")
     }
@@ -774,13 +714,6 @@ impl PyDecimalType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Decimal {
             precision: self.precision,
@@ -864,13 +797,6 @@ impl PyStringType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::String {
             collation: self.collation.clone(),
@@ -958,13 +884,6 @@ impl PyBinaryType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"binary\"")
     }
@@ -1018,13 +937,6 @@ impl PyDateType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"date\"")
     }
@@ -1075,13 +987,6 @@ impl PyTimestampType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"timestamp\"")
     }
@@ -1136,13 +1041,6 @@ impl PyTimestampNTZType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"timestamp_ntz\"")
     }
@@ -1211,13 +1109,6 @@ impl PyArrayType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Array {
             element_type: Box::new(self.element_type.clone()),
@@ -1344,13 +1235,6 @@ impl PyMapType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Map {
             key_type: Box::new(self.key_type.clone()),
@@ -1683,13 +1567,6 @@ impl PyStructType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Struct {
             fields: self.fields.clone(),
@@ -1977,13 +1854,6 @@ impl PyCharType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Char {
             length: self.length,
@@ -2044,13 +1914,6 @@ impl PyVarcharType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Varchar {
             length: self.length,
@@ -2111,13 +1974,6 @@ impl PyTimeType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::Time {
             precision: self.precision,
@@ -2324,13 +2180,6 @@ impl PyCalendarIntervalType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"calendarinterval\"")
     }
@@ -2388,13 +2237,6 @@ impl PyYearMonthIntervalType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::YearMonthInterval {
             start_field: self.start_field,
@@ -2475,13 +2317,6 @@ impl PyDayTimeIntervalType {
     fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         obj
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         let dt = DataType::DayTimeInterval {
             start_field: self.start_field,
@@ -2551,13 +2386,6 @@ impl PyVariantType {
     fn __obj_to_internal<'py>(&self, variant: Bound<'py, PyAny>) -> Bound<'py, 
PyAny> {
         variant
     }
-    #[classmethod]
-    #[pyo3(name = "fromDDL")]
-    fn __obj_from_ddl(_cls: &Bound<'_, pyo3::types::PyType>, ddl: &str) -> 
PyResult<PyDataType> {
-        DataType::from_ddl(ddl)
-            .map(PyDataType::new)
-            .map_err(|e| PyErr::new::<pyo3::exceptions::PyValueError, 
_>(e.to_string()))
-    }
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"variant\"")
     }
@@ -3062,9 +2890,10 @@ pub(crate) fn py_to_data_type(obj: &Bound<'_, PyAny>) -> 
PyResult<DataType> {
             });
         }
     }
-    // The bare base `DataType` (e.g. from `fromDDL`) is matched LAST: 
concrete subclasses
-    // (which all share the PyDataType base) are handled above via their own 
fields, so a
-    // mutated StructType reflects its live fields rather than the base's 
snapshot.
+    // The bare base `DataType`, which only `DataType()` and pure-Python 
subclasses are, is
+    // matched LAST: concrete subclasses (which all share the PyDataType base) 
are handled
+    // above via their own fields, so a mutated StructType reflects its live 
fields rather
+    // than the base's snapshot.
     if let Ok(d) = obj.extract::<PyRef<PyDataType>>() {
         return Ok(d.inner.clone());
     }
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index e7034e2..7986cc5 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -23,23 +23,22 @@ Mirrors pyspark.sql.connect.udf and 
pyspark.sql.functions.udf/pandas_udf.
 import sys
 from typing import Callable, Optional, Any
 
-from pyspark.sql.types import DataType, StringType, _parse_datatype_json_string
+from pyspark.sql.types import DataType, StringType
 from pyspark.serializers import CloudPickleSerializer
 
 
 def _as_concrete_datatype(returnType: Any) -> DataType:
     """Return a concrete, picklable pyspark ``DataType``.
 
-    A DDL string (e.g. ``"double"``, ``"a int, b string"``) is parsed via the
-    Rust DDL parser and rebuilt into a concrete type through
-    ``_parse_datatype_json_string``. This matters because the cloudpickled UDF
+    A DDL string (e.g. ``"double"``, ``"a int, b string"``) is parsed into its
+    concrete type with ``DataType.fromDDL``. This matters because the 
cloudpickled UDF
     command must carry a real ``DataType`` (not a DDL string): the pandas/Arrow
     worker pairs each result column with its spark type to build the Arrow 
batch,
     so a bare string breaks it ("not enough values to unpack"). Mirrors how the
     reference Connect client pickles the *parsed* output type.
     """
     if isinstance(returnType, str):
-        return _parse_datatype_json_string(DataType.fromDDL(returnType).json())
+        return DataType.fromDDL(returnType)
     return returnType
 
 
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index a6a35fd..41a2ad0 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -534,13 +534,62 @@ def test_structtype_add_reflected_in_json():
 
 
 def test_structtype_add_reflected_in_the_base_data_type():
-    # Regression: add() must update the base DataType too, which str() and the 
schemas of
-    # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype).
+    # Regression: add() must update the base DataType too, which the schemas of
+    # mapInArrow/mapInPandas/applyInPandas/... read (via resolve_datatype). 
The base
+    # implementation of simpleString reads it.
     built = T.StructType([T.StructField("a", T.LongType()), T.StructField("b", 
T.LongType())])
-    assert str(T.StructType().add("a", T.LongType()).add("b", T.LongType())) 
== str(built)
+    chained = T.StructType().add("a", T.LongType()).add("b", T.LongType())
     st = T.StructType([T.StructField("a", T.LongType())])
     st.add("b", T.LongType())
-    assert str(st) == str(built) == "struct<a:bigint,b:bigint>"
+    for added in (chained, st):
+        assert T.DataType.simpleString(added) == "struct<a:bigint,b:bigint>"
+        assert str(added) == str(built) == repr(built)
+
+
[email protected]("data_type,expected", [
+    (T.LongType(), "LongType()"),
+    (T.TimestampNTZType(), "TimestampNTZType()"),
+    (T.VariantType(), "VariantType()"),
+    (T.DecimalType(10, 2), "DecimalType(10,2)"),
+    (T.CharType(5), "CharType(5)"),
+    (T.DayTimeIntervalType(0, 3), "DayTimeIntervalType(0, 3)"),
+    (T.YearMonthIntervalType(), "YearMonthIntervalType(0, 1)"),
+    (T.ArrayType(T.StringType(), False), "ArrayType(StringType(), False)"),
+    (T.MapType(T.StringType(), T.LongType()), "MapType(StringType(), 
LongType(), True)"),
+    (T.StructType([T.StructField("a", T.LongType()),
+                   T.StructField("b", T.ArrayType(T.StringType()), False)]),
+     "StructType([StructField('a', LongType(), True), "
+     "StructField('b', ArrayType(StringType(), True), False)])"),
+    (T.StructType(), "StructType([])"),
+    (T.DataType(), "DataType()"),
+])
+def test_datatype_str_is_its_repr_as_pyspark(data_type, expected):
+    # pyspark data types define only __repr__, so str() is the same.
+    assert repr(data_type) == expected
+    assert str(data_type) == expected
+
+
+def test_user_defined_type_repr_is_its_class_name():
+    from pyspark.testing.objects import ExamplePointUDT
+
+    assert repr(ExamplePointUDT()) == str(ExamplePointUDT()) == 
"ExamplePointUDT()"
+
+
[email protected]("ddl, cls, expected", [
+    # The examples of pyspark's DataType.fromDDL.
+    ("b string, a int", T.StructType,
+     "StructType([StructField('b', StringType(), True), StructField('a', 
IntegerType(), True)])"),
+    ("decimal(10,10)", T.DecimalType, "DecimalType(10,10)"),
+    ("b: string, a: int", T.StructType,
+     "StructType([StructField('b', StringType(), True), StructField('a', 
IntegerType(), True)])"),
+    ("int", T.IntegerType, "IntegerType()"),
+])
+def test_datatype_from_ddl_returns_the_concrete_type(ddl, cls, expected):
+    parsed = T.DataType.fromDDL(ddl)
+    assert repr(parsed) == expected
+    assert isinstance(parsed, cls)
+    # The classmethod parses DDL the same way from any data type class.
+    assert repr(T.LongType.fromDDL(ddl)) == expected
 
 
 def test_datatype_object_model():


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to