This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 865a427  [SPARK-61080][CONNECT][PYTHON] Expose the attributes of the 
data types, typeName as a classmethod, and the singleton types in the Python 
client as PySpark does
865a427 is described below

commit 865a427339be12484e27fd3c95256232c58c4f0e
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:10:22 2026 +0900

    [SPARK-61080][CONNECT][PYTHON] Expose the attributes of the data types, 
typeName as a classmethod, and the singleton types in the Python client as 
PySpark does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-61080](https://issues.apache.org/jira/browse/SPARK-61080): the data 
types now have the attributes, the `typeName` and the singletons of PySpark's 
types.
    
    - The types have the attributes of their parameters:
      - `StringType("UTF8_LCASE").collation`
      - `CharType(3).length` and `VarcharType(4).length`
      - `TimeType(3).precision`
      - the `startField` and `endField` of `DayTimeIntervalType` and 
`YearMonthIntervalType`
      - `DecimalType(10, 2).hasPrecisionInfo`
    - `typeName` is a classmethod, as PySpark's: `IntegerType.typeName()` is 
`integer`, the name of the class without `Type`, lowercased. The same holds for 
a subclass of `DataType`. The exceptions are PySpark's `void`, `timestamp_ntz` 
and `interval` for `NullType`, `TimestampNTZType` and `CalendarIntervalType`.
    - `StructField.typeName()` raises the `PySparkTypeError` 
`INVALID_TYPENAME_CALL`.
    - The types of PySpark's `DataTypeSingleton` are singletons: `IntegerType() 
is IntegerType()`. They are `NullType`, the integral types, `FloatType`, 
`DoubleType`, `BinaryType`, `BooleanType`, `DateType`, `TimestampType`, 
`TimestampNTZType` and `CalendarIntervalType`. The client also uses these 
instances for the types it builds from schemas and from their JSON. PySpark 
Connect, by contrast, unpickles a copy of the cached schema in `df.schema`, so 
the types there are new objects.
    
    ### Why are the changes needed?
    
    - The client lacked these attributes: `StringType("UTF8_LCASE").collation` 
raised an `AttributeError`.
    - `typeName` was an instance method, so `IntegerType.typeName()` raised a 
`TypeError`.
    - `CalendarIntervalType().typeName()` was `calendarinterval`, not 
`interval`.
    - `StructField.typeName()` raised a plain `TypeError`.
    - Each call of a type class returned a new object.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The types have the attributes, the type names and the singletons of 
PySpark 4.3.0.
    
    ### How was this patch tested?
    
    - A new live test checks the attributes of the types of a schema of the 
server: a collated string, an interval, a time and a decimal. It passes with 
official PySpark 4.3.0 and fails without this change.
    - A new offline test checks the attributes, the `typeName` of the classes, 
including an abstract one and a subclass of `DataType`, the error of 
`StructField.typeName`, and the singletons.
    - The `typeName` of 37 classes, a type API sweep of 48 cases, and the 
public attributes of 28 type classes are the same as official PySpark 4.3.0's.
    - The lib tests of both crates (592 passed), the build of all tests, also 
with `wasm-udf`, the offline Python suites (2018 passed, 8 skipped), and the 
live drop-in and Arrow suites (144 passed) and live cases (10 passed) against a 
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #281 from HyukjinKwon/SPARK-61080-type-attributes.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/types.rs      | 489 ++++++++++++++++++++----------------
 python/tests/test_dropin_e2e.py     |  15 ++
 python/tests/test_dropin_offline.py |  36 +++
 3 files changed, 323 insertions(+), 217 deletions(-)

diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index e41191b..010125e 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -1,6 +1,7 @@
 //! PyO3 wrappers for Spark DataTypes.
 
 use pyo3::prelude::*;
+use pyo3::sync::PyOnceLock;
 use spark_connect::types::{DataType, StructField};
 use std::collections::BTreeMap;
 
@@ -305,9 +306,17 @@ impl PyDataType {
             .call_method1("_parse_datatype_string", (ddl,))?
             .unbind())
     }
+    /// The name of the type, as PySpark's classmethod `DataType.typeName`: 
the name of the class
+    /// without its `Type`, lowercased, such as `integer` of `IntegerType`, 
also of a subclass.
+    #[classmethod]
     #[pyo3(name = "typeName")]
-    fn __dt_type_name(&self) -> String {
-        self.inner.type_name()
+    fn __dt_type_name(cls: &Bound<'_, pyo3::types::PyType>) -> 
PyResult<String> {
+        let name = cls.name()?.to_string();
+        let stem: String = name
+            .chars()
+            .take(name.chars().count().saturating_sub(4))
+            .collect();
+        Ok(stem.to_lowercase())
     }
 
     /// As pyspark's `DataType.__repr__`: the class name, for the base class 
and for
@@ -430,9 +439,16 @@ impl PyNullType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"void\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(DataType::Null, PyNullType, [])
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyNullType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(py, init_chain!(DataType::Null, PyNullType, []))
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "NullType()".to_string()
@@ -441,8 +457,10 @@ impl PyNullType {
     fn simple_string(&self) -> &'static str {
         "void"
     }
+    /// As PySpark's, which names this type `void`.
+    #[classmethod]
     #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
+    fn type_name(_cls: &Bound<'_, pyo3::types::PyType>) -> &'static str {
         "void"
     }
 }
@@ -486,9 +504,19 @@ impl PyBooleanType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"boolean\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(DataType::Boolean, PyBooleanType, [PyAtomicType])
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyBooleanType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(DataType::Boolean, PyBooleanType, 
[PyAtomicType]),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "BooleanType()".to_string()
@@ -497,10 +525,6 @@ impl PyBooleanType {
     fn simple_string(&self) -> &'static str {
         "boolean"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "boolean"
-    }
 }
 
 #[pyclass(name = "ByteType", module = "pyspark.sql.types", extends = 
PyIntegralType)]
@@ -542,13 +566,23 @@ impl PyByteType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"byte\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Byte,
-            PyByteType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyByteType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Byte,
+                        PyByteType,
+                        [PyAtomicType, PyNumericType, PyIntegralType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "ByteType()".to_string()
@@ -557,10 +591,6 @@ impl PyByteType {
     fn simple_string(&self) -> &'static str {
         "tinyint"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "byte"
-    }
 }
 
 #[pyclass(name = "ShortType", module = "pyspark.sql.types", extends = 
PyIntegralType)]
@@ -602,13 +632,23 @@ impl PyShortType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"short\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Short,
-            PyShortType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyShortType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Short,
+                        PyShortType,
+                        [PyAtomicType, PyNumericType, PyIntegralType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "ShortType()".to_string()
@@ -617,10 +657,6 @@ impl PyShortType {
     fn simple_string(&self) -> &'static str {
         "smallint"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "short"
-    }
 }
 
 #[pyclass(name = "IntegerType", module = "pyspark.sql.types", extends = 
PyIntegralType)]
@@ -662,13 +698,23 @@ impl PyIntegerType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"integer\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Integer,
-            PyIntegerType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyIntegerType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Integer,
+                        PyIntegerType,
+                        [PyAtomicType, PyNumericType, PyIntegralType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "IntegerType()".to_string()
@@ -677,10 +723,6 @@ impl PyIntegerType {
     fn simple_string(&self) -> &'static str {
         "int"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "integer"
-    }
 }
 
 #[pyclass(name = "LongType", module = "pyspark.sql.types", extends = 
PyIntegralType)]
@@ -722,13 +764,23 @@ impl PyLongType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"long\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Long,
-            PyLongType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyLongType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Long,
+                        PyLongType,
+                        [PyAtomicType, PyNumericType, PyIntegralType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "LongType()".to_string()
@@ -737,10 +789,6 @@ impl PyLongType {
     fn simple_string(&self) -> &'static str {
         "bigint"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "long"
-    }
 }
 
 #[pyclass(name = "FloatType", module = "pyspark.sql.types", extends = 
PyFractionalType)]
@@ -782,13 +830,23 @@ impl PyFloatType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"float\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Float,
-            PyFloatType,
-            [PyAtomicType, PyNumericType, PyFractionalType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyFloatType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Float,
+                        PyFloatType,
+                        [PyAtomicType, PyNumericType, PyFractionalType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "FloatType()".to_string()
@@ -797,10 +855,6 @@ impl PyFloatType {
     fn simple_string(&self) -> &'static str {
         "float"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "float"
-    }
 }
 
 #[pyclass(name = "DoubleType", module = "pyspark.sql.types", extends = 
PyFractionalType)]
@@ -842,13 +896,23 @@ impl PyDoubleType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"double\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Double,
-            PyDoubleType,
-            [PyAtomicType, PyNumericType, PyFractionalType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyDoubleType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Double,
+                        PyDoubleType,
+                        [PyAtomicType, PyNumericType, PyFractionalType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "DoubleType()".to_string()
@@ -857,10 +921,6 @@ impl PyDoubleType {
     fn simple_string(&self) -> &'static str {
         "double"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "double"
-    }
 }
 
 #[pyclass(name = "DecimalType", module = "pyspark.sql.types", extends = 
PyFractionalType)]
@@ -871,6 +931,11 @@ pub struct PyDecimalType {
 
 #[pymethods]
 impl PyDecimalType {
+    /// As PySpark's `DecimalType.hasPrecisionInfo`, a public attribute that 
is always true.
+    #[getter(hasPrecisionInfo)]
+    fn has_precision_info(&self) -> bool {
+        true
+    }
     // --- DataType object-model methods (v4.2.0 parity) ---
     #[pyo3(name = "json")]
     fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -935,10 +1000,6 @@ impl PyDecimalType {
     fn simple_string(&self) -> String {
         format!("decimal({},{})", self.precision, self.scale)
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "decimal"
-    }
 }
 
 #[pyclass(name = "StringType", module = "pyspark.sql.types", extends = 
PyAtomicType)]
@@ -948,6 +1009,11 @@ pub struct PyStringType {
 
 #[pymethods]
 impl PyStringType {
+    /// The collation of the strings, as PySpark's `StringType.collation`.
+    #[getter]
+    fn collation(&self) -> String {
+        self.collation.clone()
+    }
     // Collation providers, mirroring 
`StringType.providerSpark/providerICU/providers`.
     #[classattr]
     #[allow(non_upper_case_globals)]
@@ -1024,10 +1090,6 @@ impl PyStringType {
             format!("string collate {}", self.collation)
         }
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "string"
-    }
 
     /// Provider ("spark" for UTF8*, else "icu") for a collation name.
     /// Mirrors `StringType.collationProvider`.
@@ -1087,9 +1149,19 @@ impl PyBinaryType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"binary\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(DataType::Binary, PyBinaryType, [PyAtomicType])
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyBinaryType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(DataType::Binary, PyBinaryType, 
[PyAtomicType]),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "BinaryType()".to_string()
@@ -1098,10 +1170,6 @@ impl PyBinaryType {
     fn simple_string(&self) -> &'static str {
         "binary"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "binary"
-    }
 }
 
 #[pyclass(name = "DateType", module = "pyspark.sql.types", extends = 
PyDatetimeType)]
@@ -1146,9 +1214,19 @@ impl PyDateType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"date\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(DataType::Date, PyDateType, [PyAtomicType, PyDatetimeType])
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyDateType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(DataType::Date, PyDateType, [PyAtomicType, 
PyDatetimeType]),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "DateType()".to_string()
@@ -1157,10 +1235,6 @@ impl PyDateType {
     fn simple_string(&self) -> &'static str {
         "date"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "date"
-    }
 }
 
 #[pyclass(name = "TimestampType", module = "pyspark.sql.types", extends = 
PyDatetimeType)]
@@ -1202,13 +1276,23 @@ impl PyTimestampType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"timestamp\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::Timestamp,
-            PyTimestampType,
-            [PyAtomicType, PyDatetimeType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyTimestampType>> = PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::Timestamp,
+                        PyTimestampType,
+                        [PyAtomicType, PyDatetimeType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "TimestampType()".to_string()
@@ -1217,10 +1301,6 @@ impl PyTimestampType {
     fn simple_string(&self) -> &'static str {
         "timestamp"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "timestamp"
-    }
 }
 
 #[pyclass(name = "TimestampNTZType", module = "pyspark.sql.types", extends = 
PyDatetimeType)]
@@ -1262,13 +1342,23 @@ impl PyTimestampNTZType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"timestamp_ntz\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
-            DataType::TimestampNtz,
-            PyTimestampNTZType,
-            [PyAtomicType, PyDatetimeType]
-        )
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyTimestampNTZType>> = 
PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(
+                        DataType::TimestampNtz,
+                        PyTimestampNTZType,
+                        [PyAtomicType, PyDatetimeType]
+                    ),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "TimestampNTZType()".to_string()
@@ -1277,8 +1367,10 @@ impl PyTimestampNTZType {
     fn simple_string(&self) -> &'static str {
         "timestamp_ntz"
     }
+    /// As PySpark's, which names this type `timestamp_ntz`.
+    #[classmethod]
     #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
+    fn type_name(_cls: &Bound<'_, pyo3::types::PyType>) -> &'static str {
         "timestamp_ntz"
     }
 }
@@ -1379,10 +1471,6 @@ impl PyArrayType {
             simple_string_with_udts(py, &self.element_type)?
         ))
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "array"
-    }
 
     /// A copy with all nullability fields set true (recursively). Mirrors 
`ArrayType.toNullable()`.
     #[pyo3(name = "toNullable")]
@@ -1531,10 +1619,6 @@ impl PyMapType {
             simple_string_with_udts(py, &self.value_type)?
         ))
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "map"
-    }
 
     /// A copy with all nullability fields set true (recursively). Mirrors 
`MapType.toNullable()`.
     #[pyo3(name = "toNullable")]
@@ -1583,11 +1667,15 @@ impl PyStructField {
             simple_string_with_udts(py, &self.field.data_type)?
         ))
     }
+    /// As PySpark's, `StructField.typeName` raises `INVALID_TYPENAME_CALL`: a 
field has no type
+    /// name, but its `dataType` has.
     #[pyo3(name = "typeName")]
-    fn __sf_type_name(&self) -> PyResult<String> {
-        // Mirrors pyspark: StructField.typeName raises; use typeName on the 
type.
-        Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
-            "StructField does not have typeName. Use typeName on its type 
explicitly instead.",
+    fn __sf_type_name(&self, py: Python<'_>) -> PyResult<String> {
+        Err(crate::column::raise_pyspark(
+            py,
+            "PySparkTypeError",
+            "INVALID_TYPENAME_CALL",
+            &[],
         ))
     }
     #[pyo3(name = "json")]
@@ -2008,10 +2096,6 @@ impl PyStructType {
     }
 
     /// `StructType.typeName()` == "struct".
-    #[pyo3(name = "typeName")]
-    fn __st_type_name(&self) -> &'static str {
-        "struct"
-    }
 
     /// Field names in order. Mirrors `StructType.fieldNames()`.
     #[pyo3(name = "fieldNames")]
@@ -2124,6 +2208,11 @@ pub struct PyCharType {
 
 #[pymethods]
 impl PyCharType {
+    /// The length of the strings, as PySpark's `CharType.length`.
+    #[getter]
+    fn length(&self) -> i32 {
+        self.length
+    }
     // --- DataType object-model methods (v4.2.0 parity) ---
     #[pyo3(name = "json")]
     fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2177,10 +2266,6 @@ impl PyCharType {
     fn simple_string(&self) -> String {
         format!("char({})", self.length)
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "char"
-    }
 }
 
 #[pyclass(name = "VarcharType", module = "pyspark.sql.types", extends = 
PyAtomicType)]
@@ -2190,6 +2275,11 @@ pub struct PyVarcharType {
 
 #[pymethods]
 impl PyVarcharType {
+    /// The maximum length of the strings, as PySpark's `VarcharType.length`.
+    #[getter]
+    fn length(&self) -> i32 {
+        self.length
+    }
     // --- DataType object-model methods (v4.2.0 parity) ---
     #[pyo3(name = "json")]
     fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2243,10 +2333,6 @@ impl PyVarcharType {
     fn simple_string(&self) -> String {
         format!("varchar({})", self.length)
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "varchar"
-    }
 }
 
 #[pyclass(name = "TimeType", module = "pyspark.sql.types", extends = 
PyAnyTimeType)]
@@ -2256,6 +2342,11 @@ pub struct PyTimeType {
 
 #[pymethods]
 impl PyTimeType {
+    /// The precision of the times, as PySpark's `TimeType.precision`.
+    #[getter]
+    fn precision(&self) -> i32 {
+        self.precision
+    }
     // --- DataType object-model methods (v4.2.0 parity) ---
     #[pyo3(name = "json")]
     fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2311,10 +2402,6 @@ impl PyTimeType {
     fn simple_string(&self) -> String {
         format!("time({})", self.precision)
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "time"
-    }
 }
 
 /// Spark's `TimestampNTZNanosType(precision)` (precision 7..=9; default 9). 
PySpark 4.3.0 has no
@@ -2514,9 +2601,19 @@ impl PyCalendarIntervalType {
     fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
         type_reduce(py, "\"calendarinterval\"")
     }
+    /// The one instance of the type, as PySpark's `DataTypeSingleton`: every 
call of the class
+    /// returns it.
     #[new]
-    fn new() -> pyo3::PyClassInitializer<Self> {
-        init_chain!(DataType::CalendarInterval, PyCalendarIntervalType, [])
+    fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+        static INSTANCE: PyOnceLock<Py<PyCalendarIntervalType>> = 
PyOnceLock::new();
+        INSTANCE
+            .get_or_try_init(py, || {
+                Py::new(
+                    py,
+                    init_chain!(DataType::CalendarInterval, 
PyCalendarIntervalType, []),
+                )
+            })
+            .map(|instance| instance.clone_ref(py))
     }
     fn __repr__(&self) -> String {
         "CalendarIntervalType()".to_string()
@@ -2525,9 +2622,11 @@ impl PyCalendarIntervalType {
     fn simple_string(&self) -> &'static str {
         "interval"
     }
+    /// As PySpark's, which names this type `interval`.
+    #[classmethod]
     #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "calendarinterval"
+    fn type_name(_cls: &Bound<'_, pyo3::types::PyType>) -> &'static str {
+        "interval"
     }
 }
 
@@ -2539,6 +2638,16 @@ pub struct PyYearMonthIntervalType {
 
 #[pymethods]
 impl PyYearMonthIntervalType {
+    /// The first field, as PySpark's `YearMonthIntervalType.startField`.
+    #[getter(startField)]
+    fn start_field(&self) -> i32 {
+        self.start_field
+    }
+    /// The last field, as PySpark's `YearMonthIntervalType.endField`.
+    #[getter(endField)]
+    fn end_field(&self) -> i32 {
+        self.end_field
+    }
     #[classattr]
     const YEAR: i32 = 0;
     #[classattr]
@@ -2607,10 +2716,6 @@ impl PyYearMonthIntervalType {
             self.start_field, self.end_field
         )
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "yearmonthinterval"
-    }
 }
 
 #[pyclass(name = "DayTimeIntervalType", module = "pyspark.sql.types", extends 
= PyAnsiIntervalType)]
@@ -2621,6 +2726,16 @@ pub struct PyDayTimeIntervalType {
 
 #[pymethods]
 impl PyDayTimeIntervalType {
+    /// The first field, as PySpark's `DayTimeIntervalType.startField`.
+    #[getter(startField)]
+    fn start_field(&self) -> i32 {
+        self.start_field
+    }
+    /// The last field, as PySpark's `DayTimeIntervalType.endField`.
+    #[getter(endField)]
+    fn end_field(&self) -> i32 {
+        self.end_field
+    }
     #[classattr]
     const DAY: i32 = 0;
     #[classattr]
@@ -2693,10 +2808,6 @@ impl PyDayTimeIntervalType {
             self.start_field, self.end_field
         )
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "daytimeinterval"
-    }
 }
 
 #[pyclass(name = "VariantType", module = "pyspark.sql.types", extends = 
PyAtomicType)]
@@ -2749,10 +2860,6 @@ impl PyVariantType {
     fn simple_string(&self) -> &'static str {
         "variant"
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "variant"
-    }
 }
 
 // Class constants shared by all spatial types (SpatialType is macro-generated 
with no
@@ -2809,10 +2916,6 @@ impl PyGeometryType {
     fn simple_string(&self) -> String {
         DataType::Geometry { srid: self.srid }.simple_string()
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "geometry"
-    }
 }
 
 #[pyclass(name = "GeographyType", module = "pyspark.sql.types", extends = 
PySpatialType)]
@@ -2861,10 +2964,6 @@ impl PyGeographyType {
     fn simple_string(&self) -> String {
         DataType::Geography { srid: self.srid }.simple_string()
     }
-    #[pyo3(name = "typeName")]
-    fn type_name(&self) -> &'static str {
-        "geography"
-    }
 }
 
 /// The SRID of a spatial type that PySpark constructs from `srid`: `"ANY"` 
gives the mixed
@@ -2995,42 +3094,14 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt: 
&DataType) -> PyResult<Py<PyAn
         };
     }
     let o = match dt {
-        DataType::Null => obj!(init_chain!(DataType::Null, PyNullType, [])),
-        DataType::Boolean => obj!(init_chain!(
-            DataType::Boolean,
-            PyBooleanType,
-            [PyAtomicType]
-        )),
-        DataType::Byte => obj!(init_chain!(
-            DataType::Byte,
-            PyByteType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )),
-        DataType::Short => obj!(init_chain!(
-            DataType::Short,
-            PyShortType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )),
-        DataType::Integer => obj!(init_chain!(
-            DataType::Integer,
-            PyIntegerType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )),
-        DataType::Long => obj!(init_chain!(
-            DataType::Long,
-            PyLongType,
-            [PyAtomicType, PyNumericType, PyIntegralType]
-        )),
-        DataType::Float => obj!(init_chain!(
-            DataType::Float,
-            PyFloatType,
-            [PyAtomicType, PyNumericType, PyFractionalType]
-        )),
-        DataType::Double => obj!(init_chain!(
-            DataType::Double,
-            PyDoubleType,
-            [PyAtomicType, PyNumericType, PyFractionalType]
-        )),
+        DataType::Null => PyNullType::new(py)?.into_any(),
+        DataType::Boolean => PyBooleanType::new(py)?.into_any(),
+        DataType::Byte => PyByteType::new(py)?.into_any(),
+        DataType::Short => PyShortType::new(py)?.into_any(),
+        DataType::Integer => PyIntegerType::new(py)?.into_any(),
+        DataType::Long => PyLongType::new(py)?.into_any(),
+        DataType::Float => PyFloatType::new(py)?.into_any(),
+        DataType::Double => PyDoubleType::new(py)?.into_any(),
         DataType::Decimal { precision, scale } => {
             let (precision, scale) = (*precision, *scale);
             obj!(init_chain!(
@@ -3050,22 +3121,10 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt: 
&DataType) -> PyResult<Py<PyAn
             [PyAtomicType],
             value
         )),
-        DataType::Binary => obj!(init_chain!(DataType::Binary, PyBinaryType, 
[PyAtomicType])),
-        DataType::Date => obj!(init_chain!(
-            DataType::Date,
-            PyDateType,
-            [PyAtomicType, PyDatetimeType]
-        )),
-        DataType::Timestamp => obj!(init_chain!(
-            DataType::Timestamp,
-            PyTimestampType,
-            [PyAtomicType, PyDatetimeType]
-        )),
-        DataType::TimestampNtz => obj!(init_chain!(
-            DataType::TimestampNtz,
-            PyTimestampNTZType,
-            [PyAtomicType, PyDatetimeType]
-        )),
+        DataType::Binary => PyBinaryType::new(py)?.into_any(),
+        DataType::Date => PyDateType::new(py)?.into_any(),
+        DataType::Timestamp => PyTimestampType::new(py)?.into_any(),
+        DataType::TimestampNtz => PyTimestampNTZType::new(py)?.into_any(),
         DataType::Array {
             element_type,
             contains_null,
@@ -3155,11 +3214,7 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt: 
&DataType) -> PyResult<Py<PyAn
                 value
             ))
         }
-        DataType::CalendarInterval => obj!(init_chain!(
-            DataType::CalendarInterval,
-            PyCalendarIntervalType,
-            []
-        )),
+        DataType::CalendarInterval => 
PyCalendarIntervalType::new(py)?.into_any(),
         DataType::YearMonthInterval {
             start_field,
             end_field,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 306f50b..5a17747 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2185,6 +2185,21 @@ def 
test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
     assert list(df.first()[0].items()) == [(2, "b"), (1, "a")]
 
 
+def test_types_of_schemas_have_the_attributes_of_pyspark(session):
+    # As PySpark's: the types of a schema of the server have the attributes of 
their
+    # parameters.
+    schema = session.sql(
+        "select 'a' collate UTF8_LCASE as s, interval '1' hour as i, 
time'12:30:00' as t, "
+        "1.5bd as d").schema
+    assert schema["s"].dataType.collation == "UTF8_LCASE"
+    assert (schema["i"].dataType.startField, schema["i"].dataType.endField) == 
(1, 1)
+    assert schema["t"].dataType.precision == 6
+    assert schema["d"].dataType.hasPrecisionInfo is True
+    assert schema["i"].dataType.typeName() == 
type(schema["i"].dataType).typeName()
+
+
+
+
 def test_decimal_literals_are_of_the_types_of_pyspark(session):
     # The types of official PySpark 4.3.0: at least the precision 10 of 
DecimalType().
     from decimal import Decimal
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 4730128..b7f41ee 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -406,6 +406,42 @@ def test_datatypes_usable_in_sets_and_dicts():
     assert len({fa, fa2, fb}) == 2
 
 
+def 
test_data_types_have_their_attributes_type_names_and_singletons_as_pyspark():
+    # As PySpark's: the attributes of the parameters of the types, typeName as 
a classmethod of
+    # the name of the class, INVALID_TYPENAME_CALL of StructField.typeName, 
and the types of
+    # DataTypeSingleton, of which every call returns the one instance.
+    from pyspark.errors import PySparkTypeError
+
+    assert T.StringType("UTF8_LCASE").collation == "UTF8_LCASE"
+    assert T.StringType().collation == "UTF8_BINARY"
+    assert (T.CharType(3).length, T.VarcharType(4).length, 
T.TimeType(3).precision) == (3, 4, 3)
+    interval = T.DayTimeIntervalType(T.DayTimeIntervalType.HOUR, 
T.DayTimeIntervalType.SECOND)
+    assert (interval.startField, interval.endField) == (1, 3)
+    assert (T.YearMonthIntervalType().startField, 
T.YearMonthIntervalType().endField) == (0, 1)
+    assert T.DecimalType(10, 2).hasPrecisionInfo is True
+    for cls, name in ((T.DataType, "data"), (T.AtomicType, "atomic"), 
(T.IntegerType, "integer"),
+                      (T.DayTimeIntervalType, "daytimeinterval"), 
(T.StructType, "struct"),
+                      (T.NullType, "void"), (T.TimestampNTZType, 
"timestamp_ntz"),
+                      (T.CalendarIntervalType, "interval")):
+        assert cls.typeName() == name
+    assert T.CharType(3).typeName() == "char"
+
+    class MyType(T.DataType):
+        pass
+
+    assert MyType.typeName() == MyType().typeName() == "my"
+    with pytest.raises(PySparkTypeError) as e:
+        T.StructField("a", T.IntegerType()).typeName()
+    assert e.value.getCondition() == "INVALID_TYPENAME_CALL"
+    for cls in (T.NullType, T.ByteType, T.IntegerType, T.LongType, 
T.DoubleType, T.BinaryType,
+                T.DateType, T.TimestampNTZType, T.CalendarIntervalType):
+        assert cls() is cls()
+    assert T.StringType() is not T.StringType()
+    assert T.ArrayType(T.IntegerType()).elementType is T.IntegerType()
+
+
+
+
 def test_user_defined_type_eq_and_hash():
     class PointUDT(T.UserDefinedType):
         @classmethod


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to