This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 865a427 [SPARK-61080][CONNECT][PYTHON] Expose the attributes of the
data types, typeName as a classmethod, and the singleton types in the Python
client as PySpark does
865a427 is described below
commit 865a427339be12484e27fd3c95256232c58c4f0e
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 16:10:22 2026 +0900
[SPARK-61080][CONNECT][PYTHON] Expose the attributes of the data types,
typeName as a classmethod, and the singleton types in the Python client as
PySpark does
### What changes were proposed in this pull request?
[SPARK-61080](https://issues.apache.org/jira/browse/SPARK-61080): the data
types now have the attributes, the `typeName` and the singletons of PySpark's
types.
- The types have the attributes of their parameters:
- `StringType("UTF8_LCASE").collation`
- `CharType(3).length` and `VarcharType(4).length`
- `TimeType(3).precision`
- the `startField` and `endField` of `DayTimeIntervalType` and
`YearMonthIntervalType`
- `DecimalType(10, 2).hasPrecisionInfo`
- `typeName` is a classmethod, as PySpark's: `IntegerType.typeName()` is
`integer`, the name of the class without `Type`, lowercased. The same holds for
a subclass of `DataType`. The exceptions are PySpark's `void`, `timestamp_ntz`
and `interval` for `NullType`, `TimestampNTZType` and `CalendarIntervalType`.
- `StructField.typeName()` raises the `PySparkTypeError`
`INVALID_TYPENAME_CALL`.
- The types of PySpark's `DataTypeSingleton` are singletons: `IntegerType()
is IntegerType()`. They are `NullType`, the integral types, `FloatType`,
`DoubleType`, `BinaryType`, `BooleanType`, `DateType`, `TimestampType`,
`TimestampNTZType` and `CalendarIntervalType`. The client also uses these
instances for the types it builds from schemas and from their JSON. PySpark
Connect, by contrast, unpickles a copy of the cached schema in `df.schema`, so
the types there are new objects.
### Why are the changes needed?
- The client lacked these attributes: `StringType("UTF8_LCASE").collation`
raised an `AttributeError`.
- `typeName` was an instance method, so `IntegerType.typeName()` raised a
`TypeError`.
- `CalendarIntervalType().typeName()` was `calendarinterval`, not
`interval`.
- `StructField.typeName()` raised a plain `TypeError`.
- Each call of a type class returned a new object.
### Does this PR introduce _any_ user-facing change?
Yes. The types have the attributes, the type names and the singletons of
PySpark 4.3.0.
### How was this patch tested?
- A new live test checks the attributes of the types of a schema of the
server: a collated string, an interval, a time and a decimal. It passes with
official PySpark 4.3.0 and fails without this change.
- A new offline test checks the attributes, the `typeName` of the classes,
including an abstract one and a subclass of `DataType`, the error of
`StructField.typeName`, and the singletons.
- The `typeName` of 37 classes, a type API sweep of 48 cases, and the
public attributes of 28 type classes are the same as official PySpark 4.3.0's.
- The lib tests of both crates (592 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (2018 passed, 8 skipped), and the
live drop-in and Arrow suites (144 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #281 from HyukjinKwon/SPARK-61080-type-attributes.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/types.rs | 489 ++++++++++++++++++++----------------
python/tests/test_dropin_e2e.py | 15 ++
python/tests/test_dropin_offline.py | 36 +++
3 files changed, 323 insertions(+), 217 deletions(-)
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index e41191b..010125e 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -1,6 +1,7 @@
//! PyO3 wrappers for Spark DataTypes.
use pyo3::prelude::*;
+use pyo3::sync::PyOnceLock;
use spark_connect::types::{DataType, StructField};
use std::collections::BTreeMap;
@@ -305,9 +306,17 @@ impl PyDataType {
.call_method1("_parse_datatype_string", (ddl,))?
.unbind())
}
+ /// The name of the type, as PySpark's classmethod `DataType.typeName`:
the name of the class
+ /// without its `Type`, lowercased, such as `integer` of `IntegerType`,
also of a subclass.
+ #[classmethod]
#[pyo3(name = "typeName")]
- fn __dt_type_name(&self) -> String {
- self.inner.type_name()
+ fn __dt_type_name(cls: &Bound<'_, pyo3::types::PyType>) ->
PyResult<String> {
+ let name = cls.name()?.to_string();
+ let stem: String = name
+ .chars()
+ .take(name.chars().count().saturating_sub(4))
+ .collect();
+ Ok(stem.to_lowercase())
}
/// As pyspark's `DataType.__repr__`: the class name, for the base class
and for
@@ -430,9 +439,16 @@ impl PyNullType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"void\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(DataType::Null, PyNullType, [])
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyNullType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(py, init_chain!(DataType::Null, PyNullType, []))
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"NullType()".to_string()
@@ -441,8 +457,10 @@ impl PyNullType {
fn simple_string(&self) -> &'static str {
"void"
}
+ /// As PySpark's, which names this type `void`.
+ #[classmethod]
#[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
+ fn type_name(_cls: &Bound<'_, pyo3::types::PyType>) -> &'static str {
"void"
}
}
@@ -486,9 +504,19 @@ impl PyBooleanType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"boolean\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(DataType::Boolean, PyBooleanType, [PyAtomicType])
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyBooleanType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(DataType::Boolean, PyBooleanType,
[PyAtomicType]),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"BooleanType()".to_string()
@@ -497,10 +525,6 @@ impl PyBooleanType {
fn simple_string(&self) -> &'static str {
"boolean"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "boolean"
- }
}
#[pyclass(name = "ByteType", module = "pyspark.sql.types", extends =
PyIntegralType)]
@@ -542,13 +566,23 @@ impl PyByteType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"byte\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Byte,
- PyByteType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyByteType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Byte,
+ PyByteType,
+ [PyAtomicType, PyNumericType, PyIntegralType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"ByteType()".to_string()
@@ -557,10 +591,6 @@ impl PyByteType {
fn simple_string(&self) -> &'static str {
"tinyint"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "byte"
- }
}
#[pyclass(name = "ShortType", module = "pyspark.sql.types", extends =
PyIntegralType)]
@@ -602,13 +632,23 @@ impl PyShortType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"short\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Short,
- PyShortType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyShortType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Short,
+ PyShortType,
+ [PyAtomicType, PyNumericType, PyIntegralType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"ShortType()".to_string()
@@ -617,10 +657,6 @@ impl PyShortType {
fn simple_string(&self) -> &'static str {
"smallint"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "short"
- }
}
#[pyclass(name = "IntegerType", module = "pyspark.sql.types", extends =
PyIntegralType)]
@@ -662,13 +698,23 @@ impl PyIntegerType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"integer\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Integer,
- PyIntegerType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyIntegerType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Integer,
+ PyIntegerType,
+ [PyAtomicType, PyNumericType, PyIntegralType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"IntegerType()".to_string()
@@ -677,10 +723,6 @@ impl PyIntegerType {
fn simple_string(&self) -> &'static str {
"int"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "integer"
- }
}
#[pyclass(name = "LongType", module = "pyspark.sql.types", extends =
PyIntegralType)]
@@ -722,13 +764,23 @@ impl PyLongType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"long\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Long,
- PyLongType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyLongType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Long,
+ PyLongType,
+ [PyAtomicType, PyNumericType, PyIntegralType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"LongType()".to_string()
@@ -737,10 +789,6 @@ impl PyLongType {
fn simple_string(&self) -> &'static str {
"bigint"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "long"
- }
}
#[pyclass(name = "FloatType", module = "pyspark.sql.types", extends =
PyFractionalType)]
@@ -782,13 +830,23 @@ impl PyFloatType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"float\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Float,
- PyFloatType,
- [PyAtomicType, PyNumericType, PyFractionalType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyFloatType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Float,
+ PyFloatType,
+ [PyAtomicType, PyNumericType, PyFractionalType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"FloatType()".to_string()
@@ -797,10 +855,6 @@ impl PyFloatType {
fn simple_string(&self) -> &'static str {
"float"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "float"
- }
}
#[pyclass(name = "DoubleType", module = "pyspark.sql.types", extends =
PyFractionalType)]
@@ -842,13 +896,23 @@ impl PyDoubleType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"double\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Double,
- PyDoubleType,
- [PyAtomicType, PyNumericType, PyFractionalType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyDoubleType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Double,
+ PyDoubleType,
+ [PyAtomicType, PyNumericType, PyFractionalType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"DoubleType()".to_string()
@@ -857,10 +921,6 @@ impl PyDoubleType {
fn simple_string(&self) -> &'static str {
"double"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "double"
- }
}
#[pyclass(name = "DecimalType", module = "pyspark.sql.types", extends =
PyFractionalType)]
@@ -871,6 +931,11 @@ pub struct PyDecimalType {
#[pymethods]
impl PyDecimalType {
+ /// As PySpark's `DecimalType.hasPrecisionInfo`, a public attribute that
is always true.
+ #[getter(hasPrecisionInfo)]
+ fn has_precision_info(&self) -> bool {
+ true
+ }
// --- DataType object-model methods (v4.2.0 parity) ---
#[pyo3(name = "json")]
fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -935,10 +1000,6 @@ impl PyDecimalType {
fn simple_string(&self) -> String {
format!("decimal({},{})", self.precision, self.scale)
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "decimal"
- }
}
#[pyclass(name = "StringType", module = "pyspark.sql.types", extends =
PyAtomicType)]
@@ -948,6 +1009,11 @@ pub struct PyStringType {
#[pymethods]
impl PyStringType {
+ /// The collation of the strings, as PySpark's `StringType.collation`.
+ #[getter]
+ fn collation(&self) -> String {
+ self.collation.clone()
+ }
// Collation providers, mirroring
`StringType.providerSpark/providerICU/providers`.
#[classattr]
#[allow(non_upper_case_globals)]
@@ -1024,10 +1090,6 @@ impl PyStringType {
format!("string collate {}", self.collation)
}
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "string"
- }
/// Provider ("spark" for UTF8*, else "icu") for a collation name.
/// Mirrors `StringType.collationProvider`.
@@ -1087,9 +1149,19 @@ impl PyBinaryType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"binary\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(DataType::Binary, PyBinaryType, [PyAtomicType])
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyBinaryType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(DataType::Binary, PyBinaryType,
[PyAtomicType]),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"BinaryType()".to_string()
@@ -1098,10 +1170,6 @@ impl PyBinaryType {
fn simple_string(&self) -> &'static str {
"binary"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "binary"
- }
}
#[pyclass(name = "DateType", module = "pyspark.sql.types", extends =
PyDatetimeType)]
@@ -1146,9 +1214,19 @@ impl PyDateType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"date\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(DataType::Date, PyDateType, [PyAtomicType, PyDatetimeType])
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyDateType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(DataType::Date, PyDateType, [PyAtomicType,
PyDatetimeType]),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"DateType()".to_string()
@@ -1157,10 +1235,6 @@ impl PyDateType {
fn simple_string(&self) -> &'static str {
"date"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "date"
- }
}
#[pyclass(name = "TimestampType", module = "pyspark.sql.types", extends =
PyDatetimeType)]
@@ -1202,13 +1276,23 @@ impl PyTimestampType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"timestamp\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::Timestamp,
- PyTimestampType,
- [PyAtomicType, PyDatetimeType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyTimestampType>> = PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::Timestamp,
+ PyTimestampType,
+ [PyAtomicType, PyDatetimeType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"TimestampType()".to_string()
@@ -1217,10 +1301,6 @@ impl PyTimestampType {
fn simple_string(&self) -> &'static str {
"timestamp"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "timestamp"
- }
}
#[pyclass(name = "TimestampNTZType", module = "pyspark.sql.types", extends =
PyDatetimeType)]
@@ -1262,13 +1342,23 @@ impl PyTimestampNTZType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"timestamp_ntz\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(
- DataType::TimestampNtz,
- PyTimestampNTZType,
- [PyAtomicType, PyDatetimeType]
- )
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyTimestampNTZType>> =
PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(
+ DataType::TimestampNtz,
+ PyTimestampNTZType,
+ [PyAtomicType, PyDatetimeType]
+ ),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"TimestampNTZType()".to_string()
@@ -1277,8 +1367,10 @@ impl PyTimestampNTZType {
fn simple_string(&self) -> &'static str {
"timestamp_ntz"
}
+ /// As PySpark's, which names this type `timestamp_ntz`.
+ #[classmethod]
#[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
+ fn type_name(_cls: &Bound<'_, pyo3::types::PyType>) -> &'static str {
"timestamp_ntz"
}
}
@@ -1379,10 +1471,6 @@ impl PyArrayType {
simple_string_with_udts(py, &self.element_type)?
))
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "array"
- }
/// A copy with all nullability fields set true (recursively). Mirrors
`ArrayType.toNullable()`.
#[pyo3(name = "toNullable")]
@@ -1531,10 +1619,6 @@ impl PyMapType {
simple_string_with_udts(py, &self.value_type)?
))
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "map"
- }
/// A copy with all nullability fields set true (recursively). Mirrors
`MapType.toNullable()`.
#[pyo3(name = "toNullable")]
@@ -1583,11 +1667,15 @@ impl PyStructField {
simple_string_with_udts(py, &self.field.data_type)?
))
}
+ /// As PySpark's, `StructField.typeName` raises `INVALID_TYPENAME_CALL`: a
field has no type
+ /// name, but its `dataType` has.
#[pyo3(name = "typeName")]
- fn __sf_type_name(&self) -> PyResult<String> {
- // Mirrors pyspark: StructField.typeName raises; use typeName on the
type.
- Err(PyErr::new::<pyo3::exceptions::PyTypeError, _>(
- "StructField does not have typeName. Use typeName on its type
explicitly instead.",
+ fn __sf_type_name(&self, py: Python<'_>) -> PyResult<String> {
+ Err(crate::column::raise_pyspark(
+ py,
+ "PySparkTypeError",
+ "INVALID_TYPENAME_CALL",
+ &[],
))
}
#[pyo3(name = "json")]
@@ -2008,10 +2096,6 @@ impl PyStructType {
}
/// `StructType.typeName()` == "struct".
- #[pyo3(name = "typeName")]
- fn __st_type_name(&self) -> &'static str {
- "struct"
- }
/// Field names in order. Mirrors `StructType.fieldNames()`.
#[pyo3(name = "fieldNames")]
@@ -2124,6 +2208,11 @@ pub struct PyCharType {
#[pymethods]
impl PyCharType {
+ /// The length of the strings, as PySpark's `CharType.length`.
+ #[getter]
+ fn length(&self) -> i32 {
+ self.length
+ }
// --- DataType object-model methods (v4.2.0 parity) ---
#[pyo3(name = "json")]
fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2177,10 +2266,6 @@ impl PyCharType {
fn simple_string(&self) -> String {
format!("char({})", self.length)
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "char"
- }
}
#[pyclass(name = "VarcharType", module = "pyspark.sql.types", extends =
PyAtomicType)]
@@ -2190,6 +2275,11 @@ pub struct PyVarcharType {
#[pymethods]
impl PyVarcharType {
+ /// The maximum length of the strings, as PySpark's `VarcharType.length`.
+ #[getter]
+ fn length(&self) -> i32 {
+ self.length
+ }
// --- DataType object-model methods (v4.2.0 parity) ---
#[pyo3(name = "json")]
fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2243,10 +2333,6 @@ impl PyVarcharType {
fn simple_string(&self) -> String {
format!("varchar({})", self.length)
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "varchar"
- }
}
#[pyclass(name = "TimeType", module = "pyspark.sql.types", extends =
PyAnyTimeType)]
@@ -2256,6 +2342,11 @@ pub struct PyTimeType {
#[pymethods]
impl PyTimeType {
+ /// The precision of the times, as PySpark's `TimeType.precision`.
+ #[getter]
+ fn precision(&self) -> i32 {
+ self.precision
+ }
// --- DataType object-model methods (v4.2.0 parity) ---
#[pyo3(name = "json")]
fn __obj_json(slf: &Bound<'_, Self>) -> PyResult<String> {
@@ -2311,10 +2402,6 @@ impl PyTimeType {
fn simple_string(&self) -> String {
format!("time({})", self.precision)
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "time"
- }
}
/// Spark's `TimestampNTZNanosType(precision)` (precision 7..=9; default 9).
PySpark 4.3.0 has no
@@ -2514,9 +2601,19 @@ impl PyCalendarIntervalType {
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"calendarinterval\"")
}
+ /// The one instance of the type, as PySpark's `DataTypeSingleton`: every
call of the class
+ /// returns it.
#[new]
- fn new() -> pyo3::PyClassInitializer<Self> {
- init_chain!(DataType::CalendarInterval, PyCalendarIntervalType, [])
+ fn new(py: Python<'_>) -> PyResult<Py<Self>> {
+ static INSTANCE: PyOnceLock<Py<PyCalendarIntervalType>> =
PyOnceLock::new();
+ INSTANCE
+ .get_or_try_init(py, || {
+ Py::new(
+ py,
+ init_chain!(DataType::CalendarInterval,
PyCalendarIntervalType, []),
+ )
+ })
+ .map(|instance| instance.clone_ref(py))
}
fn __repr__(&self) -> String {
"CalendarIntervalType()".to_string()
@@ -2525,9 +2622,11 @@ impl PyCalendarIntervalType {
fn simple_string(&self) -> &'static str {
"interval"
}
+ /// As PySpark's, which names this type `interval`.
+ #[classmethod]
#[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "calendarinterval"
+ fn type_name(_cls: &Bound<'_, pyo3::types::PyType>) -> &'static str {
+ "interval"
}
}
@@ -2539,6 +2638,16 @@ pub struct PyYearMonthIntervalType {
#[pymethods]
impl PyYearMonthIntervalType {
+ /// The first field, as PySpark's `YearMonthIntervalType.startField`.
+ #[getter(startField)]
+ fn start_field(&self) -> i32 {
+ self.start_field
+ }
+ /// The last field, as PySpark's `YearMonthIntervalType.endField`.
+ #[getter(endField)]
+ fn end_field(&self) -> i32 {
+ self.end_field
+ }
#[classattr]
const YEAR: i32 = 0;
#[classattr]
@@ -2607,10 +2716,6 @@ impl PyYearMonthIntervalType {
self.start_field, self.end_field
)
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "yearmonthinterval"
- }
}
#[pyclass(name = "DayTimeIntervalType", module = "pyspark.sql.types", extends
= PyAnsiIntervalType)]
@@ -2621,6 +2726,16 @@ pub struct PyDayTimeIntervalType {
#[pymethods]
impl PyDayTimeIntervalType {
+ /// The first field, as PySpark's `DayTimeIntervalType.startField`.
+ #[getter(startField)]
+ fn start_field(&self) -> i32 {
+ self.start_field
+ }
+ /// The last field, as PySpark's `DayTimeIntervalType.endField`.
+ #[getter(endField)]
+ fn end_field(&self) -> i32 {
+ self.end_field
+ }
#[classattr]
const DAY: i32 = 0;
#[classattr]
@@ -2693,10 +2808,6 @@ impl PyDayTimeIntervalType {
self.start_field, self.end_field
)
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "daytimeinterval"
- }
}
#[pyclass(name = "VariantType", module = "pyspark.sql.types", extends =
PyAtomicType)]
@@ -2749,10 +2860,6 @@ impl PyVariantType {
fn simple_string(&self) -> &'static str {
"variant"
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "variant"
- }
}
// Class constants shared by all spatial types (SpatialType is macro-generated
with no
@@ -2809,10 +2916,6 @@ impl PyGeometryType {
fn simple_string(&self) -> String {
DataType::Geometry { srid: self.srid }.simple_string()
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "geometry"
- }
}
#[pyclass(name = "GeographyType", module = "pyspark.sql.types", extends =
PySpatialType)]
@@ -2861,10 +2964,6 @@ impl PyGeographyType {
fn simple_string(&self) -> String {
DataType::Geography { srid: self.srid }.simple_string()
}
- #[pyo3(name = "typeName")]
- fn type_name(&self) -> &'static str {
- "geography"
- }
}
/// The SRID of a spatial type that PySpark constructs from `srid`: `"ANY"`
gives the mixed
@@ -2995,42 +3094,14 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt:
&DataType) -> PyResult<Py<PyAn
};
}
let o = match dt {
- DataType::Null => obj!(init_chain!(DataType::Null, PyNullType, [])),
- DataType::Boolean => obj!(init_chain!(
- DataType::Boolean,
- PyBooleanType,
- [PyAtomicType]
- )),
- DataType::Byte => obj!(init_chain!(
- DataType::Byte,
- PyByteType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )),
- DataType::Short => obj!(init_chain!(
- DataType::Short,
- PyShortType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )),
- DataType::Integer => obj!(init_chain!(
- DataType::Integer,
- PyIntegerType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )),
- DataType::Long => obj!(init_chain!(
- DataType::Long,
- PyLongType,
- [PyAtomicType, PyNumericType, PyIntegralType]
- )),
- DataType::Float => obj!(init_chain!(
- DataType::Float,
- PyFloatType,
- [PyAtomicType, PyNumericType, PyFractionalType]
- )),
- DataType::Double => obj!(init_chain!(
- DataType::Double,
- PyDoubleType,
- [PyAtomicType, PyNumericType, PyFractionalType]
- )),
+ DataType::Null => PyNullType::new(py)?.into_any(),
+ DataType::Boolean => PyBooleanType::new(py)?.into_any(),
+ DataType::Byte => PyByteType::new(py)?.into_any(),
+ DataType::Short => PyShortType::new(py)?.into_any(),
+ DataType::Integer => PyIntegerType::new(py)?.into_any(),
+ DataType::Long => PyLongType::new(py)?.into_any(),
+ DataType::Float => PyFloatType::new(py)?.into_any(),
+ DataType::Double => PyDoubleType::new(py)?.into_any(),
DataType::Decimal { precision, scale } => {
let (precision, scale) = (*precision, *scale);
obj!(init_chain!(
@@ -3050,22 +3121,10 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt:
&DataType) -> PyResult<Py<PyAn
[PyAtomicType],
value
)),
- DataType::Binary => obj!(init_chain!(DataType::Binary, PyBinaryType,
[PyAtomicType])),
- DataType::Date => obj!(init_chain!(
- DataType::Date,
- PyDateType,
- [PyAtomicType, PyDatetimeType]
- )),
- DataType::Timestamp => obj!(init_chain!(
- DataType::Timestamp,
- PyTimestampType,
- [PyAtomicType, PyDatetimeType]
- )),
- DataType::TimestampNtz => obj!(init_chain!(
- DataType::TimestampNtz,
- PyTimestampNTZType,
- [PyAtomicType, PyDatetimeType]
- )),
+ DataType::Binary => PyBinaryType::new(py)?.into_any(),
+ DataType::Date => PyDateType::new(py)?.into_any(),
+ DataType::Timestamp => PyTimestampType::new(py)?.into_any(),
+ DataType::TimestampNtz => PyTimestampNTZType::new(py)?.into_any(),
DataType::Array {
element_type,
contains_null,
@@ -3155,11 +3214,7 @@ pub(crate) fn data_type_to_py(py: Python<'_>, dt:
&DataType) -> PyResult<Py<PyAn
value
))
}
- DataType::CalendarInterval => obj!(init_chain!(
- DataType::CalendarInterval,
- PyCalendarIntervalType,
- []
- )),
+ DataType::CalendarInterval =>
PyCalendarIntervalType::new(py)?.into_any(),
DataType::YearMonthInterval {
start_field,
end_field,
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index 306f50b..5a17747 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -2185,6 +2185,21 @@ def
test_maps_keep_their_entries_in_order_with_keys_of_their_type(session):
assert list(df.first()[0].items()) == [(2, "b"), (1, "a")]
+def test_types_of_schemas_have_the_attributes_of_pyspark(session):
+ # As PySpark's: the types of a schema of the server have the attributes of
their
+ # parameters.
+ schema = session.sql(
+ "select 'a' collate UTF8_LCASE as s, interval '1' hour as i,
time'12:30:00' as t, "
+ "1.5bd as d").schema
+ assert schema["s"].dataType.collation == "UTF8_LCASE"
+ assert (schema["i"].dataType.startField, schema["i"].dataType.endField) ==
(1, 1)
+ assert schema["t"].dataType.precision == 6
+ assert schema["d"].dataType.hasPrecisionInfo is True
+ assert schema["i"].dataType.typeName() ==
type(schema["i"].dataType).typeName()
+
+
+
+
def test_decimal_literals_are_of_the_types_of_pyspark(session):
# The types of official PySpark 4.3.0: at least the precision 10 of
DecimalType().
from decimal import Decimal
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 4730128..b7f41ee 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -406,6 +406,42 @@ def test_datatypes_usable_in_sets_and_dicts():
assert len({fa, fa2, fb}) == 2
+def
test_data_types_have_their_attributes_type_names_and_singletons_as_pyspark():
+ # As PySpark's: the attributes of the parameters of the types, typeName as
a classmethod of
+ # the name of the class, INVALID_TYPENAME_CALL of StructField.typeName,
and the types of
+ # DataTypeSingleton, of which every call returns the one instance.
+ from pyspark.errors import PySparkTypeError
+
+ assert T.StringType("UTF8_LCASE").collation == "UTF8_LCASE"
+ assert T.StringType().collation == "UTF8_BINARY"
+ assert (T.CharType(3).length, T.VarcharType(4).length,
T.TimeType(3).precision) == (3, 4, 3)
+ interval = T.DayTimeIntervalType(T.DayTimeIntervalType.HOUR,
T.DayTimeIntervalType.SECOND)
+ assert (interval.startField, interval.endField) == (1, 3)
+ assert (T.YearMonthIntervalType().startField,
T.YearMonthIntervalType().endField) == (0, 1)
+ assert T.DecimalType(10, 2).hasPrecisionInfo is True
+ for cls, name in ((T.DataType, "data"), (T.AtomicType, "atomic"),
(T.IntegerType, "integer"),
+ (T.DayTimeIntervalType, "daytimeinterval"),
(T.StructType, "struct"),
+ (T.NullType, "void"), (T.TimestampNTZType,
"timestamp_ntz"),
+ (T.CalendarIntervalType, "interval")):
+ assert cls.typeName() == name
+ assert T.CharType(3).typeName() == "char"
+
+ class MyType(T.DataType):
+ pass
+
+ assert MyType.typeName() == MyType().typeName() == "my"
+ with pytest.raises(PySparkTypeError) as e:
+ T.StructField("a", T.IntegerType()).typeName()
+ assert e.value.getCondition() == "INVALID_TYPENAME_CALL"
+ for cls in (T.NullType, T.ByteType, T.IntegerType, T.LongType,
T.DoubleType, T.BinaryType,
+ T.DateType, T.TimestampNTZType, T.CalendarIntervalType):
+ assert cls() is cls()
+ assert T.StringType() is not T.StringType()
+ assert T.ArrayType(T.IntegerType()).elementType is T.IntegerType()
+
+
+
+
def test_user_defined_type_eq_and_hash():
class PointUDT(T.UserDefinedType):
@classmethod
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]