This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new ad8cf98 [SPARK-61075][CONNECT][PYTHON] Convert values in toInternal
and fromInternal of data types in the Python client as PySpark does
ad8cf98 is described below
commit ad8cf98dee578cdf7011c48a7f952b6b7c367616
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 15:34:13 2026 +0900
[SPARK-61075][CONNECT][PYTHON] Convert values in toInternal and
fromInternal of data types in the Python client as PySpark does
### What changes were proposed in this pull request?
[SPARK-61075](https://issues.apache.org/jira/browse/SPARK-61075): the
`toInternal` and `fromInternal` of the data types now convert values to the
internal values of Spark and back, as PySpark's do.
As PySpark's:
- a date is the days since the epoch, `DateType().toInternal(date(2020, 1,
1)) == 18262`
- a timestamp is the microseconds since the epoch, of the local time zone
or of its own
- a timestamp without a time zone is the microseconds of its wall clock in
UTC
- a time is the nanoseconds since midnight, and a day-time interval its
microseconds
- a variant, a geometry and a geography are the dicts of their values
- the elements, keys, values and fields of arrays, maps and structs are
converted
- a struct takes a tuple, a list, a `Row`, a dict or an object, and its
`fromInternal` gives a `Row`
- year-month and calendar intervals raise `NOT_IMPLEMENTED`
`needConversion` is true for variants, spatial types and user-defined
types, also in arrays and maps. It is false for a year-month interval with
`PYSPARK_YM_INTERVAL_LEGACY=1`. `StructType` has PySpark's
`_needSerializeAnyField` and `_needConversion`.
The conversions are those of the shim `pyspark.sql.types`, which follow
PySpark's code. The core's `need_conversion` is true for variants, spatial
types and UDTs.
### Why are the changes needed?
The client returned the value itself for every data type:
- `TimestampType().toInternal(datetime(2020, 1, 1))` returned the datetime
rather than `1577836800000000` (in UTC)
- `DateType().fromInternal(18262)` returned `18262` rather than the date
2020-01-01
- `StructType.fromInternal` returned the tuple rather than a `Row`
### Does this PR introduce _any_ user-facing change?
Yes. These methods convert values as in PySpark 4.3.0.
### How was this patch tested?
- A new live test converts a collected row of dates, timestamps, an
interval, an array and a map with `toInternal`. It checks the result against
the internal values that the server computes with `unix_date` and
`unix_micros`, and checks that `fromInternal` gives the row back. It passes
with official PySpark 4.3.0 and fails without this change.
- A new offline test checks the conversions of each type, the structs of
tuples, dicts and rows, the errors, and `needConversion`, including
`PYSPARK_YM_INTERVAL_LEGACY`.
- The core lib test of `need_conversion` checks variants, spatial types and
UDTs.
- On 75 cases of the conversions, and 23 more of user-defined types, nested
types and the legacy flag, the client gives the same results as official
PySpark 4.3.0 in the UTC, America/Los_Angeles and Asia/Seoul time zones.
- The lib tests of both crates (589 passed), the build of all tests, also
with `wasm-udf`, the offline Python suites (1978 passed, 8 skipped), and the
live drop-in and Arrow suites (134 passed) and live cases (10 passed) against a
local Spark 4.3.0 server. Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #277 from HyukjinKwon/SPARK-61075-internal-conversions.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/types.rs | 503 ++++++++++++++++++++++++++----------
crates/spark-connect/src/types.rs | 25 +-
python/pyspark/sql/types.py | 187 ++++++++++++++
python/tests/test_dropin_e2e.py | 20 ++
python/tests/test_dropin_offline.py | 69 +++++
5 files changed, 662 insertions(+), 142 deletions(-)
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 201077e..e41191b 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -188,6 +188,39 @@ fn py_json_value<'py>(
.call1((ordered_json_text(v, field),))
}
+/// The `toInternal` or `fromInternal` (the `_to_internal` or `_from_internal`
of the shim
+/// `pyspark.sql.types`) of `data_type`, which convert values as those of
PySpark's types do.
+fn shim_conversion<'py>(
+ data_type: &Bound<'py, PyAny>,
+ conversion: &str,
+ obj: Bound<'py, PyAny>,
+) -> PyResult<Bound<'py, PyAny>> {
+ data_type
+ .py()
+ .import("pyspark.sql.types")?
+ .getattr(conversion)?
+ .call1((data_type, obj))
+}
+
+/// `needConversion` of `data_type`, as PySpark's: that of a year-month
interval is false with
+/// `PYSPARK_YM_INTERVAL_LEGACY=1`, with which `df.collect` returns the
internal integers.
+fn need_conversion(py: Python<'_>, data_type: &DataType) -> PyResult<bool> {
+ Ok(match data_type {
+ DataType::YearMonthInterval { .. } => !py
+ .import("os")?
+ .getattr("environ")?
+ .call_method1("get", ("PYSPARK_YM_INTERVAL_LEGACY",))?
+ .eq("1")?,
+ DataType::Array { element_type, .. } => need_conversion(py,
element_type)?,
+ DataType::Map {
+ key_type,
+ value_type,
+ ..
+ } => need_conversion(py, key_type)? || need_conversion(py,
value_type)?,
+ other => other.need_conversion(),
+ })
+}
+
/// Convert the core JSON-value metadata map back to a Python dict.
fn metadata_map_to_py<'py>(
py: Python<'py>,
@@ -243,15 +276,21 @@ impl PyDataType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
/// The type of the DDL string `ddl`, as PySpark's `DataType.fromDDL`,
which the server of
/// the active session parses (`_parse_datatype_string`): the concrete
type, a StructType
@@ -372,15 +411,21 @@ impl PyNullType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"void\"")
@@ -422,15 +467,21 @@ impl PyBooleanType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"boolean\"")
@@ -472,15 +523,21 @@ impl PyByteType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"byte\"")
@@ -526,15 +583,21 @@ impl PyShortType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"short\"")
@@ -580,15 +643,21 @@ impl PyIntegerType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"integer\"")
@@ -634,15 +703,21 @@ impl PyLongType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"long\"")
@@ -688,15 +763,21 @@ impl PyFloatType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"float\"")
@@ -742,15 +823,21 @@ impl PyDoubleType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"double\"")
@@ -799,15 +886,21 @@ impl PyDecimalType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Decimal {
@@ -882,15 +975,21 @@ impl PyStringType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::String {
@@ -969,15 +1068,21 @@ impl PyBinaryType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"binary\"")
@@ -1022,15 +1127,21 @@ impl PyDateType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"date\"")
@@ -1072,15 +1183,21 @@ impl PyTimestampType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"timestamp\"")
@@ -1126,15 +1243,21 @@ impl PyTimestampNTZType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"timestamp_ntz\"")
@@ -1194,15 +1317,21 @@ impl PyArrayType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Array {
@@ -1323,15 +1452,21 @@ impl PyMapType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Map {
@@ -1465,16 +1600,22 @@ impl PyStructField {
py_json_value(py, &v, true)
}
#[pyo3(name = "needConversion")]
- fn __sf_need_conversion(&self) -> bool {
- self.field.data_type.need_conversion()
+ fn __sf_need_conversion(&self, py: Python<'_>) -> PyResult<bool> {
+ need_conversion(py, &self.field.data_type)
}
#[pyo3(name = "fromInternal")]
- fn __sf_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __sf_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __sf_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __sf_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
/// Collation metadata for the field (empty when the field has no
collations).
#[pyo3(name = "getCollationMetadata")]
@@ -1654,15 +1795,21 @@ impl PyStructType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Struct {
@@ -1872,6 +2019,26 @@ impl PyStructType {
self.fields.iter().map(|f| f.name.clone()).collect()
}
+ /// Whether a field needs conversion, as PySpark's
`_needSerializeAnyField`.
+ #[getter(_needSerializeAnyField)]
+ fn __st_need_serialize_any_field(&self, py: Python<'_>) -> PyResult<bool> {
+ for field in &self.fields {
+ if need_conversion(py, &field.data_type)? {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+ }
+
+ /// The `needConversion` of each field, as PySpark's `_needConversion`.
+ #[getter(_needConversion)]
+ fn __st_need_conversion_of_fields(&self, py: Python<'_>) ->
PyResult<Vec<bool>> {
+ self.fields
+ .iter()
+ .map(|field| need_conversion(py, &field.data_type))
+ .collect()
+ }
+
/// DDL string. Mirrors `StructType.toDDL()`, which asks the server of the
active session to
/// convert the JSON of the schema, as Spark writes DDL.
#[pyo3(name = "toDDL")]
@@ -1972,15 +2139,21 @@ impl PyCharType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Char {
@@ -2032,15 +2205,21 @@ impl PyVarcharType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Varchar {
@@ -2092,15 +2271,21 @@ impl PyTimeType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::Time {
@@ -2156,15 +2341,21 @@ impl PyTimestampNTZNanosType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::TimestampNtzNanos {
@@ -2229,15 +2420,21 @@ impl PyTimestampLTZNanosType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::TimestampLtzNanos {
@@ -2298,15 +2495,21 @@ impl PyCalendarIntervalType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"calendarinterval\"")
@@ -2355,15 +2558,21 @@ impl PyYearMonthIntervalType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::YearMonthInterval {
@@ -2435,15 +2644,21 @@ impl PyDayTimeIntervalType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", obj)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
let dt = DataType::DayTimeInterval {
@@ -2504,15 +2719,21 @@ impl PyVariantType {
}
#[pyo3(name = "needConversion")]
fn __obj_need_conversion(slf: &Bound<'_, Self>) -> PyResult<bool> {
- Ok(py_to_data_type(slf.as_any())?.need_conversion())
+ need_conversion(slf.py(), &py_to_data_type(slf.as_any())?)
}
#[pyo3(name = "fromInternal")]
- fn __obj_from_internal<'py>(&self, obj: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- obj
+ fn __obj_from_internal<'py>(
+ slf: &Bound<'py, Self>,
+ obj: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_from_internal", obj)
}
#[pyo3(name = "toInternal")]
- fn __obj_to_internal<'py>(&self, variant: Bound<'py, PyAny>) -> Bound<'py,
PyAny> {
- variant
+ fn __obj_to_internal<'py>(
+ slf: &Bound<'py, Self>,
+ variant: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ shim_conversion(slf.as_any(), "_to_internal", variant)
}
fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py<PyAny>, (String,))> {
type_reduce(py, "\"variant\"")
diff --git a/crates/spark-connect/src/types.rs
b/crates/spark-connect/src/types.rs
index 4a8d4f6..0a40263 100644
--- a/crates/spark-connect/src/types.rs
+++ b/crates/spark-connect/src/types.rs
@@ -190,6 +190,9 @@ impl DataType {
/// - DayTimeIntervalType: needs conversion to/from datetime.timedelta
/// - CalendarIntervalType: needs conversion
/// - YearMonthIntervalType: needs conversion (complex)
+ /// - VariantType: needs conversion to/from `VariantVal`
+ /// - GeometryType and GeographyType: need conversion to/from `Geometry`
and `Geography`
+ /// - UserDefinedType: needs conversion to/from its Python class
/// - ArrayType: if element type needs conversion
/// - MapType: if key or value type needs conversion
/// - StructType: always needs conversion
@@ -203,7 +206,11 @@ impl DataType {
| DataType::Time { .. }
| DataType::DayTimeInterval { .. }
| DataType::YearMonthInterval { .. }
- | DataType::CalendarInterval => true,
+ | DataType::CalendarInterval
+ | DataType::Variant
+ | DataType::Geometry { .. }
+ | DataType::Geography { .. }
+ | DataType::Udt { .. } => true,
DataType::Array { element_type, .. } =>
element_type.need_conversion(),
DataType::Map {
key_type,
@@ -2858,6 +2865,22 @@ mod tests {
assert!(DataType::TimestampNtz.need_conversion());
assert!(DataType::CalendarInterval.need_conversion());
assert!(DataType::Time { precision: 6 }.need_conversion());
+ // As PySpark's: variants, spatial values and the values of UDTs, also
in arrays.
+ assert!(DataType::Variant.need_conversion());
+ assert!(DataType::Geometry { srid: 4326 }.need_conversion());
+ assert!(DataType::Geography { srid: 4326 }.need_conversion());
+ let udt = DataType::Udt {
+ type_str: "udt".to_string(),
+ jvm_class: None,
+ python_class: Some("mymod.MyUDT".to_string()),
+ serialized_python_class: None,
+ sql_type: Some(Box::new(DataType::Integer)),
+ };
+ assert!(DataType::Array {
+ element_type: Box::new(udt),
+ contains_null: true,
+ }
+ .need_conversion());
// Types that don't need conversion
assert!(!DataType::Integer.need_conversion());
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index d7800c1..cced81b 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -560,3 +560,190 @@ def _udt_field_converters(schema):
field has a user-defined type. The client applies them to the rows that it
collects."""
converters = [_udt_converter(field.dataType) for field in schema.fields]
return None if all(c is None for c in converters) else converters
+
+
+def _to_internal(dataType, obj):
+ """The internal SQL value of the Python value ``obj``, as the
``toInternal`` of the PySpark
+ type of ``dataType``: the days of a date since the epoch, the microseconds
of a timestamp
+ and of a day-time interval, the nanoseconds of a time, and the dicts of
variants and spatial
+ values, also in the elements, keys, values and fields of arrays, maps and
structs."""
+ import calendar
+ import time
+
+ from pyspark.errors import PySparkNotImplementedError
+
+ if isinstance(dataType, StructField):
+ return dataType.dataType.toInternal(obj)
+ elif isinstance(dataType, DateType):
+ if obj is not None:
+ return obj.toordinal() - DateType.EPOCH_ORDINAL
+ elif isinstance(dataType, TimeType):
+ if obj is not None:
+ return (
+ obj.hour * 3_600_000_000_000
+ + obj.minute * 60_000_000_000
+ + obj.second * 1_000_000_000
+ + obj.microsecond * 1_000
+ )
+ elif isinstance(dataType, TimestampType):
+ if obj is not None:
+ seconds = (
+ calendar.timegm(obj.utctimetuple()) if obj.tzinfo else
time.mktime(obj.timetuple())
+ )
+ return int(seconds) * 1000000 + obj.microsecond
+ elif isinstance(dataType, TimestampNTZType):
+ if obj is not None:
+ seconds = calendar.timegm(obj.timetuple())
+ return int(seconds) * 1000000 + obj.microsecond
+ elif isinstance(dataType, DayTimeIntervalType):
+ if obj is not None:
+ return (((obj.days * 86400) + obj.seconds) * 1_000_000) +
obj.microseconds
+ elif isinstance(dataType, (YearMonthIntervalType, CalendarIntervalType)):
+ raise PySparkNotImplementedError(
+ errorClass="NOT_IMPLEMENTED",
+ messageParameters={"feature": "%s.toInternal" %
type(dataType).__name__},
+ )
+ elif isinstance(dataType, VariantType):
+ if obj is None:
+ return None
+ assert isinstance(obj, VariantVal)
+ return {"value": obj.value, "metadata": obj.metadata}
+ elif isinstance(dataType, (GeometryType, GeographyType)):
+ if obj is None:
+ return None
+ assert isinstance(obj, Geometry if isinstance(dataType, GeometryType)
else Geography)
+ return {"srid": obj.srid, "wkb": obj.wkb}
+ elif isinstance(dataType, ArrayType):
+ if not dataType.needConversion():
+ return obj
+ return obj and [dataType.elementType.toInternal(v) for v in obj]
+ elif isinstance(dataType, MapType):
+ if not dataType.needConversion():
+ return obj
+ return obj and dict(
+ (dataType.keyType.toInternal(k), dataType.valueType.toInternal(v))
+ for k, v in obj.items()
+ )
+ elif isinstance(dataType, StructType):
+ return _struct_to_internal(dataType, obj)
+ else:
+ return obj
+
+
+def _struct_to_internal(dataType, obj):
+ """``StructType.toInternal``, as PySpark's: the tuple of the values of the
fields of a
+ dict, a tuple, a list, a Row or an object, converted where a field needs
it."""
+ from pyspark.errors import PySparkValueError
+
+ if obj is None:
+ return
+ if isinstance(obj, VariantVal):
+ raise PySparkValueError("Rows cannot be of type VariantVal")
+ names, fields = dataType.names, dataType.fields
+ # A Row is a tuple in PySpark.
+ if dataType._needSerializeAnyField:
+ # Only calling toInternal function for fields that need conversion
+ need = dataType._needConversion
+ if isinstance(obj, dict):
+ return tuple(
+ f.toInternal(obj.get(n)) if c else obj.get(n)
+ for n, f, c in zip(names, fields, need)
+ )
+ elif isinstance(obj, (tuple, list, Row)):
+ return tuple(f.toInternal(v) if c else v for f, v, c in
zip(fields, obj, need))
+ elif hasattr(obj, "__dict__"):
+ d = obj.__dict__
+ return tuple(
+ f.toInternal(d.get(n)) if c else d.get(n) for n, f, c in
zip(names, fields, need)
+ )
+ else:
+ if isinstance(obj, dict):
+ return tuple(obj.get(n) for n in names)
+ elif isinstance(obj, (list, tuple, Row)):
+ return tuple(obj)
+ elif hasattr(obj, "__dict__"):
+ d = obj.__dict__
+ return tuple(d.get(n) for n in names)
+ raise PySparkValueError(
+ errorClass="UNEXPECTED_TUPLE_WITH_STRUCT",
+ messageParameters={"tuple": str(obj)},
+ )
+
+
+def _from_internal(dataType, obj):
+ """The Python value of the internal SQL value ``obj``, as the
``fromInternal`` of the
+ PySpark type of ``dataType``: the inverse of ``_to_internal``, of which a
struct is a
+ Row."""
+ import datetime
+
+ from pyspark.errors import PySparkNotImplementedError
+
+ if isinstance(dataType, StructField):
+ return dataType.dataType.fromInternal(obj)
+ elif isinstance(dataType, DateType):
+ if obj is not None:
+ return datetime.date.fromordinal(obj + DateType.EPOCH_ORDINAL)
+ elif isinstance(dataType, TimeType):
+ if obj is not None:
+ hours, remainder = divmod(obj, 3_600_000_000_000)
+ minutes, remainder = divmod(remainder, 60_000_000_000)
+ seconds, remainder = divmod(remainder, 1_000_000_000)
+ microseconds = remainder // 1_000
+ return datetime.time(hours, minutes, seconds, microseconds)
+ elif isinstance(dataType, TimestampType):
+ if obj is not None:
+ # using int to avoid precision loss in float
+ return datetime.datetime.fromtimestamp(obj // 1000000).replace(
+ microsecond=obj % 1000000
+ )
+ elif isinstance(dataType, TimestampNTZType):
+ if obj is not None:
+ # using int to avoid precision loss in float
+ return datetime.datetime.fromtimestamp(obj // 1000000,
datetime.timezone.utc).replace(
+ microsecond=obj % 1000000, tzinfo=None
+ )
+ elif isinstance(dataType, DayTimeIntervalType):
+ if obj is not None:
+ return datetime.timedelta(microseconds=obj)
+ elif isinstance(dataType, (YearMonthIntervalType, CalendarIntervalType)):
+ raise PySparkNotImplementedError(
+ errorClass="NOT_IMPLEMENTED",
+ messageParameters={"feature": "%s.fromInternal" %
type(dataType).__name__},
+ )
+ elif isinstance(dataType, VariantType):
+ if obj is None or not all(key in obj for key in ["value", "metadata"]):
+ return None
+ return VariantVal(obj["value"], obj["metadata"])
+ elif isinstance(dataType, (GeometryType, GeographyType)):
+ if obj is None or not all(key in obj for key in ["srid", "bytes"]):
+ return None
+ cls = Geometry if isinstance(dataType, GeometryType) else Geography
+ return cls(obj["bytes"], obj["srid"])
+ elif isinstance(dataType, ArrayType):
+ if not dataType.needConversion():
+ return obj
+ return obj and [dataType.elementType.fromInternal(v) for v in obj]
+ elif isinstance(dataType, MapType):
+ if not dataType.needConversion():
+ return obj
+ return obj and dict(
+ (dataType.keyType.fromInternal(k),
dataType.valueType.fromInternal(v))
+ for k, v in obj.items()
+ )
+ elif isinstance(dataType, StructType):
+ if obj is None:
+ return
+ if isinstance(obj, Row):
+ # it's already converted by pickler
+ return obj
+ if dataType._needSerializeAnyField:
+ # Only calling fromInternal function for fields that need
conversion
+ values = [
+ f.fromInternal(v) if c else v
+ for f, v, c in zip(dataType.fields, obj,
dataType._needConversion)
+ ]
+ else:
+ values = obj
+ return _create_row(dataType.names, values)
+ else:
+ return obj
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index befa3fd..a5c0d8a 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -1302,6 +1302,26 @@ def
test_collect_reads_a_result_across_stream_ends(session):
assert session.range(n).toArrow().num_rows == n
+def
test_internal_values_of_collected_rows_are_those_of_spark_as_pyspark(session):
+ # As PySpark's: the toInternal of a collected row is the internal value of
Spark that the
+ # server computes, the days and the microseconds since the epoch, and its
fromInternal is
+ # the row.
+ df = session.sql(
+ "select date'2020-01-01' as d, timestamp'2020-01-01 12:30:15.123456'
as ts, "
+ "timestamp_ntz'2020-01-01 12:30:15.123456' as ntz, "
+ "interval '1 02:03:04.000005' day to second as i,
array(date'2020-01-02') as a, "
+ "map('k', date'2020-01-03') as m")
+ row = df.first()
+ days, micros, next_days, last_days = session.sql(
+ "select unix_date(date'2020-01-01'), unix_micros(timestamp'2020-01-01
12:30:15.123456'), "
+ "unix_date(date'2020-01-02'), unix_date(date'2020-01-03')").first()
+ internal = df.schema.toInternal(row)
+ assert internal == (days, micros, micros, 93784000005, [next_days], {"k":
last_days})
+ assert df.schema.fromInternal(internal) == row
+
+
+
+
def test_time_values_round_trip_as_pyspark(session):
# As PySpark's: TIME values are sent, and collected, as datetime.time,
when the session
# enables the TIME type.
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index cd41251..13d6de9 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -280,6 +280,75 @@ def _eq_cases():
]
+def
test_data_types_convert_values_to_and_from_internal_as_pyspark(monkeypatch):
+ # As PySpark's toInternal and fromInternal: the days of a date since the
epoch, the
+ # microseconds of a timestamp and of a day-time interval, the nanoseconds
of a time, the
+ # dicts of variants and spatial values, and the elements, keys, values and
fields of
+ # arrays, maps and structs; and needConversion of the types that convert.
+ import datetime
+
+ from pyspark.errors import PySparkNotImplementedError, PySparkValueError
+ from pyspark.sql.types import Geometry, VariantVal
+ from pyspark.testing.objects import PythonOnlyUDT
+
+ d, dt = datetime.date(2020, 1, 1), datetime.datetime(2020, 1, 1, 12, 30,
15, 123456)
+ assert T.DateType().toInternal(d) == 18262 and
T.DateType().fromInternal(18262) == d
+ assert T.DateType().toInternal(None) is None
+ utc = dt.replace(tzinfo=datetime.timezone.utc)
+ assert T.TimestampType().toInternal(utc) == 1577881815123456
+ assert T.TimestampType().fromInternal(1577881815123456) == (
+
datetime.datetime.fromtimestamp(1577881815).replace(microsecond=123456))
+ assert T.TimestampNTZType().toInternal(dt) == 1577881815123456
+ assert T.TimestampNTZType().fromInternal(1577881815123456) == dt
+ time = datetime.time(12, 30, 15, 123456)
+ assert T.TimeType().toInternal(time) == 45015123456000
+ assert T.TimeType().fromInternal(45015123456000) == time
+ delta = datetime.timedelta(days=1, seconds=5, microseconds=7)
+ assert T.DayTimeIntervalType().toInternal(delta) == 86405000007
+ assert T.DayTimeIntervalType().fromInternal(86405000007) == delta
+ for data_type in (T.YearMonthIntervalType(), T.CalendarIntervalType()):
+ for convert in (data_type.toInternal, data_type.fromInternal):
+ with pytest.raises(PySparkNotImplementedError) as e:
+ convert(1)
+ assert e.value.getCondition() == "NOT_IMPLEMENTED"
+ variant = VariantVal(b"\x0c\x01", b"\x01\x00\x00")
+ internal = {"value": b"\x0c\x01", "metadata": b"\x01\x00\x00"}
+ assert T.VariantType().toInternal(variant) == internal
+ assert T.VariantType().fromInternal(internal).value == b"\x0c\x01"
+ wkb = bytes.fromhex("0101000000000000000000f03f0000000000000040")
+ assert T.GeometryType(4326).toInternal(Geometry.fromWKB(wkb, 4326)) == {
+ "srid": 4326, "wkb": wkb}
+ point = T.GeometryType(4326).fromInternal({"srid": 4326, "bytes": wkb})
+ assert (point.srid, point.wkb) == (4326, wkb)
+
+ st = T.StructType([
+ T.StructField("a", T.IntegerType()),
+ T.StructField("d", T.DateType()),
+ T.StructField("l", T.ArrayType(T.DateType())),
+ T.StructField("m", T.MapType(T.StringType(), T.DateType())),
+ ])
+ internal = (1, 18262, [18262, None], {"k": 18262})
+ row = Row(a=1, d=d, l=[d, None], m={"k": d})
+ for value in ((1, d, [d, None], {"k": d}), {"a": 1, "d": d, "l": [d,
None], "m": {"k": d}},
+ row):
+ assert st.toInternal(value) == internal
+ assert st.fromInternal(internal) == row
+ assert (st._needSerializeAnyField, st._needConversion) == (True, [False,
True, True, True])
+ assert st.fields[1].toInternal(d) == 18262
+ with pytest.raises(PySparkValueError) as e:
+ st.toInternal(5)
+ assert e.value.getCondition() == "UNEXPECTED_TUPLE_WITH_STRUCT"
+
+ for data_type in (T.VariantType(), T.GeometryType(4326),
T.GeographyType(4326),
+ T.ArrayType(PythonOnlyUDT()), T.YearMonthIntervalType()):
+ assert data_type.needConversion() is True
+ assert T.ArrayType(T.IntegerType()).needConversion() is False
+ monkeypatch.setenv("PYSPARK_YM_INTERVAL_LEGACY", "1")
+ assert T.ArrayType(T.YearMonthIntervalType()).needConversion() is False
+
+
+
+
def test_datatype_eq_and_hash_match_pyspark():
cases = _eq_cases()
left = [f() for f in cases]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]