This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new ee4b75f [SPARK-60946][CONNECT][PYTHON] Send user-defined, VARIANT,
GEOMETRY and GEOGRAPHY values in createDataFrame, and collect spatial values,
as PySpark does
ee4b75f is described below
commit ee4b75fd768cf1171ed9d3c42727378aa84c1fb9
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 00:41:28 2026 +0900
[SPARK-60946][CONNECT][PYTHON] Send user-defined, VARIANT, GEOMETRY and
GEOGRAPHY values in createDataFrame, and collect spatial values, as PySpark does
### What changes were proposed in this pull request?
[SPARK-60946](https://issues.apache.org/jira/browse/SPARK-60946):
`createDataFrame` now sends user-defined, VARIANT, GEOMETRY and GEOGRAPHY
values, and the Python client collects spatial values, as PySpark Connect does.
As PySpark's `LocalDataToArrowConversion` and `to_arrow_type`,
`createDataFrame` sends:
- a `VariantVal` as the struct of its binary value and metadata, whose
metadata field is tagged as a variant;
- a `Geometry` or a `Geography` as the struct of its srid and WKB, whose
WKB field is tagged as of its type and srid;
- a value of a user-defined type serialized by the UDT, as its SQL type.
A value of another type for a VARIANT, GEOMETRY or GEOGRAPHY raises
`MALFORMED_VARIANT`, `MALFORMED_GEOMETRY` or `MALFORMED_GEOGRAPHY`.
As PySpark's `ArrowTableToRowsConversion`, a collected GEOMETRY or
GEOGRAPHY is the `Geometry` or `Geography` of its WKB and srid. `Geometry` and
`Geography` also get the `srid` and `wkb` attributes of PySpark's.
### Why are the changes needed?
Each of these values raised `TypeError: unsupported value type for
createDataFrame`. In PySpark 4.3.0:
- `createDataFrame([(ExamplePoint(1.0, 2.0),)])` gives `struct<_1:udt>` and
the row `Row(_1=ExamplePoint(1.0,2.0))`.
- A `VariantVal` round-trips as a VARIANT.
- A `Geometry.fromWKB(wkb, 4326)` gives `geometry(4326)`.
A collected GEOMETRY, such as of `ST_GeomFromWKB`, was a `Row(srid=...,
wkb=...)` rather than a `Geometry`.
### Does this PR introduce _any_ user-facing change?
Yes. These values are sent and collected as in PySpark 4.3.0.
### How was this patch tested?
- A new live test round-trips a `VariantVal`, a `Geometry`, a `Geography`
and an `ExamplePoint` through `createDataFrame`, and collects a GEOMETRY of
SQL. It also passes with official PySpark 4.3.0, and fails without this change.
- A new offline test checks the Arrow data and the schema that
`createDataFrame` sends to a mock Connect server for these values, the errors
of malformed ones, and the attributes. Another checks the conversions of
collected spatial values. They fail without this change.
- A new lib test checks the Arrow encoding of these values.
- The lib tests and every integration test, e2e included, against a local
Spark 4.3.0 server (665 passed, and 125 of the transport crate), the build of
all tests with `wasm-udf`, the offline Python suites (1330 passed, 8 skipped),
and the live drop-in and Arrow suites (88 passed) and live cases (10 passed).
Clippy reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #215 from HyukjinKwon/SPARK-60946-cdf-values.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/dataframe.rs | 2 +-
crates/pyspark-rs/src/infer.rs | 119 ++++++++++++++++++++++
crates/pyspark-rs/src/types.rs | 19 ++++
crates/pyspark-rs/src/values.rs | 32 ++++--
crates/spark-connect/src/session.rs | 191 ++++++++++++++++++++++++++++++++++++
python/pyspark/sql/types.py | 14 ++-
python/tests/test_dropin_e2e.py | 25 +++++
python/tests/test_dropin_offline.py | 72 ++++++++++++++
8 files changed, 465 insertions(+), 9 deletions(-)
diff --git a/crates/pyspark-rs/src/dataframe.rs
b/crates/pyspark-rs/src/dataframe.rs
index de0dfc6..7bff99a 100644
--- a/crates/pyspark-rs/src/dataframe.rs
+++ b/crates/pyspark-rs/src/dataframe.rs
@@ -2790,7 +2790,7 @@ fn udt_field_converters(
py: Python<'_>,
schema: Option<&spark_connect::types::DataType>,
) -> PyResult<Option<FieldConverters>> {
- let Some(schema) = schema.filter(|schema| crate::types::has_udt(schema))
else {
+ let Some(schema) = schema.filter(|schema|
crate::types::has_converted_values(schema)) else {
return Ok(None);
};
let schema = crate::types::data_type_to_py(py, schema)?;
diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index 7b3d0fe..5ae84ca 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -1067,6 +1067,63 @@ pub(crate) fn convert(obj: &Bound<'_, PyAny>, dt:
&DataType, nullable: bool) ->
)))
}
}
+ // As PySpark's converters of these types: the value and metadata of a
VariantVal, and
+ // the srid and WKB of a Geometry or a Geography, the structs of their
Arrow types.
+ DataType::Variant => match obj.cast::<PyVariantVal>() {
+ Ok(variant) => {
+ let variant = variant.borrow();
+ Ok(Value::Variant {
+ value: variant.value.clone(),
+ metadata: variant.metadata.clone(),
+ })
+ }
+ Err(_) => Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "MALFORMED_VARIANT",
+ &[],
+ )),
+ },
+ DataType::Geometry { .. } => match obj.cast::<PyGeometry>() {
+ Ok(geometry) => {
+ let geometry = geometry.borrow();
+ Ok(spatial_value(geometry.srid, &geometry.wkb))
+ }
+ Err(_) => Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "MALFORMED_GEOMETRY",
+ &[],
+ )),
+ },
+ DataType::Geography { .. } => match obj.cast::<PyGeography>() {
+ Ok(geography) => {
+ let geography = geography.borrow();
+ Ok(spatial_value(geography.srid, &geography.wkb))
+ }
+ Err(_) => Err(raise_pyspark(
+ py,
+ "PySparkValueError",
+ "MALFORMED_GEOGRAPHY",
+ &[],
+ )),
+ },
+ // As PySpark's converter of a user-defined type: the value serialized
by the UDT, and
+ // converted as its SQL type.
+ DataType::Udt {
+ sql_type: Some(sql_type),
+ python_class,
+ serialized_python_class,
+ ..
+ } => {
+ let udt = udt_of(
+ obj,
+ python_class.as_deref(),
+ serialized_python_class.as_deref(),
+ )?;
+ let serialized = udt.call_method1(intern!(py, "serialize"),
(obj,))?;
+ convert(&serialized, sql_type, nullable)
+ }
_ => Err(unsupported(obj)?),
}
}
@@ -1081,6 +1138,68 @@ pub(crate) fn time_nanos(obj: &Bound<'_, PyAny>) ->
PyResult<i64> {
Ok(seconds * 1_000_000_000 + field(intern!(py, "microsecond"))? * 1_000)
}
+/// The value of a GEOMETRY or a GEOGRAPHY, as their `toInternal`: its srid
and WKB.
+fn spatial_value(srid: i32, wkb: &[u8]) -> Value {
+ Value::Struct(
+ spark_connect::row::Row::new(
+ vec!["srid".to_string(), "wkb".to_string()],
+ vec![Value::Integer(srid), Value::Binary(wkb.to_vec())],
+ )
+ .into(),
+ )
+}
+
+/// The user-defined type of a column whose class is `python_class`, to
serialize `obj`: that of
+/// the value, as PySpark infers a UDT by `__UDT__`, when it is of the class;
else an instance
+/// of the class, which the type sends cloudpickled for a class of `__main__`.
+fn udt_of<'py>(
+ obj: &Bound<'py, PyAny>,
+ python_class: Option<&str>,
+ serialized_python_class: Option<&str>,
+) -> PyResult<Bound<'py, PyAny>> {
+ let py = obj.py();
+ let name_of = |class: &Bound<'py, PyAny>| -> PyResult<String> {
+ let module: String = class.getattr(intern!(py,
"__module__"))?.extract()?;
+ let name: String = class.getattr(intern!(py, "__name__"))?.extract()?;
+ Ok(format!("{module}.{name}"))
+ };
+ if let Ok(udt) = obj.getattr(intern!(py, "__UDT__")) {
+ let class = udt.get_type();
+ let module_name: String = udt.call_method0(intern!(py,
"module"))?.extract()?;
+ let name: String = class.getattr(intern!(py, "__name__"))?.extract()?;
+ let matches = python_class.is_none_or(|python_class| {
+ python_class == format!("{module_name}.{name}")
+ || name_of(class.as_any()).is_ok_and(|n| n == python_class)
+ });
+ if matches {
+ return Ok(udt);
+ }
+ }
+ let class = match (serialized_python_class, python_class) {
+ (Some(serialized), _) => {
+ let pickled = py
+ .import(intern!(py, "base64"))?
+ .call_method1(intern!(py, "b64decode"), (serialized,))?;
+ py.import(intern!(py, "pyspark.serializers"))?
+ .getattr(intern!(py, "CloudPickleSerializer"))?
+ .call0()?
+ .call_method1(intern!(py, "loads"), (pickled,))?
+ }
+ (None, Some(python_class)) => {
+ let (module, name) = python_class.rsplit_once('.').ok_or_else(|| {
+ PyValueError::new_err(format!("invalid UDT class
{python_class}"))
+ })?;
+ py.import(module)?.getattr(name)?
+ }
+ (None, None) => {
+ return Err(PyValueError::new_err(
+ "a user-defined type without its Python class cannot serialize
values",
+ ))
+ }
+ };
+ class.call0()
+}
+
/// A value of a day-time interval column, as PyArrow converts one to
`duration[us]`: a
/// timedelta to its microseconds, as `DayTimeIntervalType.toInternal`, and an
int, or a
/// float truncated, as microseconds. Another value raises PyArrow's error.
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 78125cf..7aa22f2 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -2632,6 +2632,25 @@ pub(crate) fn has_udt(data_type: &DataType) -> bool {
}
}
+/// Whether the collected values of `data_type` need the converters of
+/// `pyspark.sql.types._udt_field_converters`: of a user-defined type, which
deserializes them,
+/// or of a GEOMETRY or a GEOGRAPHY, which makes their Python objects.
+pub(crate) fn has_converted_values(data_type: &DataType) -> bool {
+ match data_type {
+ DataType::Udt { .. } | DataType::Geometry { .. } | DataType::Geography
{ .. } => true,
+ DataType::Array { element_type, .. } =>
has_converted_values(element_type),
+ DataType::Map {
+ key_type,
+ value_type,
+ ..
+ } => has_converted_values(key_type) ||
has_converted_values(value_type),
+ DataType::Struct { fields } => fields
+ .iter()
+ .any(|field| has_converted_values(&field.data_type)),
+ _ => false,
+ }
+}
+
/// The `simpleString()` of `data_type` as PySpark's: of the Python class of
each of its
/// user-defined types, such as `vector` of a `VectorUDT`, which the Rust core
does not know.
pub(crate) fn simple_string_with_udts(py: Python<'_>, data_type: &DataType) ->
PyResult<String> {
diff --git a/crates/pyspark-rs/src/values.rs b/crates/pyspark-rs/src/values.rs
index e3bcce0..53d0fb7 100644
--- a/crates/pyspark-rs/src/values.rs
+++ b/crates/pyspark-rs/src/values.rs
@@ -8,8 +8,8 @@ use pyo3::types::PyBytes;
/// `pyspark.sql.types.VariantVal`.
#[pyclass(name = "VariantVal", module = "pyspark.sql.types")]
pub struct PyVariantVal {
- value: Vec<u8>,
- metadata: Vec<u8>,
+ pub(crate) value: Vec<u8>,
+ pub(crate) metadata: Vec<u8>,
}
impl PyVariantVal {
@@ -88,8 +88,8 @@ impl PyVariantVal {
/// A Geography value (WKB bytes + SRID). Mirrors
`pyspark.sql.types.Geography`.
#[pyclass(name = "Geography", module = "pyspark.sql.types")]
pub struct PyGeography {
- wkb: Vec<u8>,
- srid: i32,
+ pub(crate) wkb: Vec<u8>,
+ pub(crate) srid: i32,
}
#[pymethods]
impl PyGeography {
@@ -103,6 +103,16 @@ impl PyGeography {
fn getBytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
PyBytes::new(py, &self.wkb)
}
+ /// The spatial reference identifier, as PySpark's attribute.
+ #[getter]
+ fn srid(&self) -> i32 {
+ self.srid
+ }
+ /// The well-known binary, as PySpark's attribute.
+ #[getter]
+ fn wkb<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
+ PyBytes::new(py, &self.wkb)
+ }
#[classmethod]
fn fromWKB(_cls: &Bound<'_, pyo3::types::PyType>, wkb: Vec<u8>, srid: i32)
-> PyGeography {
PyGeography { wkb, srid }
@@ -122,8 +132,8 @@ impl PyGeography {
/// A Geometry value (WKB bytes + SRID). Mirrors `pyspark.sql.types.Geometry`.
#[pyclass(name = "Geometry", module = "pyspark.sql.types")]
pub struct PyGeometry {
- wkb: Vec<u8>,
- srid: i32,
+ pub(crate) wkb: Vec<u8>,
+ pub(crate) srid: i32,
}
#[pymethods]
impl PyGeometry {
@@ -137,6 +147,16 @@ impl PyGeometry {
fn getBytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
PyBytes::new(py, &self.wkb)
}
+ /// The spatial reference identifier, as PySpark's attribute.
+ #[getter]
+ fn srid(&self) -> i32 {
+ self.srid
+ }
+ /// The well-known binary, as PySpark's attribute.
+ #[getter]
+ fn wkb<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
+ PyBytes::new(py, &self.wkb)
+ }
#[classmethod]
fn fromWKB(_cls: &Bound<'_, pyo3::types::PyType>, wkb: Vec<u8>, srid: i32)
-> PyGeometry {
PyGeometry { wkb, srid }
diff --git a/crates/spark-connect/src/session.rs
b/crates/spark-connect/src/session.rs
index 3bee47d..d31f33a 100644
--- a/crates/spark-connect/src/session.rs
+++ b/crates/spark-connect/src/session.rs
@@ -944,6 +944,7 @@ fn schema_to_arrow_fields(schema: &DataType) ->
Result<Vec<arrow::datatypes::Fie
/// Map a Spark `DataType` to its Arrow `DataType`, recursively
(array/map/struct included).
fn datatype_to_arrow(dt: &DataType) -> Result<arrow::datatypes::DataType> {
use arrow::datatypes::{DataType as A, Field, Fields, TimeUnit};
+ use std::collections::HashMap;
Ok(match dt {
DataType::Null => A::Null,
DataType::Boolean => A::Boolean,
@@ -1010,6 +1011,39 @@ fn datatype_to_arrow(dt: &DataType) ->
Result<arrow::datatypes::DataType> {
.into();
A::Struct(f)
}
+ // As PySpark's `to_arrow_type`: the struct of the binary value and
metadata of a
+ // VARIANT, and of the srid and WKB of a GEOMETRY or a GEOGRAPHY,
whose metadata tags
+ // the struct as one of these types.
+ DataType::Variant => A::Struct(
+ vec![
+ Field::new("value", A::Binary, false),
+ Field::new("metadata", A::Binary, false)
+ .with_metadata(HashMap::from([("variant".to_string(),
"true".to_string())])),
+ ]
+ .into(),
+ ),
+ DataType::Geometry { srid } | DataType::Geography { srid } => {
+ let kind = if matches!(dt, DataType::Geometry { .. }) {
+ "geometry"
+ } else {
+ "geography"
+ };
+ A::Struct(
+ vec![
+ Field::new("srid", A::Int32, false),
+ Field::new("wkb", A::Binary,
false).with_metadata(HashMap::from([
+ (kind.to_string(), "true".to_string()),
+ ("srid".to_string(), srid.to_string()),
+ ])),
+ ]
+ .into(),
+ )
+ }
+ // As PySpark's, a user-defined type is sent as its SQL type.
+ DataType::Udt {
+ sql_type: Some(sql_type),
+ ..
+ } => datatype_to_arrow(sql_type)?,
other => {
return Err(SparkError::connect_msg(format!(
"Unsupported type for Arrow createDataFrame conversion:
{other:?}"
@@ -1204,6 +1238,79 @@ fn values_to_arrow(values: &[&Value], dt: &DataType) ->
Result<Arc<dyn arrow::ar
Some(NullBuffer::new(valid.finish())),
)))
}
+ // The struct of a VARIANT's value and metadata, and of a GEOMETRY's
or a GEOGRAPHY's
+ // srid and WKB, as `datatype_to_arrow` declares; their fields are not
nullable, so a
+ // null value has empty ones.
+ DataType::Variant | DataType::Geometry { .. } | DataType::Geography {
.. } => {
+ let mut first = Vec::with_capacity(values.len());
+ let mut second: Vec<&[u8]> = Vec::with_capacity(values.len());
+ let mut valid =
arrow::array::builder::BooleanBufferBuilder::new(values.len());
+ for v in values {
+ match (*v, dt) {
+ (Value::Null, _) => {
+ valid.append(false);
+ first.push(None);
+ second.push(&[]);
+ }
+ (Value::Variant { value, metadata }, DataType::Variant) =>
{
+ valid.append(true);
+ first.push(Some(value.as_slice()));
+ second.push(metadata.as_slice());
+ }
+ (Value::Struct(sf), DataType::Geometry { .. } |
DataType::Geography { .. }) => {
+ let (Some(Value::Integer(_)),
Some(Value::Binary(wkb))) =
+ (sf.get_by_name("srid"), sf.get_by_name("wkb"))
+ else {
+ return Err(SparkError::connect_msg("Type mismatch
in row data"));
+ };
+ valid.append(true);
+ first.push(None);
+ second.push(wkb.as_slice());
+ }
+ _ => return Err(SparkError::connect_msg("Type mismatch in
row data")),
+ }
+ }
+ let arrow::datatypes::DataType::Struct(afields) =
datatype_to_arrow(dt)? else {
+ unreachable!("these types are structs")
+ };
+ let nulls = Some(NullBuffer::new(valid.finish()));
+ let arrays: Vec<Arc<dyn Array>> = match dt {
+ DataType::Variant => vec![
+ Arc::new(BinaryArray::from(
+ first
+ .iter()
+ .map(|v| v.unwrap_or(&[]))
+ .collect::<Vec<&[u8]>>(),
+ )),
+ Arc::new(BinaryArray::from(second)),
+ ],
+ _ => {
+ let srids: Vec<i32> = values
+ .iter()
+ .map(|v| match *v {
+ Value::Struct(sf) => match sf.get_by_name("srid") {
+ Some(Value::Integer(srid)) => *srid,
+ _ => 0,
+ },
+ _ => 0,
+ })
+ .collect();
+ vec![
+ Arc::new(Int32Array::from(srids)),
+ Arc::new(BinaryArray::from(second)),
+ ]
+ }
+ };
+ Ok(Arc::new(
+ StructArray::try_new(afields, arrays, nulls).map_err(|e| {
+ SparkError::connect_msg(format!("Arrow struct build error:
{e}"))
+ })?,
+ ))
+ }
+ DataType::Udt {
+ sql_type: Some(sql_type),
+ ..
+ } => values_to_arrow(values, sql_type),
DataType::Struct { fields } => {
let mut cols: Vec<Vec<&Value>> =
vec![Vec::with_capacity(values.len()); fields.len()];
let mut valid =
arrow::array::builder::BooleanBufferBuilder::new(values.len());
@@ -2231,6 +2338,90 @@ mod tests {
);
}
+ #[test]
+ fn input_ipc_sends_variants_spatial_and_udt_values_as_pyspark() {
+ // As PySpark's `to_arrow_type`: the structs of a VARIANT's value and
metadata, and of a
+ // GEOMETRY's srid and WKB, which their metadata tags, and a UDT as
its SQL type.
+ use arrow::array::{Array, BinaryArray, Int32Array, ListArray,
StructArray};
+ let udt = DataType::Udt {
+ type_str: "udt".to_string(),
+ jvm_class: None,
+ python_class:
Some("pyspark.testing.objects.ExamplePointUDT".to_string()),
+ serialized_python_class: None,
+ sql_type: Some(Box::new(DataType::Array {
+ element_type: Box::new(DataType::Double),
+ contains_null: false,
+ })),
+ };
+ let schema = schema(vec![
+ DataType::Variant,
+ DataType::Geometry { srid: 4326 },
+ udt,
+ ]);
+ let geometry = Value::Struct(
+ Row::new(
+ vec!["srid".to_string(), "wkb".to_string()],
+ vec![Value::Integer(4326), Value::Binary(vec![1, 2])],
+ )
+ .into(),
+ );
+ let rows = vec![
+ row(vec![
+ Value::Variant {
+ value: vec![12, 1],
+ metadata: vec![1, 0, 0],
+ },
+ geometry,
+ Value::List(vec![Value::Double(1.0), Value::Double(2.0)]),
+ ]),
+ row(vec![Value::Null, Value::Null, Value::Null]),
+ ];
+ let batch = read_batch(&rows, &schema);
+ let variant = batch
+ .column(0)
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .unwrap();
+ let fields = variant.fields();
+ assert_eq!(
+ fields[1].metadata().get("variant").map(String::as_str),
+ Some("true")
+ );
+ let metadata = variant
+ .column(1)
+ .as_any()
+ .downcast_ref::<BinaryArray>()
+ .unwrap();
+ assert_eq!(metadata.value(0), [1, 0, 0]);
+ assert!(variant.is_null(1));
+ let geometry = batch
+ .column(1)
+ .as_any()
+ .downcast_ref::<StructArray>()
+ .unwrap();
+ let wkb_field = &geometry.fields()[1];
+ assert_eq!(
+ wkb_field.metadata().get("geometry").map(String::as_str),
+ Some("true")
+ );
+ assert_eq!(
+ wkb_field.metadata().get("srid").map(String::as_str),
+ Some("4326")
+ );
+ let srid = geometry
+ .column(0)
+ .as_any()
+ .downcast_ref::<Int32Array>()
+ .unwrap();
+ assert_eq!(srid.value(0), 4326);
+ assert!(geometry.is_null(1));
+ assert!(batch
+ .column(2)
+ .as_any()
+ .downcast_ref::<ListArray>()
+ .is_some());
+ }
+
#[test]
fn input_ipc_scales_nanos_timestamps_and_keeps_zone() {
use arrow::array::{ListArray, TimestampNanosecondArray};
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 71b1c3f..fa8cace 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -405,8 +405,18 @@ def _create_row(fields, values):
def _udt_converter(dataType):
"""The function that turns a collected value of ``dataType`` into what
PySpark gives for it,
- deserializing the values of its user-defined types as PySpark's
- ``ArrowTableToRowsConversion`` does, or None for a type without a
user-defined type."""
+ deserializing the values of its user-defined types, and making the
Geometry and Geography
+ of its spatial values, as PySpark's ``ArrowTableToRowsConversion`` does,
or None for a type
+ without either."""
+ if isinstance(dataType, (GeometryType, GeographyType)):
+ spatial = Geometry if isinstance(dataType, GeometryType) else Geography
+
+ def convert_spatial(value):
+ if value is None:
+ return None
+ return spatial.fromWKB(value["wkb"], value["srid"])
+
+ return convert_spatial
if isinstance(dataType, UserDefinedType):
inner = _udt_converter(dataType.sqlType())
diff --git a/python/tests/test_dropin_e2e.py b/python/tests/test_dropin_e2e.py
index b19fc9c..2847def 100644
--- a/python/tests/test_dropin_e2e.py
+++ b/python/tests/test_dropin_e2e.py
@@ -792,6 +792,31 @@ def
test_apply_in_pandas_with_state_runs_as_pyspark(session, tmp_path):
session.conf.unset("spark.sql.shuffle.partitions")
+def
test_variant_spatial_and_user_defined_values_round_trip_as_pyspark(session):
+ # As PySpark's: createDataFrame sends a VariantVal, a Geometry, a
Geography and a value of
+ # a user-defined type, and collects them as they are.
+ from pyspark.sql.types import Geography, Geometry
+ from pyspark.testing.objects import ExamplePoint
+
+ wkb = bytes.fromhex("0101000000000000000000F03F0000000000000040")
+ variant = session.sql("""SELECT parse_json('{"a": 1}') AS v""").first()[0]
+ df = session.createDataFrame([
+ (variant, Geometry.fromWKB(wkb, 4326), Geography.fromWKB(wkb, 4326),
ExamplePoint(1.0, 2.0))
+ ])
+ assert df.schema.simpleString() == (
+ "struct<_1:variant,_2:geometry(4326),_3:geography(4326),_4:udt>"
+ )
+ row = df.first()
+ assert row[0].toJson() == '{"a":1}'
+ assert row[1] == Geometry.fromWKB(wkb, 4326) and row[2] ==
Geography.fromWKB(wkb, 4326)
+ assert row[3] == ExamplePoint(1.0, 2.0)
+ # A collected GEOMETRY is a Geometry.
+ geometry = session.sql(
+ "SELECT ST_GeomFromWKB(X'0101000000000000000000F03F0000000000000040')
AS g"
+ ).first()[0]
+ assert geometry == Geometry.fromWKB(wkb, 0)
+
+
def test_create_dataframe_types_pandas_columns_as_pyspark(session):
from decimal import Decimal
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 42da6f8..d3ae644 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -1626,6 +1626,78 @@ def test_udtf_analyze_result_family():
assert issubclass(SkipRestOfInputTableException, Exception)
+def
test_variant_spatial_and_user_defined_values_are_sent_as_pyspark(mock_connect):
+ # As PySpark's LocalDataToArrowConversion: a VariantVal is the struct of
its value and
+ # metadata, a Geometry or a Geography that of its srid and WKB, whose
metadata tags the
+ # struct as of its type, and a value of a user-defined type is serialized
by the UDT and
+ # sent as its SQL type; a value of another type for these is malformed.
+ import pyarrow as pa
+ from pyspark.errors import PySparkValueError
+ from pyspark.testing.objects import ExamplePoint
+
+ spark, service = mock_connect
+ wkb = bytes.fromhex("0101000000000000000000F03F0000000000000040")
+ variant = T.VariantVal(b"\x0c\x01", b"\x01\x00\x00")
+
+ def local_relation(df):
+ df.toArrow()
+ request = parse_message(service.execute_requests[-1])
+ relation = parse_message(parse_message(request[3][0])[1][0])
+ while 18 in relation: # Relation.to_df -> ToDF.input.
+ relation = parse_message(parse_message(relation[18][0])[1][0])
+ # Relation.local_relation=11 -> LocalRelation.data=1, schema=2.
+ local = parse_message(relation[11][0])
+ table = pa.ipc.open_stream(local[1][0]).read_all()
+ return table, [field["type"] for field in
json.loads(local[2][0])["fields"]]
+
+ table, types = local_relation(spark.createDataFrame([
+ (variant, T.Geometry.fromWKB(wkb, 4326), T.Geography.fromWKB(wkb,
4326),
+ ExamplePoint(1.0, 2.0)),
+ (None, None, None, None),
+ ]))
+ assert types[:3] == [
+ T.VariantType().jsonValue(), T.GeometryType(4326).jsonValue(),
+ T.GeographyType(4326).jsonValue(),
+ ]
+ assert types[3]["type"] == "udt" and types[3]["sqlType"]["type"] == "array"
+ variant_type, geometry_type, geography_type, udt_type = table.schema.types
+ assert variant_type.field("metadata").metadata == {b"variant": b"true"}
+ assert geometry_type.field("wkb").metadata == {b"geometry": b"true",
b"srid": b"4326"}
+ assert geography_type.field("wkb").metadata == {b"geography": b"true",
b"srid": b"4326"}
+ assert geometry_type.field("srid").type == pa.int32()
+ assert pa.types.is_list(udt_type) and udt_type.value_type == pa.float64()
+ assert table.to_pylist() == [
+ {"_1": {"value": b"\x0c\x01", "metadata": b"\x01\x00\x00"},
+ "_2": {"srid": 4326, "wkb": wkb}, "_3": {"srid": 4326, "wkb": wkb},
"_4": [1.0, 2.0]},
+ {"_1": None, "_2": None, "_3": None, "_4": None},
+ ]
+ for ddl, condition in (("v variant", "MALFORMED_VARIANT"),
+ ("g geometry(4326)", "MALFORMED_GEOMETRY"),
+ ("g geography(4326)", "MALFORMED_GEOGRAPHY")):
+ with pytest.raises(PySparkValueError) as error:
+ spark.createDataFrame([("x",)], ddl)
+ assert error.value.getCondition() == condition
+ # The attributes of PySpark's spatial values.
+ geometry = T.Geometry.fromWKB(wkb, 4326)
+ assert (geometry.srid, geometry.wkb) == (4326, wkb)
+
+
+def test_collected_spatial_values_are_their_objects_as_pyspark():
+ # As PySpark's ArrowTableToRowsConversion, a collected GEOMETRY or
GEOGRAPHY is the
+ # Geometry or Geography of its WKB and srid.
+ from pyspark.sql.types import _udt_field_converters
+
+ wkb = b"\x01\x01"
+ schema = T.StructType([
+ T.StructField("g", T.GeometryType(4326)),
+ T.StructField("gs", T.ArrayType(T.GeographyType(4326))),
+ ])
+ g, gs = _udt_field_converters(schema)
+ assert g(Row(srid=4326, wkb=wkb)) == T.Geometry.fromWKB(wkb, 4326)
+ assert gs([Row(srid=4326, wkb=wkb), None]) == [T.Geography.fromWKB(wkb,
4326), None]
+ assert g(None) is None
+
+
def test_variant_and_geo_values():
from pyspark.sql.types import VariantVal, Geometry, Geography
v = VariantVal.parseJson('{"a": 1, "b": [2, 3]}')
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]