This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new c09d5fd [SPARK-60922][CONNECT] Construct GeometryType and
GeographyType as PySpark does
c09d5fd is described below
commit c09d5fde295c4b87b80487026c459677d4be7722
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Thu Nov 5 22:09:01 2026 +0900
[SPARK-60922][CONNECT] Construct GeometryType and GeographyType as PySpark
does
### What changes were proposed in this pull request?
[SPARK-60922](https://issues.apache.org/jira/browse/SPARK-60922):
`GeometryType(srid)` and `GeographyType(srid)` in the Python client now
construct as PySpark's do.
- `"ANY"` gives the type of a mixed SRID, which prints as
`GeometryType(ANY)` or `GeographyType(ANY)`.
- An integer must be an SRID that PySpark's spatial reference system
mappers know: 0, 3857 and 4326 for a geometry, and 4326 for a geography in
PySpark 4.3.0. Anything else, including a string other than `"ANY"`, raises
`IllegalArgumentException` (`ST_INVALID_SRID_VALUE`, with the `srid`).
- The SRID is required, as in PySpark.
Types that come from the server, as in `df.schema`, are built without this
validation, so a schema with any SRID still materializes.
### Why are the changes needed?
The constructors took any integer and no string, with a default of 0:
`GeometryType("ANY")` raised `TypeError`, `GeometryType(2000)` and
`GeographyType(3857)` were accepted, and `GeometryType()` gave
`GeometryType(0)`.
### Does this PR introduce _any_ user-facing change?
Yes, the spatial type constructors now behave as in PySpark Connect 4.3.0.
### How was this patch tested?
- A probe of fourteen constructions, with `"ANY"`, the known SRIDs, other
SRIDs, strings, a float and no argument, gives output identical to official
PySpark 4.3.0, for the repr, `srid` and `simpleString()`, or the error and its
parameters.
- New offline tests check the accepted SRIDs, the rejected ones with
`ST_INVALID_SRID_VALUE`, and that the SRID is required. They also pass with
official PySpark 4.3.0.
- The offline Python suites (527 passed, 8 live cases skipped), and the
live drop-in and Arrow suites (51 passed) and live cases (10 passed). Clippy
reports no diagnostics in the changed lines.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #160 from HyukjinKwon/SPARK-60922-spatial-ctor.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/types.rs | 62 +++++++++++++++++++++++++++++++------
python/tests/test_dropin_offline.py | 38 +++++++++++++++++++++++
2 files changed, 90 insertions(+), 10 deletions(-)
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 1fb7f1a..17f4586 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -2431,22 +2431,27 @@ impl PyGeometryType {
const MIXED_CRS: &'static str = "SRID:ANY";
#[classattr]
const MIXED_SRID: i32 = -1;
+ /// As PySpark's `GeometryType(srid)`: `"ANY"` for a mixed SRID, or an
SRID of a
+ /// coordinate reference system that PySpark knows.
#[new]
- #[pyo3(signature = (srid=0))]
- fn new(srid: i32) -> pyo3::PyClassInitializer<Self> {
- init_chain!(
+ fn new(srid: &Bound<'_, PyAny>) ->
PyResult<pyo3::PyClassInitializer<Self>> {
+ let srid = spatial_type_srid(srid, false)?;
+ Ok(init_chain!(
DataType::Geometry { srid },
PyGeometryType { srid },
[PyAtomicType, PySpatialType],
value
- )
+ ))
}
#[getter]
fn srid(&self) -> i32 {
self.srid
}
fn __repr__(&self) -> String {
- format!("GeometryType({})", self.srid)
+ match self.srid {
+ -1 => "GeometryType(ANY)".to_string(),
+ srid => format!("GeometryType({srid})"),
+ }
}
#[pyo3(name = "simpleString")]
fn simple_string(&self) -> String {
@@ -2475,22 +2480,27 @@ impl PyGeographyType {
const MIXED_CRS: &'static str = "SRID:ANY";
#[classattr]
const MIXED_SRID: i32 = -1;
+ /// As PySpark's `GeographyType(srid)`: `"ANY"` for a mixed SRID, or an
SRID of a
+ /// geographic coordinate reference system that PySpark knows.
#[new]
- #[pyo3(signature = (srid=0))]
- fn new(srid: i32) -> pyo3::PyClassInitializer<Self> {
- init_chain!(
+ fn new(srid: &Bound<'_, PyAny>) ->
PyResult<pyo3::PyClassInitializer<Self>> {
+ let srid = spatial_type_srid(srid, true)?;
+ Ok(init_chain!(
DataType::Geography { srid },
PyGeographyType { srid },
[PyAtomicType, PySpatialType],
value
- )
+ ))
}
#[getter]
fn srid(&self) -> i32 {
self.srid
}
fn __repr__(&self) -> String {
- format!("GeographyType({})", self.srid)
+ match self.srid {
+ -1 => "GeographyType(ANY)".to_string(),
+ srid => format!("GeographyType({srid})"),
+ }
}
#[pyo3(name = "simpleString")]
fn simple_string(&self) -> String {
@@ -2502,6 +2512,38 @@ impl PyGeographyType {
}
}
+/// The SRID of a spatial type that PySpark constructs from `srid`: `"ANY"`
gives the mixed
+/// SRID, and an integer must be an SRID that PySpark's spatial reference
system mappers know,
+/// 0, 3857 and 4326 for a geometry and 4326 for a geography in PySpark 4.3.0.
Anything else
+/// raises `ST_INVALID_SRID_VALUE`, as in PySpark.
+fn spatial_type_srid(srid: &Bound<'_, PyAny>, geographic: bool) ->
PyResult<i32> {
+ use pyo3::types::{PyInt, PyString};
+ if let Ok(name) = srid.cast::<PyString>() {
+ if name.to_cow()? == "ANY" {
+ return Ok(spark_connect::types::SpatialType::MIXED_SRID);
+ }
+ }
+ let known: &[i64] = if geographic {
+ &[4326]
+ } else {
+ &[0, 3857, 4326]
+ };
+ if srid.is_instance_of::<PyInt>() {
+ if let Ok(value) = srid.extract::<i64>() {
+ if known.contains(&value) {
+ return Ok(value as i32);
+ }
+ }
+ }
+ let srid_str = srid.str()?.to_string();
+ Err(crate::column::raise_pyspark(
+ srid.py(),
+ "IllegalArgumentException",
+ "ST_INVALID_SRID_VALUE",
+ &[("srid", &srid_str)],
+ ))
+}
+
/// Convert a Python DataType object (any of the type classes) or a DDL string
into
/// the core `DataType`. Mirrors pyspark accepting `Union[DataType, str]`
everywhere.
/// Materialize a core [`DataType`] into its concrete Python type object — a
real
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 65e1365..381ee9d 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -512,6 +512,44 @@ def test_type_class_hierarchy():
assert issubclass(T.AtomicType, T.DataType)
[email protected]("make, expected", [
+ (lambda: T.GeometryType("ANY"), "GeometryType(ANY)"),
+ (lambda: T.GeographyType("ANY"), "GeographyType(ANY)"),
+ (lambda: T.GeometryType(0), "GeometryType(0)"),
+ (lambda: T.GeometryType(3857), "GeometryType(3857)"),
+ (lambda: T.GeometryType(4326), "GeometryType(4326)"),
+ (lambda: T.GeographyType(4326), "GeographyType(4326)"),
+])
+def test_spatial_types_take_the_srids_that_pyspark_knows(make, expected):
+ spatial = make()
+ assert repr(spatial) == expected
+ # "ANY" is the mixed SRID.
+ assert (spatial.srid == -1) is ("ANY" in expected)
+
+
[email protected]("make, srid", [
+ (lambda: T.GeometryType(2000), "2000"),
+ (lambda: T.GeometryType(-1), "-1"),
+ (lambda: T.GeometryType("any"), "any"),
+ (lambda: T.GeometryType("4326"), "4326"),
+ (lambda: T.GeographyType(3857), "3857"),
+ (lambda: T.GeographyType(0), "0"),
+])
+def test_spatial_types_reject_other_srids_as_pyspark(make, srid):
+ from pyspark.errors import IllegalArgumentException
+
+ with pytest.raises(IllegalArgumentException) as error:
+ make()
+ assert error.value.getCondition() == "ST_INVALID_SRID_VALUE"
+ assert error.value.getMessageParameters() == {"srid": srid}
+
+
+def test_spatial_types_need_an_srid():
+ for spatial in [T.GeometryType, T.GeographyType]:
+ with pytest.raises(TypeError):
+ spatial()
+
+
def test_spatial_types():
g = T.GeometryType(4326)
assert [c.__name__ for c in type(g).__mro__ if c.__name__ != "object"] == [
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]