This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new c09d5fd  [SPARK-60922][CONNECT] Construct GeometryType and 
GeographyType as PySpark does
c09d5fd is described below

commit c09d5fde295c4b87b80487026c459677d4be7722
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Thu Nov 5 22:09:01 2026 +0900

    [SPARK-60922][CONNECT] Construct GeometryType and GeographyType as PySpark 
does
    
    ### What changes were proposed in this pull request?
    
    [SPARK-60922](https://issues.apache.org/jira/browse/SPARK-60922): 
`GeometryType(srid)` and `GeographyType(srid)` in the Python client now 
construct as PySpark's do.
    
    - `"ANY"` gives the type of a mixed SRID, which prints as 
`GeometryType(ANY)` or `GeographyType(ANY)`.
    - An integer must be an SRID that PySpark's spatial reference system 
mappers know: 0, 3857 and 4326 for a geometry, and 4326 for a geography in 
PySpark 4.3.0. Anything else, including a string other than `"ANY"`, raises 
`IllegalArgumentException` (`ST_INVALID_SRID_VALUE`, with the `srid`).
    - The SRID is required, as in PySpark.
    
    Types that come from the server, as in `df.schema`, are built without this 
validation, so a schema with any SRID still materializes.
    
    ### Why are the changes needed?
    
    The constructors took any integer and no string, with a default of 0: 
`GeometryType("ANY")` raised `TypeError`, `GeometryType(2000)` and 
`GeographyType(3857)` were accepted, and `GeometryType()` gave 
`GeometryType(0)`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, the spatial type constructors now behave as in PySpark Connect 4.3.0.
    
    ### How was this patch tested?
    
    - A probe of fourteen constructions, with `"ANY"`, the known SRIDs, other 
SRIDs, strings, a float and no argument, gives output identical to official 
PySpark 4.3.0, for the repr, `srid` and `simpleString()`, or the error and its 
parameters.
    - New offline tests check the accepted SRIDs, the rejected ones with 
`ST_INVALID_SRID_VALUE`, and that the SRID is required. They also pass with 
official PySpark 4.3.0.
    - The offline Python suites (527 passed, 8 live cases skipped), and the 
live drop-in and Arrow suites (51 passed) and live cases (10 passed). Clippy 
reports no diagnostics in the changed lines.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #160 from HyukjinKwon/SPARK-60922-spatial-ctor.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/types.rs      | 62 +++++++++++++++++++++++++++++++------
 python/tests/test_dropin_offline.py | 38 +++++++++++++++++++++++
 2 files changed, 90 insertions(+), 10 deletions(-)

diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index 1fb7f1a..17f4586 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -2431,22 +2431,27 @@ impl PyGeometryType {
     const MIXED_CRS: &'static str = "SRID:ANY";
     #[classattr]
     const MIXED_SRID: i32 = -1;
+    /// As PySpark's `GeometryType(srid)`: `"ANY"` for a mixed SRID, or an 
SRID of a
+    /// coordinate reference system that PySpark knows.
     #[new]
-    #[pyo3(signature = (srid=0))]
-    fn new(srid: i32) -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
+    fn new(srid: &Bound<'_, PyAny>) -> 
PyResult<pyo3::PyClassInitializer<Self>> {
+        let srid = spatial_type_srid(srid, false)?;
+        Ok(init_chain!(
             DataType::Geometry { srid },
             PyGeometryType { srid },
             [PyAtomicType, PySpatialType],
             value
-        )
+        ))
     }
     #[getter]
     fn srid(&self) -> i32 {
         self.srid
     }
     fn __repr__(&self) -> String {
-        format!("GeometryType({})", self.srid)
+        match self.srid {
+            -1 => "GeometryType(ANY)".to_string(),
+            srid => format!("GeometryType({srid})"),
+        }
     }
     #[pyo3(name = "simpleString")]
     fn simple_string(&self) -> String {
@@ -2475,22 +2480,27 @@ impl PyGeographyType {
     const MIXED_CRS: &'static str = "SRID:ANY";
     #[classattr]
     const MIXED_SRID: i32 = -1;
+    /// As PySpark's `GeographyType(srid)`: `"ANY"` for a mixed SRID, or an 
SRID of a
+    /// geographic coordinate reference system that PySpark knows.
     #[new]
-    #[pyo3(signature = (srid=0))]
-    fn new(srid: i32) -> pyo3::PyClassInitializer<Self> {
-        init_chain!(
+    fn new(srid: &Bound<'_, PyAny>) -> 
PyResult<pyo3::PyClassInitializer<Self>> {
+        let srid = spatial_type_srid(srid, true)?;
+        Ok(init_chain!(
             DataType::Geography { srid },
             PyGeographyType { srid },
             [PyAtomicType, PySpatialType],
             value
-        )
+        ))
     }
     #[getter]
     fn srid(&self) -> i32 {
         self.srid
     }
     fn __repr__(&self) -> String {
-        format!("GeographyType({})", self.srid)
+        match self.srid {
+            -1 => "GeographyType(ANY)".to_string(),
+            srid => format!("GeographyType({srid})"),
+        }
     }
     #[pyo3(name = "simpleString")]
     fn simple_string(&self) -> String {
@@ -2502,6 +2512,38 @@ impl PyGeographyType {
     }
 }
 
+/// The SRID of a spatial type that PySpark constructs from `srid`: `"ANY"` 
gives the mixed
+/// SRID, and an integer must be an SRID that PySpark's spatial reference 
system mappers know,
+/// 0, 3857 and 4326 for a geometry and 4326 for a geography in PySpark 4.3.0. 
Anything else
+/// raises `ST_INVALID_SRID_VALUE`, as in PySpark.
+fn spatial_type_srid(srid: &Bound<'_, PyAny>, geographic: bool) -> 
PyResult<i32> {
+    use pyo3::types::{PyInt, PyString};
+    if let Ok(name) = srid.cast::<PyString>() {
+        if name.to_cow()? == "ANY" {
+            return Ok(spark_connect::types::SpatialType::MIXED_SRID);
+        }
+    }
+    let known: &[i64] = if geographic {
+        &[4326]
+    } else {
+        &[0, 3857, 4326]
+    };
+    if srid.is_instance_of::<PyInt>() {
+        if let Ok(value) = srid.extract::<i64>() {
+            if known.contains(&value) {
+                return Ok(value as i32);
+            }
+        }
+    }
+    let srid_str = srid.str()?.to_string();
+    Err(crate::column::raise_pyspark(
+        srid.py(),
+        "IllegalArgumentException",
+        "ST_INVALID_SRID_VALUE",
+        &[("srid", &srid_str)],
+    ))
+}
+
 /// Convert a Python DataType object (any of the type classes) or a DDL string 
into
 /// the core `DataType`. Mirrors pyspark accepting `Union[DataType, str]` 
everywhere.
 /// Materialize a core [`DataType`] into its concrete Python type object — a 
real
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 65e1365..381ee9d 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -512,6 +512,44 @@ def test_type_class_hierarchy():
     assert issubclass(T.AtomicType, T.DataType)
 
 
[email protected]("make, expected", [
+    (lambda: T.GeometryType("ANY"), "GeometryType(ANY)"),
+    (lambda: T.GeographyType("ANY"), "GeographyType(ANY)"),
+    (lambda: T.GeometryType(0), "GeometryType(0)"),
+    (lambda: T.GeometryType(3857), "GeometryType(3857)"),
+    (lambda: T.GeometryType(4326), "GeometryType(4326)"),
+    (lambda: T.GeographyType(4326), "GeographyType(4326)"),
+])
+def test_spatial_types_take_the_srids_that_pyspark_knows(make, expected):
+    spatial = make()
+    assert repr(spatial) == expected
+    # "ANY" is the mixed SRID.
+    assert (spatial.srid == -1) is ("ANY" in expected)
+
+
[email protected]("make, srid", [
+    (lambda: T.GeometryType(2000), "2000"),
+    (lambda: T.GeometryType(-1), "-1"),
+    (lambda: T.GeometryType("any"), "any"),
+    (lambda: T.GeometryType("4326"), "4326"),
+    (lambda: T.GeographyType(3857), "3857"),
+    (lambda: T.GeographyType(0), "0"),
+])
+def test_spatial_types_reject_other_srids_as_pyspark(make, srid):
+    from pyspark.errors import IllegalArgumentException
+
+    with pytest.raises(IllegalArgumentException) as error:
+        make()
+    assert error.value.getCondition() == "ST_INVALID_SRID_VALUE"
+    assert error.value.getMessageParameters() == {"srid": srid}
+
+
+def test_spatial_types_need_an_srid():
+    for spatial in [T.GeometryType, T.GeographyType]:
+        with pytest.raises(TypeError):
+            spatial()
+
+
 def test_spatial_types():
     g = T.GeometryType(4326)
     assert [c.__name__ for c in type(g).__mro__ if c.__name__ != "object"] == [


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to