This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git


The following commit(s) were added to refs/heads/master by this push:
     new 2aa7443  [SPARK-61068][CONNECT][PYTHON] Raise ST_INVALID_SRID_VALUE 
for the spatial values of SRIDs that their type does not take in 
createDataFrame as PySpark does
2aa7443 is described below

commit 2aa74436b65d6dae86a5d6d934fd35d01c18d0ba
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:28:06 2026 +0900

    [SPARK-61068][CONNECT][PYTHON] Raise ST_INVALID_SRID_VALUE for the spatial 
values of SRIDs that their type does not take in createDataFrame as PySpark does
    
    ### What changes were proposed in this pull request?
    
    The inference of a `Geometry` or `Geography` value in `createDataFrame` 
validates its SRID with the validator that the type constructors already use 
(`spatial_type_srid`, now `pub(crate)`), as PySpark's `_infer_type` does 
(`crates/pyspark-rs/src/infer.rs`): Geometry takes 0, 3857 and 4326, Geography 
only 4326, and any other SRID raises `IllegalArgumentException` 
`ST_INVALID_SRID_VALUE` (`srid`) on the client, at the row of the value, 
top-level or nested. Explicit schemas are unchanged [...]
    
    ### Why are the changes needed?
    
    [SPARK-61068](https://issues.apache.org/jira/browse/SPARK-61068): the raw 
SRID reached the inferred type. Against a local Spark 4.3.0 server:
    
    | Input | Current master | This change and PySpark 4.3.0 |
    | --- | --- | --- |
    | `[(Geography.fromWKB(wkb, 4269),)]` (also Geometry 4269, 99999) | server 
`ParseException [PARSE_SYNTAX_ERROR]` | `IllegalArgumentException 
[ST_INVALID_SRID_VALUE]` (4269) |
    | Geography 4326 and 4269 in two rows | silently `geography(any)` | 
`ST_INVALID_SRID_VALUE` (4269) |
    | Geography 4326 and 0 | the server's error, after sending | the client's 
`ST_INVALID_SRID_VALUE` (0) |
    | Geometry 0, 3857 and 4326 | `geometry(any)` | `geometry(any)` (unchanged) 
|
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, invalid SRIDs raise PySpark's error.
    
    ### How was this patch tested?
    
    - New offline tests in `python/tests/test_dropin_offline.py`: each invalid 
SRID of each type, top-level, in an array and after a null row; mixed Geography 
SRIDs; the inferred types of valid SRIDs, from the local relation's schema; and 
an explicit `GeographyType("ANY")` still taking any SRID.
    - The offline suite (967 passed, 8 skipped) and the live spatial test; a 
probe of these cases against both clients gives the same output.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5.5
    
    This pull request and its description were written by Isaac.
    
    Closes #258 from HyukjinKwon/SPARK-61068-spatial-srid.
    
    Authored-by: Hyukjin Kwon <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 crates/pyspark-rs/src/infer.rs      |  9 +++-
 crates/pyspark-rs/src/types.rs      |  2 +-
 python/tests/test_dropin_offline.py | 82 +++++++++++++++++++++++++++++++++++++
 3 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index 3ed7d6d..319c4ce 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -334,12 +334,17 @@ pub(crate) fn infer_type(obj: &Bound<'_, PyAny>, o: 
Options<'_>) -> PyResult<Dat
             end_field: 3,
         });
     }
+    // As PySpark's `_infer_type`, the type of a value is `GeometryType(srid)` 
or
+    // `GeographyType(srid)`, which raises `ST_INVALID_SRID_VALUE` for an SRID 
that the type does
+    // not take (the explicit schema is not checked here, the server checks 
the values of it).
     if obj.is_exact_instance_of::<PyGeometry>() {
-        let srid = obj.call_method0(intern!(py, "getSrid"))?.extract()?;
+        let srid = obj.call_method0(intern!(py, "getSrid"))?;
+        let srid = crate::types::spatial_type_srid(&srid, false)?;
         return Ok(DataType::Geometry { srid });
     }
     if obj.is_exact_instance_of::<PyGeography>() {
-        let srid = obj.call_method0(intern!(py, "getSrid"))?.extract()?;
+        let srid = obj.call_method0(intern!(py, "getSrid"))?;
+        let srid = crate::types::spatial_type_srid(&srid, true)?;
         return Ok(DataType::Geography { srid });
     }
     if obj.hasattr(intern!(py, "__UDT__"))? {
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index f5b8fc7..6e45a9f 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -2575,7 +2575,7 @@ impl PyGeographyType {
 /// SRID, and an integer must be an SRID that PySpark's spatial reference 
system mappers know,
 /// 0, 3857 and 4326 for a geometry and 4326 for a geography in PySpark 4.3.0. 
Anything else
 /// raises `ST_INVALID_SRID_VALUE`, as in PySpark.
-fn spatial_type_srid(srid: &Bound<'_, PyAny>, geographic: bool) -> 
PyResult<i32> {
+pub(crate) fn spatial_type_srid(srid: &Bound<'_, PyAny>, geographic: bool) -> 
PyResult<i32> {
     use pyo3::types::{PyInt, PyString};
     if let Ok(name) = srid.cast::<PyString>() {
         if name.to_cow()? == "ANY" {
diff --git a/python/tests/test_dropin_offline.py 
b/python/tests/test_dropin_offline.py
index 3d0f7aa..f35818c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -10007,6 +10007,88 @@ def 
test_exact_string_subclass_uses_generic_conversion():
             "data_type1": "StructType", "data_type2": data_type}
 
 
+# --------------------------------------------------------------------------- 
spatial values with invalid or mixed SRIDs
+_SPATIAL_WKB = bytes.fromhex("0101000000000000000000F03F0000000000000040")
+
+
+def _sent_local_field_types(spark, service, data, schema=None):
+    # The JSON types of the fields of the local relation that createDataFrame 
sends.
+    df = spark.createDataFrame(data, schema) if schema is not None else 
spark.createDataFrame(data)
+    df.toArrow()
+    request = parse_message(service.execute_requests[-1])
+    relation = parse_message(parse_message(request[3][0])[1][0])
+    while 18 in relation:  # Relation.to_df -> ToDF.input.
+        relation = parse_message(parse_message(relation[18][0])[1][0])
+    local = parse_message(relation[11][0])  # Relation.local_relation=11 -> 
LocalRelation.schema=2.
+    return [field["type"] for field in json.loads(local[2][0])["fields"]]
+
+
[email protected]("make, srid", [
+    (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 4269), 4269),
+    (lambda: T.Geometry.fromWKB(_SPATIAL_WKB, 4269), 4269),
+    (lambda: T.Geometry.fromWKB(_SPATIAL_WKB, 99999), 99999),
+    (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 99999), 99999),
+    (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 0), 0),
+    (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 3857), 3857),
+    (lambda: T.Geography.fromWKB(_SPATIAL_WKB, -1), -1),
+])
+def 
test_inferred_spatial_value_with_an_srid_its_type_does_not_take_is_rejected(mock_connect,
 make, srid):
+    # As PySpark's _infer_type, which builds GeometryType(srid) or 
GeographyType(srid) for each
+    # value: createDataFrame raises ST_INVALID_SRID_VALUE for a top-level 
value, for one nested
+    # in an array, and for the value after a null.
+    from pyspark.errors import IllegalArgumentException
+
+    spark, _ = mock_connect
+    for data in ([(make(),)], [([make()],)], [(None,), (make(),)]):
+        with pytest.raises(IllegalArgumentException) as error:
+            spark.createDataFrame(data)
+        assert error.value.getCondition() == "ST_INVALID_SRID_VALUE"
+        assert error.value.getMessageParameters() == {"srid": str(srid)}
+
+
+def 
test_mixed_geography_srids_fail_at_the_row_with_the_invalid_srid(mock_connect):
+    # Official PySpark infers each row's type before merging the rows, so the 
second row of
+    # GEOGRAPHY SRIDs 4326 and 4269 raises the error with its SRID, as does 0 
after 4326.
+    from pyspark.errors import IllegalArgumentException
+
+    spark, _ = mock_connect
+    for second in (4269, 0):
+        with pytest.raises(IllegalArgumentException) as error:
+            spark.createDataFrame([
+                (T.Geography.fromWKB(_SPATIAL_WKB, 4326),),
+                (T.Geography.fromWKB(_SPATIAL_WKB, second),),
+            ])
+        assert error.value.getCondition() == "ST_INVALID_SRID_VALUE"
+        assert error.value.getMessageParameters() == {"srid": str(second)}
+
+
+def test_inferred_spatial_types_of_valid_srids_are_as_pyspark(mock_connect):
+    # GEOMETRY takes the SRIDs 0, 3857 and 4326, and merges differing SRIDs 
into GEOMETRY(ANY)
+    # as PySpark's _merge_type does; GEOGRAPHY takes only 4326.
+    from pyspark.sql.types import Geometry
+
+    spark, service = mock_connect
+    geometry = lambda srid: Geometry.fromWKB(_SPATIAL_WKB, srid)  # noqa: E731
+    types = lambda data: _sent_local_field_types(spark, service, data)  # 
noqa: E731
+    assert types([(geometry(0),), (geometry(3857),), (geometry(4326),)]) == [
+        T.GeometryType("ANY").jsonValue()]
+    assert types([(geometry(4326),), (None,), (geometry(4326),)]) == [
+        T.GeometryType(4326).jsonValue()]
+    assert types([([geometry(4326)],), ([geometry(3857)],)]) == [
+        T.ArrayType(T.GeometryType("ANY")).jsonValue()]
+    assert types([(T.Geography.fromWKB(_SPATIAL_WKB, 4326),)]) == [
+        T.GeographyType(4326).jsonValue()]
+
+
+def 
test_explicit_schema_spatial_values_are_not_inferred_from_their_srids(mock_connect):
+    # With an explicit schema, PySpark does not build the type from each 
value's SRID, so a
+    # GEOGRAPHY(ANY) takes a value of SRID 4269 without the client checking it.
+    spark, service = mock_connect
+    schema = T.StructType([T.StructField("g", T.GeographyType("ANY"))])
+    data = [(T.Geography.fromWKB(_SPATIAL_WKB, 4269),), 
(T.Geography.fromWKB(_SPATIAL_WKB, 4326),)]
+    assert _sent_local_field_types(spark, service, data, schema) == 
[T.GeographyType("ANY").jsonValue()]
+
+
 # --------------------------------------------------------------------------- 
map functions
 
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to