This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark-connect-rust.git
The following commit(s) were added to refs/heads/master by this push:
new 2aa7443 [SPARK-61068][CONNECT][PYTHON] Raise ST_INVALID_SRID_VALUE
for the spatial values of SRIDs that their type does not take in
createDataFrame as PySpark does
2aa7443 is described below
commit 2aa74436b65d6dae86a5d6d934fd35d01c18d0ba
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Sat Nov 7 12:28:06 2026 +0900
[SPARK-61068][CONNECT][PYTHON] Raise ST_INVALID_SRID_VALUE for the spatial
values of SRIDs that their type does not take in createDataFrame as PySpark does
### What changes were proposed in this pull request?
The inference of a `Geometry` or `Geography` value in `createDataFrame`
validates its SRID with the validator that the type constructors already use
(`spatial_type_srid`, now `pub(crate)`), as PySpark's `_infer_type` does
(`crates/pyspark-rs/src/infer.rs`): Geometry takes 0, 3857 and 4326, Geography
only 4326, and any other SRID raises `IllegalArgumentException`
`ST_INVALID_SRID_VALUE` (`srid`) on the client, at the row of the value,
top-level or nested. Explicit schemas are unchanged [...]
### Why are the changes needed?
[SPARK-61068](https://issues.apache.org/jira/browse/SPARK-61068): the raw
SRID reached the inferred type. Against a local Spark 4.3.0 server:
| Input | Current master | This change and PySpark 4.3.0 |
| --- | --- | --- |
| `[(Geography.fromWKB(wkb, 4269),)]` (also Geometry 4269, 99999) | server
`ParseException [PARSE_SYNTAX_ERROR]` | `IllegalArgumentException
[ST_INVALID_SRID_VALUE]` (4269) |
| Geography 4326 and 4269 in two rows | silently `geography(any)` |
`ST_INVALID_SRID_VALUE` (4269) |
| Geography 4326 and 0 | the server's error, after sending | the client's
`ST_INVALID_SRID_VALUE` (0) |
| Geometry 0, 3857 and 4326 | `geometry(any)` | `geometry(any)` (unchanged)
|
### Does this PR introduce _any_ user-facing change?
Yes, invalid SRIDs raise PySpark's error.
### How was this patch tested?
- New offline tests in `python/tests/test_dropin_offline.py`: each invalid
SRID of each type, top-level, in an array and after a null row; mixed Geography
SRIDs; the inferred types of valid SRIDs, from the local relation's schema; and
an explicit `GeographyType("ANY")` still taking any SRID.
- The offline suite (967 passed, 8 skipped) and the live spatial test; a
probe of these cases against both clients gives the same output.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #258 from HyukjinKwon/SPARK-61068-spatial-srid.
Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
crates/pyspark-rs/src/infer.rs | 9 +++-
crates/pyspark-rs/src/types.rs | 2 +-
python/tests/test_dropin_offline.py | 82 +++++++++++++++++++++++++++++++++++++
3 files changed, 90 insertions(+), 3 deletions(-)
diff --git a/crates/pyspark-rs/src/infer.rs b/crates/pyspark-rs/src/infer.rs
index 3ed7d6d..319c4ce 100644
--- a/crates/pyspark-rs/src/infer.rs
+++ b/crates/pyspark-rs/src/infer.rs
@@ -334,12 +334,17 @@ pub(crate) fn infer_type(obj: &Bound<'_, PyAny>, o:
Options<'_>) -> PyResult<Dat
end_field: 3,
});
}
+ // As PySpark's `_infer_type`, the type of a value is `GeometryType(srid)`
or
+ // `GeographyType(srid)`, which raises `ST_INVALID_SRID_VALUE` for an SRID
that the type does
+ // not take (the explicit schema is not checked here, the server checks
the values of it).
if obj.is_exact_instance_of::<PyGeometry>() {
- let srid = obj.call_method0(intern!(py, "getSrid"))?.extract()?;
+ let srid = obj.call_method0(intern!(py, "getSrid"))?;
+ let srid = crate::types::spatial_type_srid(&srid, false)?;
return Ok(DataType::Geometry { srid });
}
if obj.is_exact_instance_of::<PyGeography>() {
- let srid = obj.call_method0(intern!(py, "getSrid"))?.extract()?;
+ let srid = obj.call_method0(intern!(py, "getSrid"))?;
+ let srid = crate::types::spatial_type_srid(&srid, true)?;
return Ok(DataType::Geography { srid });
}
if obj.hasattr(intern!(py, "__UDT__"))? {
diff --git a/crates/pyspark-rs/src/types.rs b/crates/pyspark-rs/src/types.rs
index f5b8fc7..6e45a9f 100644
--- a/crates/pyspark-rs/src/types.rs
+++ b/crates/pyspark-rs/src/types.rs
@@ -2575,7 +2575,7 @@ impl PyGeographyType {
/// SRID, and an integer must be an SRID that PySpark's spatial reference
system mappers know,
/// 0, 3857 and 4326 for a geometry and 4326 for a geography in PySpark 4.3.0.
Anything else
/// raises `ST_INVALID_SRID_VALUE`, as in PySpark.
-fn spatial_type_srid(srid: &Bound<'_, PyAny>, geographic: bool) ->
PyResult<i32> {
+pub(crate) fn spatial_type_srid(srid: &Bound<'_, PyAny>, geographic: bool) ->
PyResult<i32> {
use pyo3::types::{PyInt, PyString};
if let Ok(name) = srid.cast::<PyString>() {
if name.to_cow()? == "ANY" {
diff --git a/python/tests/test_dropin_offline.py
b/python/tests/test_dropin_offline.py
index 3d0f7aa..f35818c 100644
--- a/python/tests/test_dropin_offline.py
+++ b/python/tests/test_dropin_offline.py
@@ -10007,6 +10007,88 @@ def
test_exact_string_subclass_uses_generic_conversion():
"data_type1": "StructType", "data_type2": data_type}
+# ---------------------------------------------------------------------------
spatial values with invalid or mixed SRIDs
+_SPATIAL_WKB = bytes.fromhex("0101000000000000000000F03F0000000000000040")
+
+
+def _sent_local_field_types(spark, service, data, schema=None):
+ # The JSON types of the fields of the local relation that createDataFrame
sends.
+ df = spark.createDataFrame(data, schema) if schema is not None else
spark.createDataFrame(data)
+ df.toArrow()
+ request = parse_message(service.execute_requests[-1])
+ relation = parse_message(parse_message(request[3][0])[1][0])
+ while 18 in relation: # Relation.to_df -> ToDF.input.
+ relation = parse_message(parse_message(relation[18][0])[1][0])
+ local = parse_message(relation[11][0]) # Relation.local_relation=11 ->
LocalRelation.schema=2.
+ return [field["type"] for field in json.loads(local[2][0])["fields"]]
+
+
[email protected]("make, srid", [
+ (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 4269), 4269),
+ (lambda: T.Geometry.fromWKB(_SPATIAL_WKB, 4269), 4269),
+ (lambda: T.Geometry.fromWKB(_SPATIAL_WKB, 99999), 99999),
+ (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 99999), 99999),
+ (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 0), 0),
+ (lambda: T.Geography.fromWKB(_SPATIAL_WKB, 3857), 3857),
+ (lambda: T.Geography.fromWKB(_SPATIAL_WKB, -1), -1),
+])
+def
test_inferred_spatial_value_with_an_srid_its_type_does_not_take_is_rejected(mock_connect,
make, srid):
+ # As PySpark's _infer_type, which builds GeometryType(srid) or
GeographyType(srid) for each
+ # value: createDataFrame raises ST_INVALID_SRID_VALUE for a top-level
value, for one nested
+ # in an array, and for the value after a null.
+ from pyspark.errors import IllegalArgumentException
+
+ spark, _ = mock_connect
+ for data in ([(make(),)], [([make()],)], [(None,), (make(),)]):
+ with pytest.raises(IllegalArgumentException) as error:
+ spark.createDataFrame(data)
+ assert error.value.getCondition() == "ST_INVALID_SRID_VALUE"
+ assert error.value.getMessageParameters() == {"srid": str(srid)}
+
+
+def
test_mixed_geography_srids_fail_at_the_row_with_the_invalid_srid(mock_connect):
+ # Official PySpark infers each row's type before merging the rows, so the
second row of
+ # GEOGRAPHY SRIDs 4326 and 4269 raises the error with its SRID, as does 0
after 4326.
+ from pyspark.errors import IllegalArgumentException
+
+ spark, _ = mock_connect
+ for second in (4269, 0):
+ with pytest.raises(IllegalArgumentException) as error:
+ spark.createDataFrame([
+ (T.Geography.fromWKB(_SPATIAL_WKB, 4326),),
+ (T.Geography.fromWKB(_SPATIAL_WKB, second),),
+ ])
+ assert error.value.getCondition() == "ST_INVALID_SRID_VALUE"
+ assert error.value.getMessageParameters() == {"srid": str(second)}
+
+
+def test_inferred_spatial_types_of_valid_srids_are_as_pyspark(mock_connect):
+ # GEOMETRY takes the SRIDs 0, 3857 and 4326, and merges differing SRIDs
into GEOMETRY(ANY)
+ # as PySpark's _merge_type does; GEOGRAPHY takes only 4326.
+ from pyspark.sql.types import Geometry
+
+ spark, service = mock_connect
+ geometry = lambda srid: Geometry.fromWKB(_SPATIAL_WKB, srid) # noqa: E731
+ types = lambda data: _sent_local_field_types(spark, service, data) #
noqa: E731
+ assert types([(geometry(0),), (geometry(3857),), (geometry(4326),)]) == [
+ T.GeometryType("ANY").jsonValue()]
+ assert types([(geometry(4326),), (None,), (geometry(4326),)]) == [
+ T.GeometryType(4326).jsonValue()]
+ assert types([([geometry(4326)],), ([geometry(3857)],)]) == [
+ T.ArrayType(T.GeometryType("ANY")).jsonValue()]
+ assert types([(T.Geography.fromWKB(_SPATIAL_WKB, 4326),)]) == [
+ T.GeographyType(4326).jsonValue()]
+
+
+def
test_explicit_schema_spatial_values_are_not_inferred_from_their_srids(mock_connect):
+ # With an explicit schema, PySpark does not build the type from each
value's SRID, so a
+ # GEOGRAPHY(ANY) takes a value of SRID 4269 without the client checking it.
+ spark, service = mock_connect
+ schema = T.StructType([T.StructField("g", T.GeographyType("ANY"))])
+ data = [(T.Geography.fromWKB(_SPATIAL_WKB, 4269),),
(T.Geography.fromWKB(_SPATIAL_WKB, 4326),)]
+ assert _sent_local_field_types(spark, service, data, schema) ==
[T.GeographyType("ANY").jsonValue()]
+
+
# ---------------------------------------------------------------------------
map functions
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]