This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new fd0828852bfd [SPARK-54202][GEO][SQL] Allow casting from
GeometryType(srid) to GeometryType(ANY)
fd0828852bfd is described below
commit fd0828852bfdc7201259e7e823ff83b46180d6e1
Author: Uros Bojanic <[email protected]>
AuthorDate: Fri Nov 7 06:48:31 2025 -0800
[SPARK-54202][GEO][SQL] Allow casting from GeometryType(srid) to
GeometryType(ANY)
### What changes were proposed in this pull request?
This PR allows casting fixed SRID type `GEOMETRY(<srid>)` to mixed SRID
type `GEOMETRY(ANY)`.
### Why are the changes needed?
Enable explicit casting between geometry types.
### Does this PR introduce _any_ user-facing change?
Yes, casting `GEOMETRY(<srid>)` to `GEOMETRY(ANY)` is now allowed.
### How was this patch tested?
Added new unit tests:
- `StUtilsSuite`
- `CastSuiteBase`
Added new e2e SQL tests:
- `st-functions`
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52904 from uros-db/geo-cast-geom_any.
Authored-by: Uros Bojanic <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
(cherry picked from commit 7bc3e75c03e9001aac20f0ae1f00eee856b99c83)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/catalyst/expressions/Cast.scala | 11 ++++++
.../sql/catalyst/expressions/CastSuiteBase.scala | 23 ++++++++++++
.../analyzer-results/nonansi/st-functions.sql.out | 29 +++++++++++++++
.../analyzer-results/st-functions.sql.out | 29 +++++++++++++++
.../resources/sql-tests/inputs/st-functions.sql | 5 +++
.../sql-tests/results/nonansi/st-functions.sql.out | 32 +++++++++++++++++
.../sql-tests/results/st-functions.sql.out | 32 +++++++++++++++++
.../org/apache/spark/sql/STExpressionsSuite.scala | 41 ++++++++++++++++++++++
8 files changed, 202 insertions(+)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 22f1ad73a0e8..10f4c5c00f04 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -170,6 +170,9 @@ object Cast extends QueryErrorsBase {
// Casting from GEOGRAPHY to GEOMETRY with the same SRID is allowed.
case (geog: GeographyType, geom: GeometryType) if geog.srid == geom.srid =>
true
+ // Casts from concrete GEOMETRY(srid) to mixed GEOMETRY(ANY) is allowed.
+ case (gt1: GeometryType, gt2: GeometryType) if !gt1.isMixedSrid &&
gt2.isMixedSrid =>
+ true
case _ => false
}
@@ -303,6 +306,9 @@ object Cast extends QueryErrorsBase {
// Casting from GEOGRAPHY to GEOMETRY with the same SRID is allowed.
case (geog: GeographyType, geom: GeometryType) if geog.srid == geom.srid =>
true
+ // Casts from concrete GEOMETRY(srid) to mixed GEOMETRY(ANY) is allowed.
+ case (gt1: GeometryType, gt2: GeometryType) if !gt1.isMixedSrid &&
gt2.isMixedSrid =>
+ true
case _ => false
}
@@ -1157,6 +1163,8 @@ case class Cast(
private[this] def castToGeometry(from: DataType): Any => Any = from match {
case _: GeographyType =>
buildCast[GeographyVal](_, STUtils.geographyToGeometry)
+ case _: GeometryType =>
+ identity
}
private[this] def castArray(fromType: DataType, toType: DataType): Any =>
Any = {
@@ -2201,6 +2209,9 @@ case class Cast(
case _: GeographyType =>
(c, evPrim, _) =>
code"$evPrim =
org.apache.spark.sql.catalyst.util.STUtils.geographyToGeometry($c);"
+ case _: GeometryType =>
+ (c, evPrim, _) =>
+ code"$evPrim = $c;"
}
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index 2220f56255f0..e18a489d36f3 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -1544,6 +1544,29 @@ abstract class CastSuiteBase extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ test("Casting GeometryType to GeometryType") {
+ // Casting from fixed SRID GEOMETRY(<srid>) to mixed SRID GEOMETRY(ANY) is
always allowed.
+ // Type casting is always safe in this direction, so no additional
constraints are imposed.
+ // Casting from mixed SRID GEOMETRY(ANY) to fixed SRID GEOMETRY(<srid>) is
not allowed.
+ // Type casting can be unsafe in this direction, because per-row SRID
values may be different.
+
+ // Valid cast test cases.
+ val canCastTestCases: Seq[(DataType, DataType)] = Seq(
+ (GeometryType(0), GeometryType("ANY")),
+ (GeometryType(3857), GeometryType("ANY")),
+ (GeometryType(4326), GeometryType("ANY"))
+ )
+ // Iterate over the test cases and verify casting.
+ canCastTestCases.foreach { case (fromType, toType) =>
+ // Cast can be performed from `fromType` to `toType`.
+ assert(Cast.canCast(fromType, toType))
+ assert(Cast.canAnsiCast(fromType, toType))
+ // Cast cannot be performed from `toType` to `fromType`.
+ assert(!Cast.canCast(toType, fromType))
+ assert(!Cast.canAnsiCast(toType, fromType))
+ }
+ }
+
test("cast string to time") {
checkEvaluation(cast(Literal.create("0:0:0"), TimeType()), 0L)
checkEvaluation(cast(Literal.create(" 01:2:3.01 "), TimeType(2)),
localTime(1, 2, 3, 10000))
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/st-functions.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/st-functions.sql.out
index 1d2094f3b9ef..a564c6a32932 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/st-functions.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/st-functions.sql.out
@@ -124,6 +124,35 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+SELECT
hex(ST_AsBinary(CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')
AS GEOMETRY(ANY)))) AS result
+-- !query analysis
+Project
[hex(st_asbinary(cast(st_geomfromwkb(0x0101000000000000000000F03F0000000000000040)
as geometry(any)))) AS result#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326)) AS result
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "sqlExpr" :
"\"CAST(CAST(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040') AS
GEOMETRY(ANY)) AS GEOMETRY(4326))\"",
+ "srcType" : "\"GEOMETRY(ANY)\"",
+ "targetType" : "\"GEOMETRY(4326)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 107,
+ "fragment" :
"CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326))"
+ } ]
+}
+
+
-- !query
SELECT
hex(ST_AsBinary(ST_GeogFromWKB(X'0101000000000000000000f03f0000000000000040')))
AS result
-- !query analysis
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/st-functions.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/st-functions.sql.out
index 1d2094f3b9ef..a564c6a32932 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/st-functions.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/st-functions.sql.out
@@ -124,6 +124,35 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+SELECT
hex(ST_AsBinary(CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')
AS GEOMETRY(ANY)))) AS result
+-- !query analysis
+Project
[hex(st_asbinary(cast(st_geomfromwkb(0x0101000000000000000000F03F0000000000000040)
as geometry(any)))) AS result#x]
++- OneRowRelation
+
+
+-- !query
+SELECT
CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326)) AS result
+-- !query analysis
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "sqlExpr" :
"\"CAST(CAST(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040') AS
GEOMETRY(ANY)) AS GEOMETRY(4326))\"",
+ "srcType" : "\"GEOMETRY(ANY)\"",
+ "targetType" : "\"GEOMETRY(4326)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 107,
+ "fragment" :
"CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326))"
+ } ]
+}
+
+
-- !query
SELECT
hex(ST_AsBinary(ST_GeogFromWKB(X'0101000000000000000000f03f0000000000000040')))
AS result
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/st-functions.sql
b/sql/core/src/test/resources/sql-tests/inputs/st-functions.sql
index 7f4b77c7e0f9..ceda71398305 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/st-functions.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/st-functions.sql
@@ -23,6 +23,11 @@ SELECT
hex(ST_AsBinary(CAST(ST_GeogFromWKB(X'0101000000000000000000f03f000000000
-- Error handling: mismatched SRIDs.
SELECT CAST(ST_GeogFromWKB(X'0101000000000000000000f03f0000000000000040') AS
GEOMETRY(ANY)) AS result;
+-- Casting GEOMETRY(<srid>) to GEOMETRY(ANY) is allowed.
+SELECT
hex(ST_AsBinary(CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')
AS GEOMETRY(ANY)))) AS result;
+-- Casting GEOMETRY(ANY) to GEOMETRY(<srid>) is not allowed.
+SELECT
CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326)) AS result;
+
---- ST reader/writer expressions
-- WKB (Well-Known Binary) round-trip tests for GEOGRAPHY and GEOMETRY types.
diff --git
a/sql/core/src/test/resources/sql-tests/results/nonansi/st-functions.sql.out
b/sql/core/src/test/resources/sql-tests/results/nonansi/st-functions.sql.out
index 088c49cb030a..c01534c64e7c 100644
--- a/sql/core/src/test/resources/sql-tests/results/nonansi/st-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/nonansi/st-functions.sql.out
@@ -137,6 +137,38 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+SELECT
hex(ST_AsBinary(CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')
AS GEOMETRY(ANY)))) AS result
+-- !query schema
+struct<result:string>
+-- !query output
+0101000000000000000000F03F0000000000000040
+
+
+-- !query
+SELECT
CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326)) AS result
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "sqlExpr" :
"\"CAST(CAST(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040') AS
GEOMETRY(ANY)) AS GEOMETRY(4326))\"",
+ "srcType" : "\"GEOMETRY(ANY)\"",
+ "targetType" : "\"GEOMETRY(4326)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 107,
+ "fragment" :
"CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326))"
+ } ]
+}
+
+
-- !query
SELECT
hex(ST_AsBinary(ST_GeogFromWKB(X'0101000000000000000000f03f0000000000000040')))
AS result
-- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/st-functions.sql.out
b/sql/core/src/test/resources/sql-tests/results/st-functions.sql.out
index 088c49cb030a..c01534c64e7c 100644
--- a/sql/core/src/test/resources/sql-tests/results/st-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/st-functions.sql.out
@@ -137,6 +137,38 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
}
+-- !query
+SELECT
hex(ST_AsBinary(CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')
AS GEOMETRY(ANY)))) AS result
+-- !query schema
+struct<result:string>
+-- !query output
+0101000000000000000000F03F0000000000000040
+
+
+-- !query
+SELECT
CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326)) AS result
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.sql.catalyst.ExtendedAnalysisException
+{
+ "errorClass" : "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION",
+ "sqlState" : "42K09",
+ "messageParameters" : {
+ "sqlExpr" :
"\"CAST(CAST(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040') AS
GEOMETRY(ANY)) AS GEOMETRY(4326))\"",
+ "srcType" : "\"GEOMETRY(ANY)\"",
+ "targetType" : "\"GEOMETRY(4326)\""
+ },
+ "queryContext" : [ {
+ "objectType" : "",
+ "objectName" : "",
+ "startIndex" : 8,
+ "stopIndex" : 107,
+ "fragment" :
"CAST(ST_GeomFromWKB(X'0101000000000000000000f03f0000000000000040')::GEOMETRY(ANY)
AS GEOMETRY(4326))"
+ } ]
+}
+
+
-- !query
SELECT
hex(ST_AsBinary(ST_GeogFromWKB(X'0101000000000000000000f03f0000000000000040')))
AS result
-- !query schema
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/STExpressionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/STExpressionsSuite.scala
index 4c34a5c8f78f..79e7ecbdf4c3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/STExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/STExpressionsSuite.scala
@@ -34,6 +34,7 @@ class STExpressionsSuite
private final val mixedSridGeographyType: DataType = GeographyType("ANY")
private final val defaultGeometrySrid: Int =
ExpressionDefaults.DEFAULT_GEOMETRY_SRID
private final val defaultGeometryType: DataType =
GeometryType(defaultGeometrySrid)
+ private final val mixedSridGeometryType: DataType = GeometryType("ANY")
// Private helper method to assert the data type of a query result.
private def assertType(query: String, expectedDataType: DataType) = {
@@ -82,6 +83,46 @@ class STExpressionsSuite
}
}
+ test("Cast GEOMETRY(srid) to GEOMETRY(ANY)") {
+ // Test data: WKB representation of POINT(1 2).
+ val wkbString = "0101000000000000000000F03F0000000000000040"
+ val wkb = Hex.unhex(wkbString.getBytes())
+ val wkbLiteral = Literal.create(wkb, BinaryType)
+
+ // Construct the input GEOMETRY expression.
+ val geomExpr = ST_GeomFromWKB(wkbLiteral)
+ assert(geomExpr.dataType.sameType(defaultGeometryType))
+ checkEvaluation(ST_AsBinary(geomExpr), wkb)
+ // Cast the GEOMETRY with fixed SRID to GEOMETRY with mixed SRID.
+ val castExpr = Cast(geomExpr, mixedSridGeometryType)
+ assert(castExpr.dataType.sameType(mixedSridGeometryType))
+ checkEvaluation(ST_AsBinary(castExpr), wkb)
+
+ // Construct the input GEOMETRY SQL query, using WKB literal.
+ val geomQueryLit: String = s"ST_GeomFromWKB(X'$wkbString')"
+ assertType(s"SELECT $geomQueryLit", defaultGeometryType)
+ checkAnswer(sql(s"SELECT ST_AsBinary($geomQueryLit)"), Row(wkb))
+ // Cast the GEOMETRY with fixed SRID to GEOMETRY with mixed SRID.
+ val castQueryLit = s"$geomQueryLit::GEOMETRY(ANY)"
+ assertType(s"SELECT $castQueryLit", mixedSridGeometryType)
+ checkAnswer(sql(s"SELECT ST_AsBinary($castQueryLit)"), Row(wkb))
+
+ withTable("tbl") {
+ // Construct the test table with WKB.
+ sql(s"CREATE TABLE tbl (wkb BINARY)")
+ sql(s"INSERT INTO tbl VALUES (X'$wkbString')")
+
+ // Construct the input GEOMETRY SQL query, using WKB column.
+ val geomQueryCol: String = s"ST_GeomFromWKB(wkb)"
+ assertType(s"SELECT $geomQueryCol FROM tbl", defaultGeometryType)
+ checkAnswer(sql(s"SELECT ST_AsBinary($geomQueryCol) FROM tbl"), Row(wkb))
+ // Cast the GEOMETRY with fixed SRID to GEOMETRY with mixed SRID.
+ val castQueryCol = s"$geomQueryCol::GEOMETRY(ANY)"
+ assertType(s"SELECT $castQueryCol FROM tbl", mixedSridGeometryType)
+ checkAnswer(sql(s"SELECT ST_AsBinary($castQueryCol) FROM tbl"), Row(wkb))
+ }
+ }
+
/** ST reader/writer expressions. */
test("ST_AsBinary") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
