szehon-ho commented on code in PR #52491: URL: https://github.com/apache/spark/pull/52491#discussion_r2393530938
########## sql/api/src/main/scala/org/apache/spark/sql/types/CrsMappings.java: ########## @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types; + +import java.util.HashMap; + +/* + * Class for maintaining mappings between supported SRID values and the string ID of the + * corresponding CRS. + */ +public class CrsMappings { + + // We implement this class as a singleton (we disallow construction). + private CrsMappings() {} + + private static CrsMappings instance = new CrsMappings(); Review Comment: nit: is it usually capitalized? ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeographyType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeographyType can only accept another type if the other type is also a GeographyType, + * and the SRID values are compatible (see `acceptsGeographyType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeographyType => + // For GeographyType, we need to check the SRID values. + acceptsGeographyType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeographyType with mixed SRID can accept any other GeographyType, i.e. either a fixed SRID + * GeographyType or another mixed SRID GeographyType. Conversely, a GeographyType with fixed SRID + * can only accept another GeographyType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeographyType(gt: GeographyType): Boolean = { + // If the SRID is mixed, we can accept any other GeographyType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_MIXED_CRS)) { + GeographyType.GEOGRAPHY_MIXED_SRID + } + // As for other valid CRS values, we currently offer limited support. + else if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_CRS) || + crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_EPSG_CRS)) { + GeographyType.GEOGRAPHY_DEFAULT_SRID + } else { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + } +} + +@Experimental +object GeographyType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeographyType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOGRAPHY_MIXED_SRID = -1 + final val GEOGRAPHY_MIXED_CRS = "SRID:ANY" + + /** + * Default CRS value for GeographyType depends on storage specification. + * Parquet and Iceberg use OGC:CRS84, which translates to SRID 4326 here. + */ + final val GEOGRAPHY_DEFAULT_CRS = "OGC:CRS84" + final val GEOGRAPHY_DEFAULT_SRID = 4326 + + // The default edge interpolation algorithm value for GeographyType. + final val GEOGRAPHY_DEFAULT_ALGORITHM = EdgeInterpolationAlgorithm.SPHERICAL + + // Another way to represent the default parquet crs value (OGC:CRS84). + final val GEOGRAPHY_DEFAULT_EPSG_CRS = s"EPSG:$GEOGRAPHY_DEFAULT_SRID" + + /** The default GeographyType for storage. */ + val GEOGRAPHY_MIXED_TYPE: GeographyType = + GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + + /** + * Constructors for GeographyType. + */ + def apply(srid: Int): GeographyType = { + if (!isValidSrid(srid)) { + // Limited geographic SRID values are allowed. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_SRID_VALUE", + messageParameters = Map("srid" -> srid.toString) + ) + } + new GeographyType(GEOGRAPHY_DEFAULT_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + } + + def apply(crs: String): GeographyType = { + crs match { + case "ANY" => + // Special value "ANY" is used for mixed SRID values. + // This should be available to users in the Scala API. + new GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + case _ => + // Otherwise, we need to further check the CRS value. + // This shouldn't be available to users in the Scala API. + GeographyType(crs, GEOGRAPHY_DEFAULT_ALGORITHM.toString) Review Comment: why discrepancy between new (above) and no new? ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeometryType.scala: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOMETRY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a Cartesian coordinate system. + */ +@Experimental +class GeometryType private (val crs: String) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geometry type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeometryType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeometryType is a mixed SRID type iff the SRID is GEOMETRY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeometryType.GEOMETRY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geometry(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geometry($srid)" + } + } + + /** + * String representation of the GeometryType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeometryType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeometryType($srid)" + } + } + + /** + * JSON representation of the GeometryType, which uses the CRS string, in line with the current + * storage specifications (e.g. Parquet, Delta, Iceberg). Note that mixed SRID is disallowed, + * and only fixed SRID types can be stored. This is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geometry($crs)") + + private[spark] override def asNullable: GeometryType = this + + /** + * Two types are considered equal iff they are both GeometryTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeometryType => + // Iff two GeometryTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeometryType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeometryType can only accept another type if the other type is also a GeometryType, + * and the SRID values are compatible (see `acceptsGeometryType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeometryType => + // For GeometryType, we need to check the SRID values. + acceptsGeometryType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeometryType with a mixed SRID can accept any other GeometryType, i.e. either a fixed SRID + * GeometryType or another mixed SRID GeometryType. Conversely, a GeometryType with a fixed SRID + * can only accept another GeometryType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeometryType(gt: GeometryType): Boolean = { + // If the SRID is mixed, we can accept any other GeometryType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeometryType.GEOMETRY_MIXED_CRS)) { + return GeometryType.GEOMETRY_MIXED_SRID + } + // For all other CRS values, we need to look up the corresponding SRID. + val srid = CrsMappings.get().getSrid(crs) + if (srid == null) { + // If the CRS value is not recognized, we throw an exception. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + srid + } +} + +@Experimental +object GeometryType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeometryType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOMETRY_MIXED_SRID = -1 + final val GEOMETRY_MIXED_CRS = "SRID:ANY" Review Comment: For consideration, there seems some repeated code, both these and Geography mixed and default srid/crs have the same value. Do you consider combining them into just MIXED_SRID and MIXED_CRS in a common object? ########## sql/api/src/main/scala/org/apache/spark/sql/types/DataType.scala: ########## @@ -217,6 +221,14 @@ object DataType { case CHAR_TYPE(length) => CharType(length.toInt) case VARCHAR_TYPE(length) => VarcharType(length.toInt) case STRING_WITH_COLLATION(collation) => StringType(collation) + // If the coordinate reference system (CRS) value is omitted, Parquet and other storage + // formats (Delta, Iceberg) consider "OGC:CRS84" to be the default value of the crs. + case "geometry" => GeometryType(GeometryType.DEFAULT_STORAGE_CRS) + case GEOMETRY_TYPE(crs) => GeometryType(crs) + case "geography" => GeographyType(GeographyType.GEOGRAPHY_DEFAULT_CRS) + case GEOGRAPHY_TYPE_CRS(crs) => GeographyType(crs, GeographyType.GEOGRAPHY_DEFAULT_ALGORITHM) + case GEOGRAPHY_TYPE_ALG(alg) => GeographyType(GeographyType.GEOGRAPHY_DEFAULT_CRS, alg) Review Comment: to confirm, we dont support algorithm as user-input, what is the use case? Is it when reading type metadata from storage that may have algorithm param? ########## sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/DataTypeAstBuilder.scala: ########## @@ -118,6 +118,32 @@ class DataTypeAstBuilder extends SqlBaseParserBaseVisitor[AnyRef] { currentCtx.precision.getText.toInt } TimeType(precision) + case GEOGRAPHY => + // Unparameterized geometry type isn't supported and will be caught by the default branch. + // Here, we only handle the parameterized GEOGRAPHY type syntax, which comes in two forms: + if (currentCtx.srid.getText.toLowerCase(Locale.ROOT) == "any") { + // The special parameterized GEOGRAPHY type syntax has a single "ANY" string value. + // This implies a mixed GEOGRAPHY type, with potentially different SRID values rows. Review Comment: nit: potentially different SRID values rows sounds a bit strange, how about: .. 'with every row potentially having a different SRID' ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeographyType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeographyType can only accept another type if the other type is also a GeographyType, + * and the SRID values are compatible (see `acceptsGeographyType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeographyType => + // For GeographyType, we need to check the SRID values. + acceptsGeographyType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeographyType with mixed SRID can accept any other GeographyType, i.e. either a fixed SRID + * GeographyType or another mixed SRID GeographyType. Conversely, a GeographyType with fixed SRID + * can only accept another GeographyType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeographyType(gt: GeographyType): Boolean = { + // If the SRID is mixed, we can accept any other GeographyType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_MIXED_CRS)) { + GeographyType.GEOGRAPHY_MIXED_SRID + } + // As for other valid CRS values, we currently offer limited support. + else if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_CRS) || + crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_EPSG_CRS)) { + GeographyType.GEOGRAPHY_DEFAULT_SRID + } else { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + } +} + +@Experimental +object GeographyType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeographyType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOGRAPHY_MIXED_SRID = -1 + final val GEOGRAPHY_MIXED_CRS = "SRID:ANY" + + /** + * Default CRS value for GeographyType depends on storage specification. + * Parquet and Iceberg use OGC:CRS84, which translates to SRID 4326 here. + */ + final val GEOGRAPHY_DEFAULT_CRS = "OGC:CRS84" + final val GEOGRAPHY_DEFAULT_SRID = 4326 + + // The default edge interpolation algorithm value for GeographyType. + final val GEOGRAPHY_DEFAULT_ALGORITHM = EdgeInterpolationAlgorithm.SPHERICAL + + // Another way to represent the default parquet crs value (OGC:CRS84). + final val GEOGRAPHY_DEFAULT_EPSG_CRS = s"EPSG:$GEOGRAPHY_DEFAULT_SRID" + + /** The default GeographyType for storage. */ + val GEOGRAPHY_MIXED_TYPE: GeographyType = + GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + + /** + * Constructors for GeographyType. + */ + def apply(srid: Int): GeographyType = { + if (!isValidSrid(srid)) { + // Limited geographic SRID values are allowed. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_SRID_VALUE", + messageParameters = Map("srid" -> srid.toString) + ) + } + new GeographyType(GEOGRAPHY_DEFAULT_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) Review Comment: not lookup crs from CRSMappings? ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid Review Comment: how about alg? ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeographyType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeographyType can only accept another type if the other type is also a GeographyType, + * and the SRID values are compatible (see `acceptsGeographyType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeographyType => + // For GeographyType, we need to check the SRID values. + acceptsGeographyType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeographyType with mixed SRID can accept any other GeographyType, i.e. either a fixed SRID + * GeographyType or another mixed SRID GeographyType. Conversely, a GeographyType with fixed SRID + * can only accept another GeographyType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeographyType(gt: GeographyType): Boolean = { + // If the SRID is mixed, we can accept any other GeographyType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_MIXED_CRS)) { + GeographyType.GEOGRAPHY_MIXED_SRID + } + // As for other valid CRS values, we currently offer limited support. + else if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_CRS) || + crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_EPSG_CRS)) { + GeographyType.GEOGRAPHY_DEFAULT_SRID + } else { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + } +} + +@Experimental +object GeographyType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeographyType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOGRAPHY_MIXED_SRID = -1 + final val GEOGRAPHY_MIXED_CRS = "SRID:ANY" + + /** + * Default CRS value for GeographyType depends on storage specification. + * Parquet and Iceberg use OGC:CRS84, which translates to SRID 4326 here. + */ + final val GEOGRAPHY_DEFAULT_CRS = "OGC:CRS84" + final val GEOGRAPHY_DEFAULT_SRID = 4326 Review Comment: nit: srid can go above to match the other one. ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeographyType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeographyType can only accept another type if the other type is also a GeographyType, + * and the SRID values are compatible (see `acceptsGeographyType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeographyType => + // For GeographyType, we need to check the SRID values. + acceptsGeographyType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeographyType with mixed SRID can accept any other GeographyType, i.e. either a fixed SRID + * GeographyType or another mixed SRID GeographyType. Conversely, a GeographyType with fixed SRID + * can only accept another GeographyType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeographyType(gt: GeographyType): Boolean = { + // If the SRID is mixed, we can accept any other GeographyType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_MIXED_CRS)) { + GeographyType.GEOGRAPHY_MIXED_SRID + } + // As for other valid CRS values, we currently offer limited support. + else if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_CRS) || + crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_EPSG_CRS)) { + GeographyType.GEOGRAPHY_DEFAULT_SRID + } else { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + } +} + +@Experimental +object GeographyType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeographyType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOGRAPHY_MIXED_SRID = -1 + final val GEOGRAPHY_MIXED_CRS = "SRID:ANY" + + /** + * Default CRS value for GeographyType depends on storage specification. + * Parquet and Iceberg use OGC:CRS84, which translates to SRID 4326 here. + */ + final val GEOGRAPHY_DEFAULT_CRS = "OGC:CRS84" + final val GEOGRAPHY_DEFAULT_SRID = 4326 + + // The default edge interpolation algorithm value for GeographyType. + final val GEOGRAPHY_DEFAULT_ALGORITHM = EdgeInterpolationAlgorithm.SPHERICAL + + // Another way to represent the default parquet crs value (OGC:CRS84). + final val GEOGRAPHY_DEFAULT_EPSG_CRS = s"EPSG:$GEOGRAPHY_DEFAULT_SRID" + + /** The default GeographyType for storage. */ + val GEOGRAPHY_MIXED_TYPE: GeographyType = Review Comment: nit: final ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeographyType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeographyType can only accept another type if the other type is also a GeographyType, + * and the SRID values are compatible (see `acceptsGeographyType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeographyType => + // For GeographyType, we need to check the SRID values. + acceptsGeographyType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeographyType with mixed SRID can accept any other GeographyType, i.e. either a fixed SRID + * GeographyType or another mixed SRID GeographyType. Conversely, a GeographyType with fixed SRID + * can only accept another GeographyType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeographyType(gt: GeographyType): Boolean = { + // If the SRID is mixed, we can accept any other GeographyType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_MIXED_CRS)) { + GeographyType.GEOGRAPHY_MIXED_SRID + } + // As for other valid CRS values, we currently offer limited support. + else if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_CRS) || + crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_EPSG_CRS)) { + GeographyType.GEOGRAPHY_DEFAULT_SRID + } else { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + } +} + +@Experimental +object GeographyType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeographyType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOGRAPHY_MIXED_SRID = -1 + final val GEOGRAPHY_MIXED_CRS = "SRID:ANY" + + /** + * Default CRS value for GeographyType depends on storage specification. + * Parquet and Iceberg use OGC:CRS84, which translates to SRID 4326 here. + */ + final val GEOGRAPHY_DEFAULT_CRS = "OGC:CRS84" + final val GEOGRAPHY_DEFAULT_SRID = 4326 + + // The default edge interpolation algorithm value for GeographyType. + final val GEOGRAPHY_DEFAULT_ALGORITHM = EdgeInterpolationAlgorithm.SPHERICAL + + // Another way to represent the default parquet crs value (OGC:CRS84). + final val GEOGRAPHY_DEFAULT_EPSG_CRS = s"EPSG:$GEOGRAPHY_DEFAULT_SRID" + + /** The default GeographyType for storage. */ + val GEOGRAPHY_MIXED_TYPE: GeographyType = + GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + + /** + * Constructors for GeographyType. + */ + def apply(srid: Int): GeographyType = { + if (!isValidSrid(srid)) { + // Limited geographic SRID values are allowed. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_SRID_VALUE", + messageParameters = Map("srid" -> srid.toString) + ) + } + new GeographyType(GEOGRAPHY_DEFAULT_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + } + + def apply(crs: String): GeographyType = { + crs match { + case "ANY" => + // Special value "ANY" is used for mixed SRID values. + // This should be available to users in the Scala API. + new GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + case _ => + // Otherwise, we need to further check the CRS value. + // This shouldn't be available to users in the Scala API. Review Comment: confused about the comment, is there some guard against someone calling this API and not pass 'ANY'? Do you mean the scala api as in : GeographyType("OGC:CRS84") ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeometryType.scala: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOMETRY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a Cartesian coordinate system. + */ +@Experimental +class GeometryType private (val crs: String) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geometry type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeometryType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeometryType is a mixed SRID type iff the SRID is GEOMETRY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeometryType.GEOMETRY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geometry(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geometry($srid)" + } + } + + /** + * String representation of the GeometryType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeometryType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeometryType($srid)" + } + } + + /** + * JSON representation of the GeometryType, which uses the CRS string, in line with the current + * storage specifications (e.g. Parquet, Delta, Iceberg). Note that mixed SRID is disallowed, + * and only fixed SRID types can be stored. This is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geometry($crs)") + + private[spark] override def asNullable: GeometryType = this + + /** + * Two types are considered equal iff they are both GeometryTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeometryType => + // Iff two GeometryTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeometryType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeometryType can only accept another type if the other type is also a GeometryType, + * and the SRID values are compatible (see `acceptsGeometryType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeometryType => + // For GeometryType, we need to check the SRID values. + acceptsGeometryType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeometryType with a mixed SRID can accept any other GeometryType, i.e. either a fixed SRID + * GeometryType or another mixed SRID GeometryType. Conversely, a GeometryType with a fixed SRID + * can only accept another GeometryType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeometryType(gt: GeometryType): Boolean = { + // If the SRID is mixed, we can accept any other GeometryType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeometryType.GEOMETRY_MIXED_CRS)) { + return GeometryType.GEOMETRY_MIXED_SRID + } + // For all other CRS values, we need to look up the corresponding SRID. + val srid = CrsMappings.get().getSrid(crs) + if (srid == null) { + // If the CRS value is not recognized, we throw an exception. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + srid + } +} + +@Experimental +object GeometryType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeometryType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOMETRY_MIXED_SRID = -1 + final val GEOMETRY_MIXED_CRS = "SRID:ANY" + + /** + * The default coordinate reference system (CRS) value used for geometries, + * as specified by the Parquet, Delta, and Iceberg specifications. + * If crs is omitted, it should always default to this. + */ + final val DEFAULT_STORAGE_CRS = "OGC:CRS84" Review Comment: nit: should be GEOMETRY_DEFAULT_CRS? (to match Geography). I think be nice to extract to common ########## sql/api/src/main/scala/org/apache/spark/sql/types/CrsMappings.java: ########## @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types; + +import java.util.HashMap; + +/* + * Class for maintaining mappings between supported SRID values and the string ID of the + * corresponding CRS. + */ +public class CrsMappings { + + // We implement this class as a singleton (we disallow construction). + private CrsMappings() {} + + private static CrsMappings instance = new CrsMappings(); Review Comment: also , wonder is it worth to do lazily or not? (else might as well just expose INSTANCE) ########## sql/api/src/main/scala/org/apache/spark/sql/types/GeometryType.scala: ########## @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOMETRY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a Cartesian coordinate system. + */ +@Experimental +class GeometryType private (val crs: String) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geometry type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeometryType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeometryType is a mixed SRID type iff the SRID is GEOMETRY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeometryType.GEOMETRY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geometry(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geometry($srid)" + } + } + + /** + * String representation of the GeometryType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeometryType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeometryType($srid)" + } + } + + /** + * JSON representation of the GeometryType, which uses the CRS string, in line with the current + * storage specifications (e.g. Parquet, Delta, Iceberg). Note that mixed SRID is disallowed, + * and only fixed SRID types can be stored. This is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geometry($crs)") + + private[spark] override def asNullable: GeometryType = this + + /** + * Two types are considered equal iff they are both GeometryTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeometryType => + // Iff two GeometryTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeometryType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeometryType can only accept another type if the other type is also a GeometryType, + * and the SRID values are compatible (see `acceptsGeometryType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeometryType => + // For GeometryType, we need to check the SRID values. + acceptsGeometryType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeometryType with a mixed SRID can accept any other GeometryType, i.e. either a fixed SRID + * GeometryType or another mixed SRID GeometryType. Conversely, a GeometryType with a fixed SRID + * can only accept another GeometryType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeometryType(gt: GeometryType): Boolean = { + // If the SRID is mixed, we can accept any other GeometryType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeometryType.GEOMETRY_MIXED_CRS)) { + return GeometryType.GEOMETRY_MIXED_SRID + } + // For all other CRS values, we need to look up the corresponding SRID. + val srid = CrsMappings.get().getSrid(crs) + if (srid == null) { + // If the CRS value is not recognized, we throw an exception. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + srid + } +} + +@Experimental +object GeometryType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeometryType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOMETRY_MIXED_SRID = -1 + final val GEOMETRY_MIXED_CRS = "SRID:ANY" + + /** + * The default coordinate reference system (CRS) value used for geometries, + * as specified by the Parquet, Delta, and Iceberg specifications. + * If crs is omitted, it should always default to this. + */ + final val DEFAULT_STORAGE_CRS = "OGC:CRS84" + final val DEFAULT_STORAGE_CRS_SRID = 4326 + + /** The default GeometryType for storage. */ + val GEOMETRY_MIXED_TYPE: GeometryType = GeometryType(GEOMETRY_MIXED_CRS) + + /** + * Constructors for GeometryType. + */ + def apply(srid: Int): GeometryType = { + val crs = CrsMappings.get().getStringId(srid) + if (crs == null) { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_SRID_VALUE", + messageParameters = Map("srid" -> srid.toString) + ) + } + new GeometryType(crs) + } + + def apply(crs: String): GeometryType = { + crs match { Review Comment: Why not just delegate to new GeometryType(crs) which does all the checks? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
