jiayuasu commented on code in PR #52491: URL: https://github.com/apache/spark/pull/52491#discussion_r2418728828
########## sql/api/src/main/scala/org/apache/spark/sql/types/GeographyType.scala: ########## @@ -0,0 +1,274 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.types + +import org.json4s.JsonAST.{JString, JValue} + +import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.annotation.Experimental + +/** + * The data type representing GEOGRAPHY values which are spatial objects, as defined in the + * Open Geospatial Consortium (OGC) Simple Feature Access specification + * (https://portal.ogc.org/files/?artifact_id=25355), with a geographic coordinate system. + */ +@Experimental +class GeographyType private (val crs: String, val algorithm: EdgeInterpolationAlgorithm) + extends AtomicType + with Serializable { + + /** + * Spatial Reference Identifier (SRID) value of the geography type. + */ + val srid: Int = toSrid(crs) + + /** + * The default size of a value of the GeographyType is 2048 bytes, which can store roughly 120 + * 2D points. + */ + override def defaultSize: Int = 2048 + + /** + * The GeographyType is a mixed SRID type iff the SRID is GEOGRAPHY_MIXED_SRID. + * Semantically, this means that different SRID values per row are allowed. + */ + def isMixedSrid: Boolean = srid == GeographyType.GEOGRAPHY_MIXED_SRID + + /** + * Type name that is displayed to users. + */ + override def typeName: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "geography(any)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"geography($srid)" + } + } + + /** + * String representation of the GeographyType, which uses SRID for fixed SRID types and "ANY" + * for mixed SRID types, providing a clear and concise user-friendly format for this type. + */ + override def toString: String = { + if (isMixedSrid) { + // The mixed SRID type is displayed with a special specifier value "ANY". + "GeographyType(ANY)" + } else { + // The fixed SRID type is always displayed with the appropriate SRID value. + s"GeographyType($srid)" + } + } + + /** + * JSON representation of the GeographyType, which uses the CRS string and edge interpolation + * algorithm string, in line with the current storage specifications (e.g. Parquet, Delta, + * Iceberg). Note that mixed SRID is disallowed, and only fixed SRID types can be stored. This + * is also in accordance to storage formats. + */ + override def jsonValue: JValue = JString(s"geography($crs, $algorithm)") + + private[spark] override def asNullable: GeographyType = this + + /** + * Two types are considered equal iff they are both GeographyTypes and have the same SRID value. + */ + override def equals(obj: Any): Boolean = { + obj match { + case g: GeographyType => + // Iff two GeographyTypes have the same SRID, they are considered equal. + g.srid == srid + case _ => + // In all other cases, the two types are considered not equal. + false + } + } + + /** + * The hash code of the GeographyType is derived from its SRID value. + */ + override def hashCode(): Int = srid.hashCode + + /** + * The GeographyType can only accept another type if the other type is also a GeographyType, + * and the SRID values are compatible (see `acceptsGeographyType` below for more details). + */ + override private[sql] def acceptsType(other: DataType): Boolean = { + other match { + case gt: GeographyType => + // For GeographyType, we need to check the SRID values. + acceptsGeographyType(gt) + case _ => + // In all other cases, the two types are considered different. + false + } + } + + /** + * The GeographyType with mixed SRID can accept any other GeographyType, i.e. either a fixed SRID + * GeographyType or another mixed SRID GeographyType. Conversely, a GeographyType with fixed SRID + * can only accept another GeographyType with the same fixed SRID value, and not a mixed SRID. + */ + def acceptsGeographyType(gt: GeographyType): Boolean = { + // If the SRID is mixed, we can accept any other GeographyType. + // If the SRID is not mixed, we can only accept the same SRID. + isMixedSrid || gt.srid == srid + } + + /** + * Converts a CRS string to its corresponding SRID integer value. + */ + private[types] def toSrid(crs: String): Int = { + // The special value "SRID:ANY" is used to represent mixed SRID values. + if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_MIXED_CRS)) { + GeographyType.GEOGRAPHY_MIXED_SRID + } + // As for other valid CRS values, we currently offer limited support. + else if (crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_CRS) || + crs.equalsIgnoreCase(GeographyType.GEOGRAPHY_DEFAULT_EPSG_CRS)) { + GeographyType.GEOGRAPHY_DEFAULT_SRID + } else { + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_CRS_VALUE", + messageParameters = Map("crs" -> crs) + ) + } + } +} + +@Experimental +object GeographyType extends AbstractDataType { + + /** + * Mixed SRID value and the corresponding CRS value for GeographyType. + * This value is completely internal, and should not be exposed to users. + * It means we can have different SRIDs per row. + */ + final val GEOGRAPHY_MIXED_SRID = -1 + final val GEOGRAPHY_MIXED_CRS = "SRID:ANY" + + /** + * Default CRS value for GeographyType depends on storage specification. + * Parquet and Iceberg use OGC:CRS84, which translates to SRID 4326 here. + */ + final val GEOGRAPHY_DEFAULT_CRS = "OGC:CRS84" + final val GEOGRAPHY_DEFAULT_SRID = 4326 + + // The default edge interpolation algorithm value for GeographyType. + final val GEOGRAPHY_DEFAULT_ALGORITHM = EdgeInterpolationAlgorithm.SPHERICAL + + // Another way to represent the default parquet crs value (OGC:CRS84). + final val GEOGRAPHY_DEFAULT_EPSG_CRS = s"EPSG:$GEOGRAPHY_DEFAULT_SRID" + + /** The default GeographyType for storage. */ + val GEOGRAPHY_MIXED_TYPE: GeographyType = + GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + + /** + * Constructors for GeographyType. + */ + def apply(srid: Int): GeographyType = { + if (!isValidSrid(srid)) { + // Limited geographic SRID values are allowed. + throw new SparkIllegalArgumentException( + errorClass = "ST_INVALID_SRID_VALUE", + messageParameters = Map("srid" -> srid.toString) + ) + } + new GeographyType(GEOGRAPHY_DEFAULT_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + } + + def apply(crs: String): GeographyType = { + crs match { + case "ANY" => + // Special value "ANY" is used for mixed SRID values. + // This should be available to users in the Scala API. + new GeographyType(GEOGRAPHY_MIXED_CRS, GEOGRAPHY_DEFAULT_ALGORITHM) + case _ => + // Otherwise, we need to further check the CRS value. + // This shouldn't be available to users in the Scala API. Review Comment: I don't think at the Iceberg / Parquet level, the string `OGC:CRS84` is a naming convention. A unique string of CRS (if not projjson) usually means `authority:code` and the `code` is supposed to be an integer. The intention of calling `OGC:CRS84` in the Parquet / Iceberg spec is to refer to the corresponding projjson. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
