This is an automated email from the ASF dual-hosted git repository. jiayu pushed a commit to branch clean-up-flink-dependency in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 63e444605922736ca5bb7e4798ad8219485ab0d3 Author: Jia Yu <[email protected]> AuthorDate: Tue May 9 23:08:11 2023 -0700 Get rid of Sedona-Spark dependencies in Sedona-Flink --- common/pom.xml | 4 + .../common/geometrySerde}/GeometrySerde.java | 2 +- .../common/geometrySerde}/SpatialIndexSerde.java | 3 +- .../jts/index/quadtree/IndexSerde.java | 31 +-- .../locationtech/jts/index/strtree/IndexSerde.java | 31 +-- .../java/org/apache/sedona/common}/CircleTest.java | 2 +- .../geometrySerde}/SpatialIndexSerdeTest.java | 3 +- .../sedona/core/serde/SedonaKryoRegistrator.java | 4 +- .../shapefileParser/shapes/GeometrySerdeTest.java | 2 +- flink/pom.xml | 10 - .../sedona/flink/SedonaFlinkRegistrator.java | 5 +- .../sedona/flink/expressions/Constructors.java | 13 +- .../sedona/flink/expressions/Predicates.java | 213 +++++---------------- pom.xml | 7 +- .../expressions/geohash/GeoHashDecoder.scala | 78 -------- .../sedona/sql/functions/geohash/Fixtures.scala | 7 +- .../sql/functions/geohash/TestGeoHashDecoder.scala | 6 +- 17 files changed, 118 insertions(+), 303 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index ed685ea7..7a291375 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -73,6 +73,10 @@ <groupId>com.google.geometry</groupId> <artifactId>s2-geometry</artifactId> </dependency> + <dependency> + <groupId>com.esotericsoftware</groupId> + <artifactId>kryo</artifactId> + </dependency> </dependencies> <build> <sourceDirectory>src/main/java</sourceDirectory> diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java similarity index 99% rename from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java rename to common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java index e4aa6c0d..0b779e2f 100755 --- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/GeometrySerde.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.sedona.core.geometryObjects; +package org.apache.sedona.common.geometrySerde; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Registration; diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java b/common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java similarity index 97% rename from core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java rename to common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java index f4b5f6d2..efaf2e97 100644 --- a/core/src/main/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerde.java +++ b/common/src/main/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerde.java @@ -17,13 +17,12 @@ * under the License. */ -package org.apache.sedona.core.geometryObjects; +package org.apache.sedona.common.geometrySerde; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Serializer; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.log4j.Logger; import org.locationtech.jts.index.quadtree.IndexSerde; import org.locationtech.jts.index.quadtree.Quadtree; import org.locationtech.jts.index.strtree.STRtree; diff --git a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java b/common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java similarity index 79% rename from core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java rename to common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java index 5dd65566..46f8218c 100644 --- a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java +++ b/common/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java @@ -1,22 +1,27 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.locationtech.jts.index.quadtree; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.common.geometrySerde.GeometrySerde; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; diff --git a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java b/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java similarity index 85% rename from core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java rename to common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java index bf3d308b..2501befb 100644 --- a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java +++ b/common/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java @@ -1,22 +1,27 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.locationtech.jts.index.strtree; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.common.geometrySerde.GeometrySerde; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java b/common/src/test/java/org/apache/sedona/common/CircleTest.java similarity index 99% rename from core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java rename to common/src/test/java/org/apache/sedona/common/CircleTest.java index 58781dd4..f4bef207 100644 --- a/core/src/test/java/org/apache/sedona/core/geometryObjects/CircleTest.java +++ b/common/src/test/java/org/apache/sedona/common/CircleTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.core.geometryObjects; +package org.apache.sedona.common; import org.apache.sedona.common.geometryObjects.Circle; import org.apache.sedona.common.utils.GeomUtils; diff --git a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java b/common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java similarity index 98% rename from core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java rename to common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java index acb08736..84dfc964 100644 --- a/core/src/test/java/org/apache/sedona/core/geometryObjects/SpatialIndexSerdeTest.java +++ b/common/src/test/java/org/apache/sedona/common/geometrySerde/SpatialIndexSerdeTest.java @@ -17,11 +17,12 @@ * under the License. */ -package org.apache.sedona.core.geometryObjects; +package org.apache.sedona.common.geometrySerde; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; +import org.apache.sedona.common.geometrySerde.SpatialIndexSerde; import org.junit.Test; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Envelope; diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java index f3c62665..113fb434 100644 --- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java +++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java @@ -22,8 +22,8 @@ package org.apache.sedona.core.serde; import com.esotericsoftware.kryo.Kryo; import org.apache.log4j.Logger; import org.apache.sedona.common.geometryObjects.Circle; -import org.apache.sedona.core.geometryObjects.GeometrySerde; -import org.apache.sedona.core.geometryObjects.SpatialIndexSerde; +import org.apache.sedona.common.geometrySerde.GeometrySerde; +import org.apache.sedona.common.geometrySerde.SpatialIndexSerde; import org.apache.spark.serializer.KryoRegistrator; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.GeometryCollection; diff --git a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java index 0e8d88ab..924e9f13 100644 --- a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java +++ b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java @@ -24,7 +24,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import org.apache.sedona.common.utils.GeomUtils; import org.apache.sedona.common.geometryObjects.Circle; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.common.geometrySerde.GeometrySerde; import org.junit.Test; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; diff --git a/flink/pom.xml b/flink/pom.xml index 9c52f3b9..1c52fa14 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -44,16 +44,6 @@ <artifactId>sedona-common</artifactId> <version>${project.version}</version> </dependency> - <dependency> - <groupId>org.apache.sedona</groupId> - <artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.sedona</groupId> - <artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId> - <version>${project.version}</version> - </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-core</artifactId> diff --git a/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java b/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java index 36a6e492..ef55cdd6 100644 --- a/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java +++ b/flink/src/main/java/org/apache/sedona/flink/SedonaFlinkRegistrator.java @@ -16,9 +16,8 @@ package org.apache.sedona.flink; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.sedona.common.geometryObjects.Circle; -import org.apache.sedona.core.geometryObjects.GeometrySerde; -import org.apache.sedona.core.geometryObjects.SpatialIndexSerde; -import org.apache.sedona.core.spatialPartitioning.PartitioningUtils; +import org.apache.sedona.common.geometrySerde.GeometrySerde; +import org.apache.sedona.common.geometrySerde.SpatialIndexSerde; import org.locationtech.jts.geom.*; import org.locationtech.jts.index.quadtree.Quadtree; import org.locationtech.jts.index.strtree.STRtree; diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java index 08cc0ed9..36bbeba8 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Constructors.java @@ -18,7 +18,7 @@ import org.apache.flink.table.functions.ScalarFunction; import org.apache.sedona.common.enums.FileDataSplitter; import org.apache.sedona.common.enums.GeometryType; import org.apache.sedona.common.utils.FormatUtils; -import org.apache.spark.sql.sedona_sql.expressions.geohash.GeoHashDecoder; +import org.apache.sedona.common.utils.GeoHashDecoder; import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryFactory; @@ -155,14 +155,17 @@ public class Constructors { public static class ST_GeomFromGeoHash extends ScalarFunction { @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) public Geometry eval(@DataTypeHint("String") String value, - @DataTypeHint("Int") Integer precision) throws ParseException { + @DataTypeHint("Int") Integer precision) + throws ParseException, GeoHashDecoder.InvalidGeoHashException + { // The default precision is the geohash length. Otherwise, use the precision given by the user - scala.Option<Object> optionPrecision = scala.Option.apply(precision); - return GeoHashDecoder.decode(value, optionPrecision); + return GeoHashDecoder.decode(value, precision); } @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) - public Geometry eval(@DataTypeHint("String") String value) throws ParseException { + public Geometry eval(@DataTypeHint("String") String value) + throws ParseException, GeoHashDecoder.InvalidGeoHashException + { return eval(value, null); } } diff --git a/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java b/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java index e7f797d9..b29e6ad5 100644 --- a/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java +++ b/flink/src/main/java/org/apache/sedona/flink/expressions/Predicates.java @@ -15,258 +15,141 @@ package org.apache.sedona.flink.expressions; import org.apache.flink.table.annotation.DataTypeHint; import org.apache.flink.table.functions.ScalarFunction; -import org.apache.sedona.common.utils.GeomUtils; -import org.apache.sedona.core.spatialPartitioning.PartitioningUtils; -import org.apache.sedona.common.utils.HalfOpenRectangle; -import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; -import java.util.List; -import java.util.Objects; - public class Predicates { - public static class ST_Intersects extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_Intersects(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } + public static class ST_Intersects extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_Intersects() { + public ST_Intersects() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.intersects(geom2); } - - /** - * Check spatial relation with duplicates removal - * @param key - * @param o1 - * @param o2 - * @return True if intersecting, false otherwise - */ - @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { - Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner."); - Geometry geom1 = (Geometry) o1; - Geometry geom2 = (Geometry) o2; - HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key)); - return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.intersects(geom2); - } } - public static class ST_Contains extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_Contains(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } + public static class ST_Contains + extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_Contains() { + public ST_Contains() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.contains(geom2); } - - /** - * Check spatial relation with duplicates removal - * @param key - * @param o1 - * @param o2 - * @return True if o1 contains o2, false otherwise - */ - @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { - Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner."); - Geometry geom1 = (Geometry) o1; - Geometry geom2 = (Geometry) o2; - HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key)); - return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.contains(geom2); - } } - public static class ST_Within extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_Within(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } - + public static class ST_Within + extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_Within() { + public ST_Within() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.within(geom2); } - - /** - * Check spatial relation with duplicates removal - * @param key - * @param o1 - * @param o2 - * @return True if o1 is within o2, false otherwise - */ - @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { - Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner."); - Geometry geom1 = (Geometry) o1; - Geometry geom2 = (Geometry) o2; - HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key)); - return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.within(geom2); - } } - public static class ST_Covers extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_Covers(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } + public static class ST_Covers + extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_Covers() { + public ST_Covers() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.covers(geom2); } - - /** - * Check spatial relation with duplicates removal - * @param key - * @param o1 - * @param o2 - * @return True if o1 covers o2, false otherwise - */ - @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { - Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner."); - Geometry geom1 = (Geometry) o1; - Geometry geom2 = (Geometry) o2; - HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key)); - return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.covers(geom2); - } } - public static class ST_CoveredBy extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_CoveredBy(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } + public static class ST_CoveredBy + extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_CoveredBy() { + public ST_CoveredBy() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.coveredBy(geom2); } - - /** - * Check spatial relation with duplicates removal - * @param key - * @param o1 - * @param o2 - * @return True if o1 is covered by o2, false otherwise - */ - @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint("INT") Integer key, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { - Objects.requireNonNull(grids, "This predicate has to be initialized by a partitioner."); - Geometry geom1 = (Geometry) o1; - Geometry geom2 = (Geometry) o2; - HalfOpenRectangle halfOpenRectangle = new HalfOpenRectangle(grids.get(key)); - return !GeomUtils.isDuplicate(geom1, geom2, halfOpenRectangle) && geom1.coveredBy(geom2); - } } - public static class ST_Disjoint extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_Disjoint(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } + public static class ST_Disjoint + extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_Disjoint() { + public ST_Disjoint() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.disjoint(geom2); } } - public static class ST_OrderingEquals extends ScalarFunction { - private List<Envelope> grids; - - /** - * Constructor for duplicate removal - */ - public ST_OrderingEquals(PartitioningUtils partitioner) { - grids = partitioner.fetchLeafZones(); - } + public static class ST_OrderingEquals + extends ScalarFunction + { /** * Constructor for relation checking without duplicate removal */ - public ST_OrderingEquals() { + public ST_OrderingEquals() + { } @DataTypeHint("Boolean") - public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) { + public Boolean eval(@DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o1, @DataTypeHint(value = "RAW", bridgedTo = org.locationtech.jts.geom.Geometry.class) Object o2) + { Geometry geom1 = (Geometry) o1; Geometry geom2 = (Geometry) o2; return geom1.equalsExact(geom2); diff --git a/pom.xml b/pom.xml index a98c97df..7c7aa490 100644 --- a/pom.xml +++ b/pom.xml @@ -279,7 +279,12 @@ <artifactId>commons-lang</artifactId> <version>2.6</version> </dependency> - + <dependency> + <groupId>com.esotericsoftware</groupId> + <artifactId>kryo</artifactId> + <version>4.0.2</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> diff --git a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala b/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala deleted file mode 100644 index 9e510cd0..00000000 --- a/sql/common/src/main/scala/org/apache/spark/sql/sedona_sql/expressions/geohash/GeoHashDecoder.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.spark.sql.sedona_sql.expressions.geohash - -import org.apache.sedona.common.utils.BBox -import org.locationtech.jts.geom.{Geometry, GeometryFactory} - -import scala.collection.mutable - -class InvalidGeoHashException(message: String) extends Exception(message) - -object GeoHashDecoder { - private val bits = Seq(16, 8, 4, 2, 1) - private val base32 = "0123456789bcdefghjkmnpqrstuvwxyz" - val geometryFactory = new GeometryFactory() - - def decode(geohash: String, precision: Option[Int] = None): Geometry = { - decodeGeoHashBBox(geohash, precision).getBbox().toPolygon() - } - - private def decodeGeoHashBBox(geohash: String, precision: Option[Int] = None): LatLon = { - val latLon = LatLon(mutable.Seq(-180, 180), mutable.Seq(-90, 90)) - val geoHashLowered = geohash.toLowerCase() - val geoHashLength = geohash.length - val targetPrecision = precision match { - case Some(value) => if (value < 0) throw new InvalidGeoHashException("Precision can not be negative") - else math.min(geoHashLength, value) - case None => geoHashLength - } - var isEven = true - - for (i <- Range(0, targetPrecision)){ - val c = geoHashLowered(i) - val cd = base32.indexOf(c).toByte - if (cd == -1){ - throw new InvalidGeoHashException(s"Invalid character '$c' found at index $i") - } - - for (j <- Range(0, 5)){ - val mask = bits(j).toByte - val index = if ((mask & cd) == 0) 1 else 0 - if (isEven){ - latLon.lons(index) = (latLon.lons.head + latLon.lons(1)) / 2 - } - else { - latLon.lats(index) = (latLon.lats.head + latLon.lats(1)) / 2 - } - isEven = !isEven - } - } - latLon - } -} - -private case class LatLon(var lons: mutable.Seq[Double], var lats: mutable.Seq[Double]){ - def getBbox(): BBox = new BBox( - lons.head, - lons(1), - lats.head, - lats(1) - ) -} diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala index 1d729286..6470e37f 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/Fixtures.scala @@ -18,10 +18,9 @@ */ package org.apache.sedona.sql.functions.geohash -import org.apache.sedona.common.utils.GeometryGeoHashEncoder -import org.apache.spark.sql.sedona_sql.expressions.geohash.GeoHashDecoder +import org.apache.sedona.common.utils.{GeoHashDecoder, GeometryGeoHashEncoder} import org.locationtech.jts.geom.Geometry -import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor2, TableFor3, TableFor4} +import org.scalatest.prop.{TableDrivenPropertyChecks, TableFor3, TableFor4} object Fixtures extends TableDrivenPropertyChecks { @@ -67,7 +66,7 @@ object Fixtures extends TableDrivenPropertyChecks { Option(GeometryGeoHashEncoder.calculate(geom, precision)) } - def decodeGeoHash(geohash: String, precision: Option[Int]): Geometry = + def decodeGeoHash(geohash: String, precision: Int): Geometry = GeoHashDecoder.decode(geohash, precision) } \ No newline at end of file diff --git a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala index 5cffc1f9..3740a72a 100644 --- a/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala +++ b/sql/common/src/test/scala/org/apache/sedona/sql/functions/geohash/TestGeoHashDecoder.scala @@ -18,8 +18,8 @@ */ package org.apache.sedona.sql.functions.geohash +import org.apache.sedona.common.utils.GeoHashDecoder.InvalidGeoHashException import org.apache.sedona.sql.functions.FunctionsHelper -import org.apache.spark.sql.sedona_sql.expressions.geohash.InvalidGeoHashException import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks @@ -27,13 +27,13 @@ import org.scalatest.prop.TableDrivenPropertyChecks class TestGeoHashDecoder extends AnyFunSuite with Matchers with TableDrivenPropertyChecks with FunctionsHelper{ for ((statement: String, geoHash: String, precision: Int, geometry: String) <- Fixtures.geometriesFromGeoHash) { test("it should decode " + statement) { - Fixtures.decodeGeoHash(geoHash, Some(precision)) shouldBe wktReader.read(geometry) + Fixtures.decodeGeoHash(geoHash, precision) shouldBe wktReader.read(geometry) } } for ((statement: String, geoHash: String, precision: Int) <- Fixtures.invalidGeoHashes) { test("it should raise InvalidGeoHashException when " + statement) { - an[InvalidGeoHashException] shouldBe thrownBy(Fixtures.decodeGeoHash(geoHash, Some(precision))) + an[InvalidGeoHashException] shouldBe thrownBy(Fixtures.decodeGeoHash(geoHash, precision)) } } }
