This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new ec3c6596 [DOCS] Update Sedona website, examples, jupyter notebooks to
1.4.0 (#802)
ec3c6596 is described below
commit ec3c659620bed5479b19ad13a4ceb97db7c83099
Author: Jia Yu <[email protected]>
AuthorDate: Sun Mar 19 20:44:37 2023 -0700
[DOCS] Update Sedona website, examples, jupyter notebooks to 1.4.0 (#802)
---
.github/workflows/example.yml | 13 +-
binder/ApacheSedonaCore.ipynb | 10 +-
binder/ApacheSedonaRaster.ipynb | 2 +-
binder/ApacheSedonaSQL.ipynb | 3 +-
...eSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb | 6 +-
binder/Pipfile | 4 +-
binder/postBuild | 6 +-
binder/start | 2 +-
docs/api/sql/Optimizer.md | 29 ++-
docs/community/publish.md | 3 +-
docs/download.md | 14 +-
docs/index.md | 5 +-
docs/setup/release-notes.md | 6 +-
docs/tutorial/demo.md | 23 ++-
docs/tutorial/flink/sql.md | 108 +++++++++-
docs/tutorial/geopandas-shapely.md | 3 +
docs/tutorial/rdd.md | 2 +
examples/{sql => flink-sql}/.gitignore | 0
examples/flink-sql/pom.xml | 222 +++++++++++++++++++++
examples/flink-sql/src/main/java/FlinkExample.java | 80 ++++++++
examples/flink-sql/src/main/java/Utils.java | 117 +++++++++++
.../src/test/resources/.gitignore | 0
.../src/test/resources/raster/test1.tiff | Bin
.../src/test/resources/raster/test2.tiff | Bin
.../src/test/resources/scalastyle_config.xml | 0
.../src/test/resources/shapefiles/dbf/map.dbf | Bin
.../src/test/resources/shapefiles/dbf/map.shp | Bin
.../src/test/resources/shapefiles/dbf/map.shx | Bin
.../src/test/resources/testenvelope.csv | 0
.../src/test/resources/testpoint.csv | 0
.../.gitignore | 0
.../README.md | 0
.../build.sbt | 6 +-
.../colocationMap.png | Bin
.../project/.gitignore | 0
.../project/build.properties | 0
.../project/plugins.sbt | 0
.../src/main/scala/ScalaExample.scala | 0
.../src/test/resources/arealandmark.png | Bin
.../src/test/resources/colocation.png | Bin
.../src/test/resources/colocationMap.png | Bin
.../src/test/resources/colocationResult.png | Bin
.../nyc-area-landmark-shapefile/nyu_2451_34514.cst | 0
.../nyc-area-landmark-shapefile/nyu_2451_34514.dbf | Bin
.../nyc-area-landmark-shapefile/nyu_2451_34514.prj | 0
.../nyc-area-landmark-shapefile/nyu_2451_34514.shp | Bin
.../nyc-area-landmark-shapefile/nyu_2451_34514.shx | Bin
.../nyc-area-landmark-shapefile/wfsrequest.txt | 0
.../src/test/resources/nyccolocation.png | Bin
.../resources/yellow_tripdata_2009-01-subset.csv | 0
examples/{sql => spark-sql}/.gitignore | 0
examples/{sql => spark-sql}/build.sbt | 6 +-
examples/{sql => spark-sql}/project/.gitignore | 0
.../{sql => spark-sql}/project/build.properties | 0
examples/{sql => spark-sql}/project/plugins.sbt | 0
.../src/main/scala/ScalaExample.scala | 0
.../src/test/resources/.gitignore | 0
.../src/test/resources/raster/test1.tiff | Bin
.../src/test/resources/raster/test2.tiff | Bin
.../src/test/resources/scalastyle_config.xml | 0
.../src/test/resources/shapefiles/dbf/map.dbf | Bin
.../src/test/resources/shapefiles/dbf/map.shp | Bin
.../src/test/resources/shapefiles/dbf/map.shx | Bin
.../src/test/resources/testenvelope.csv | 0
.../src/test/resources/testpoint.csv | 0
examples/{viz => spark-viz}/.gitignore | 0
examples/{viz => spark-viz}/build.sbt | 6 +-
examples/{viz => spark-viz}/project/.gitignore | 0
.../{viz => spark-viz}/project/build.properties | 0
examples/{viz => spark-viz}/project/plugins.sbt | 0
.../src/main/scala/ScalaExample.scala | 0
.../src/test/resources/arealm.csv | 0
.../src/test/resources/babylon.point.properties | 0
.../src/test/resources/babylon.polygon.properties | 0
.../test/resources/babylon.rectangle.properties | 0
.../src/test/resources/primaryroads-polygon.csv | 0
.../src/test/resources/zcta510.csv | 0
mkdocs.yml | 7 +-
78 files changed, 613 insertions(+), 70 deletions(-)
diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml
index eccc97dd..41fab7cd 100644
--- a/.github/workflows/example.yml
+++ b/.github/workflows/example.yml
@@ -32,6 +32,13 @@ jobs:
~/.ivy2/cache
~/.sbt
key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }}
- - run: (cd examples/rdd-colocation-mining;sbt clean assembly;java -jar
target/scala-2.12/*.jar)
- - run: (cd examples/sql;sbt clean assembly;java -jar
target/scala-2.12/*.jar)
- - run: (cd examples/viz;sbt clean assembly;java -jar
target/scala-2.12/*.jar)
+ - name: Cache Maven packages
+ uses: actions/cache@v2
+ with:
+ path: ~/.m2
+ key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ runner.os }}-m2
+ - run: (cd examples/spark-rdd-colocation-mining;sbt clean assembly;java
-jar target/scala-2.12/*.jar)
+ - run: (cd examples/spark-sql;sbt clean assembly;java -jar
target/scala-2.12/*.jar)
+ - run: (cd examples/spark-viz;sbt clean assembly;java -jar
target/scala-2.12/*.jar)
+ - run: (cd examples/flink-sql;mvn clean install;java -jar
target/sedona-flink-example-1.0.0.jar)
\ No newline at end of file
diff --git a/binder/ApacheSedonaCore.ipynb b/binder/ApacheSedonaCore.ipynb
index 57983a81..d219ee98 100644
--- a/binder/ApacheSedonaCore.ipynb
+++ b/binder/ApacheSedonaCore.ipynb
@@ -75,7 +75,7 @@
" appName(\"Sedona App\").\\\n",
" config(\"spark.serializer\", KryoSerializer.getName).\\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)
.\\\n",
- " config(\"spark.jars.packages\",
\"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\")
.\\\n",
+ " config(\"spark.jars.packages\",
\"org.apache.sedona:sedona-spark-shaded-3.0_2.12:1.4.0,org.datasyslab:geotools-wrapper:1.4.0-28.2\")
.\\\n",
" getOrCreate()"
]
},
@@ -183,9 +183,7 @@
"outputs": [
{
"data": {
- "image/svg+xml": [
- "<svg xmlns=\"http://www.w3.org/2000/svg\";
xmlns:xlink=\"http://www.w3.org/1999/xlink\"; width=\"100.0\" height=\"100.0\"
viewBox=\"-176.64696132 26.718666680000002 95.20719264000002
48.162659640000015\" preserveAspectRatio=\"xMinYMin meet\"><g
transform=\"matrix(1,0,0,-1,0,101.59999300000001)\"><path fill-rule=\"evenodd\"
fill=\"#66cc99\" stroke=\"#555555\" stroke-width=\"1.9041438528000003\"
opacity=\"0.6\" d=\"M -173.120769,30.244859 L -173.120769,71.355134 L
-84.965961,71.35513 [...]
- ],
+ "image/svg+xml": "<svg xmlns=\"http://www.w3.org/2000/svg\";
xmlns:xlink=\"http://www.w3.org/1999/xlink\"; width=\"100.0\" height=\"100.0\"
viewBox=\"-176.64696132 26.718666680000002 95.20719264000002
48.162659640000015\" preserveAspectRatio=\"xMinYMin meet\"><g
transform=\"matrix(1,0,0,-1,0,101.59999300000001)\"><path fill-rule=\"evenodd\"
fill=\"#66cc99\" stroke=\"#555555\" stroke-width=\"1.9041438528000003\"
opacity=\"0.6\" d=\"M -173.120769,30.244859 L -173.120769,71.355134 L -84 [...]
"text/plain": [
"Envelope(-173.120769, -84.965961, 30.244859, 71.355134)"
]
@@ -229,9 +227,7 @@
"outputs": [
{
"data": {
- "image/svg+xml": [
- "<svg xmlns=\"http://www.w3.org/2000/svg\";
xmlns:xlink=\"http://www.w3.org/1999/xlink\"; width=\"100.0\" height=\"100.0\"
viewBox=\"-176.64696132 26.718666680000002 95.20719264000002
48.162659640000015\" preserveAspectRatio=\"xMinYMin meet\"><g
transform=\"matrix(1,0,0,-1,0,101.59999300000001)\"><path fill-rule=\"evenodd\"
fill=\"#66cc99\" stroke=\"#555555\" stroke-width=\"1.9041438528000003\"
opacity=\"0.6\" d=\"M -173.120769,30.244859 L -173.120769,71.355134 L
-84.965961,71.35513 [...]
- ],
+ "image/svg+xml": "<svg xmlns=\"http://www.w3.org/2000/svg\";
xmlns:xlink=\"http://www.w3.org/1999/xlink\"; width=\"100.0\" height=\"100.0\"
viewBox=\"-176.64696132 26.718666680000002 95.20719264000002
48.162659640000015\" preserveAspectRatio=\"xMinYMin meet\"><g
transform=\"matrix(1,0,0,-1,0,101.59999300000001)\"><path fill-rule=\"evenodd\"
fill=\"#66cc99\" stroke=\"#555555\" stroke-width=\"1.9041438528000003\"
opacity=\"0.6\" d=\"M -173.120769,30.244859 L -173.120769,71.355134 L -84 [...]
"text/plain": [
"Envelope(-173.120769, -84.965961, 30.244859, 71.355134)"
]
diff --git a/binder/ApacheSedonaRaster.ipynb b/binder/ApacheSedonaRaster.ipynb
index 0952c8a6..525e7d6d 100644
--- a/binder/ApacheSedonaRaster.ipynb
+++ b/binder/ApacheSedonaRaster.ipynb
@@ -63,7 +63,7 @@
" appName(\"Demo-app\").\\\n",
" config(\"spark.serializer\", KryoSerializer.getName).\\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)
.\\\n",
- " config(\"spark.jars.packages\",
\"org.apache.sedona:sedona-python-adapter-3.0_2.12:1.2.1-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2\")
.\\\n",
+ " config(\"spark.jars.packages\",
\"org.apache.sedona:sedona-spark-shaded-3.0_2.12:1.4.0,org.datasyslab:geotools-wrapper:1.4.0-28.2\")
.\\\n",
" getOrCreate()\n",
"\n",
"SedonaRegistrator.registerAll(spark)\n",
diff --git a/binder/ApacheSedonaSQL.ipynb b/binder/ApacheSedonaSQL.ipynb
index 6443e0a3..f5ffd914 100644
--- a/binder/ApacheSedonaSQL.ipynb
+++ b/binder/ApacheSedonaSQL.ipynb
@@ -121,8 +121,7 @@
" appName('appName'). \\\n",
" config(\"spark.serializer\", KryoSerializer.getName). \\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName).
\\\n",
- " config('spark.jars.packages',\n",
- "
'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,org.datasyslab:geotools-wrapper:1.1.0-25.2').
\\\n",
+ " config(\"spark.jars.packages\",
\"org.apache.sedona:sedona-spark-shaded-3.0_2.12:1.4.0,org.datasyslab:geotools-wrapper:1.4.0-28.2\")
.\\\n",
" getOrCreate()"
]
},
diff --git a/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
b/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
index 0b6f73b7..119d0c70 100644
--- a/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
+++ b/binder/ApacheSedonaSQL_SpatialJoin_AirportsPerCountry.ipynb
@@ -125,9 +125,7 @@
" appName(\"SedonaSQL-Example\").\\\n",
" config(\"spark.serializer\", KryoSerializer.getName).\\\n",
" config(\"spark.kryo.registrator\", SedonaKryoRegistrator.getName)
.\\\n",
- " config('spark.jars.packages',\n",
- "
'org.apache.sedona:sedona-python-adapter-3.0_2.12:1.1.0-incubating,'\n",
- " 'org.datasyslab:geotools-wrapper:1.1.0-25.2'). \\\n",
+ " config(\"spark.jars.packages\",
\"org.apache.sedona:sedona-spark-shaded-3.0_2.12:1.4.0,org.datasyslab:geotools-wrapper:1.4.0-28.2\")
.\\\n",
" getOrCreate()\n",
"SedonaRegistrator.registerAll(spark)\n",
"sc = spark.sparkContext\n",
@@ -530,7 +528,7 @@
},
{
"data": {
- "image/png":
"iVBORw0KGgoAAAANSUhEUgAAAYwAAACXCAYAAAAGX2ymAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABIAElEQVR4nO29eZgsaVng+3sjIpdasvY6+3769N50033obnYQQUDZBgTUUeZeZhgdvep4x7mg4+ijjpfxGa8Pc8dHBYcrOgyIKIjIsMpio9ALvTe9nO4+fZY++zlVp9ZcIt77x/tFLlWZWVlrZtaJ3/PkU5mRGRFfRkV+7/fuoqokJCQkJCQshdfuASQkJCQkdAeJwEhISEhIaIlEYCQkJCQktEQiMBISEhISWiIRGAkJCQkJLRG0ewDVjI2N6b59+9o9jISEhIQ157777juvquP13rtKRGdXcexT8CVVff0qDtES
[...]
+ "image/png":
"iVBORw0KGgoAAAANSUhEUgAAAYwAAACXCAYAAAAGX2ymAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABIAElEQVR4nO29eZgsaVng+3sjIpdasvY6+3769N50033obnYQQUDZBgTUUeZeZhgdvep4x7mg4+ijjpfxGa8Pc8dHBYcrOgyIKIjIsMpio9ALvTe9nO4+fZY++zlVp9ZcIt77x/tFLlWZWVlrZtaJ3/PkU5mRGRFfRkV+7/fuoqokJCQkJCQshdfuASQkJCQkdAeJwEhISEhIaIlEYCQkJCQktEQiMBISEhISWiIRGAkJCQkJLRG0ewDVjI2N6b59+9o9jISEhIQ157777juvquP13rtKRGdXcexT8CVVff0qDtES
[...]
"text/plain": [
"<Figure size 432x288 with 2 Axes>"
]
diff --git a/binder/Pipfile b/binder/Pipfile
index 47b80f4c..94b28405 100644
--- a/binder/Pipfile
+++ b/binder/Pipfile
@@ -12,11 +12,11 @@ pytest-cov = "*"
[packages]
pandas="*"
-geopandas="==0.6.0"
+geopandas="==0.10.2"
pyspark="==3.1.2"
attrs="*"
ipykernel = "*"
-apache-sedona="==1.1.0"
+apache-sedona="==1.4.0"
matplotlib = "*"
descartes = "*"
diff --git a/binder/postBuild b/binder/postBuild
index 95949698..88f44ad8 100644
--- a/binder/postBuild
+++ b/binder/postBuild
@@ -1,6 +1,6 @@
#Download Apache Spark
-wget
https://archive.apache.org/dist/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz
-tar -xzf spark-3.1.2-bin-hadoop3.2.tgz
+wget
https://archive.apache.org/dist/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz
+tar -xzf spark-3.3.2-bin-hadoop3.tgz
#Tidy up
-rm spark-3.1.2-bin-hadoop3.2.tgz
\ No newline at end of file
+rm spark-3.3.2-bin-hadoop3.tgz
\ No newline at end of file
diff --git a/binder/start b/binder/start
index 59e5ccff..fe679bda 100644
--- a/binder/start
+++ b/binder/start
@@ -1,6 +1,6 @@
#!/bin/bash
-SPARK_HOME=$HOME/spark-3.1.2-bin-hadoop3.2
+SPARK_HOME=$HOME/spark-3.3.2-bin-hadoop3
export PATH=$SPARK_HOME/bin:$PATH
export PYTHONPATH=$SPARK_HOME/python:$PYTHONPATH
export PYSPARK_SUBMIT_ARGS="--master local[*] pyspark-shell"
diff --git a/docs/api/sql/Optimizer.md b/docs/api/sql/Optimizer.md
index 2019d51a..025b964b 100644
--- a/docs/api/sql/Optimizer.md
+++ b/docs/api/sql/Optimizer.md
@@ -43,6 +43,7 @@ RangeJoin polygonshape#20: geometry, pointshape#43: geometry,
false
All join queries in SedonaSQL are inner joins
## Distance join
+
Introduction: Find geometries from A and geometries from B such that the
internal Euclidean distance of each geometry pair is less or equal than a
certain distance
Spark SQL Example:
@@ -72,7 +73,7 @@ DistanceJoin pointshape1#12: geometry, pointshape2#33:
geometry, 2.0, true
```
!!!warning
- Sedona doesn't control the distance's unit (degree or meter). It is
same with the geometry. To change the geometry's unit, please transform the
coordinate reference system. See [ST_Transform](Function.md#st_transform).
+ Sedona doesn't control the distance's unit (degree or meter). It is
same with the geometry. If your coordinates are in the longitude and latitude
system, the unit of `distance` should be degree instead of meter or mile. To
change the geometry's unit, please either transform the coordinate reference
system to a meter-based system. See [ST_Transform](Function.md#st_transform).
If you don't want to transform your data and are ok with sacrificing the query
accuracy, you can use an approxima [...]
## Broadcast index join
@@ -127,9 +128,9 @@ Note: If the distance is an expression, it is only
evaluated on the first argume
When one table involved a spatial join query is smaller than a threadhold,
Sedona will automatically choose broadcast index join instead of Sedona
optimized join. The current threshold is controlled by
[sedona.join.autoBroadcastJoinThreshold](../Parameter) and set to the same as
`spark.sql.autoBroadcastJoinThreshold`.
-## Google S2 based equi-join
+## Google S2 based approximate equi-join
-If the performance of Sedona optimized join is not ideal, which is possibly
caused by complicated and overlapping geometries, you can resort to Sedona
built-in Google S2-based equi-join. This equi-join leverages Spark's internal
equi-join algorithm and might be performant in some cases given that the
refinement step is optional.
+If the performance of Sedona optimized join is not ideal, which is possibly
caused by complicated and overlapping geometries, you can resort to Sedona
built-in Google S2-based approximate equi-join. This equi-join leverages
Spark's internal equi-join algorithm and might be performant given that you can
opt to skip the refinement step by sacrificing query accuracy.
Please use the following steps:
@@ -161,14 +162,16 @@ FROM lcs JOIN rcs ON lcs.cellId = rcs.cellId
Due to the nature of S2 Cellid, the equi-join results might have a few
false-positives depending on the S2 level you choose. A smaller level indicates
bigger cells, less exploded rows, but more false positives.
-To ensure the correctness, you can use [Spatial Predicate](../Predicate/) to
filter out them.
+To ensure the correctness, you can use one of the [Spatial
Predicates](../Predicate/) to filter out them. Use this query instead of the
query in Step 2.
```sql
-SELECT *
-FROM joinresult
-WHERE ST_Contains(lcs.geom, rcs.geom)
+SELECT lcs.id as lcs_id, lcs.geom as lcs_geom, lcs.name as lcs_name, rcs.id as
rcs_id, rcs.geom as rcs_geom, rcs.name as rcs_name
+FROM lcs, rcs
+WHERE lcs.cellId = rcs.cellId AND ST_Contains(lcs.geom, rcs.geom)
```
+As you see, compared to the query in Step 2, we added one more filter, which
is `ST_Contains`, to remove false positives. You can also use `ST_Intersects`
and so on.
+
!!!tip
You can skip this step if you don't need 100% accuracy and want faster
query speed.
@@ -195,6 +198,18 @@ GROUP BY (lcs_geom, rcs_geom)
!!!note
If you are doing point-in-polygon join, this is not a problem and you
can safely discard this issue. This issue only happens when you do
polygon-polygon, polygon-linestring, linestring-linestring join.
+### S2 for distance join
+
+This also works for distance join. You first need to use `ST_Buffer(geometry,
distance)` to wrap one of your original geometry column. If your original
geometry column contains points, this `ST_Buffer` will make them become circles
with a radius of `distance`.
+
+For example. run this query first on the left table before Step 1.
+
+```sql
+SELECT id, ST_Buffer(geom, DISTANCE), name
+FROM lefts
+```
+
+Since the coordinates are in the longitude and latitude system, so the unit of
`distance` should be degree instead of meter or mile. You will have to estimate
the corresponding degrees based on your meter values. Please use [this
calculator](https://lucidar.me/en/online-unit-converter-length-to-angle/convert-degrees-to-meters/#online-converter).
## Regular spatial predicate pushdown
Introduction: Given a join query and a predicate in the same WHERE clause,
first executes the Predicate as a filter, then executes the join query.
diff --git a/docs/community/publish.md b/docs/community/publish.md
index fb3eae8c..dfdb5404 100644
--- a/docs/community/publish.md
+++ b/docs/community/publish.md
@@ -413,6 +413,8 @@ rm *.asc
You must have the maintainer privilege of
`https://pypi.org/project/apache-sedona/` and
`https://www.npmjs.com/package/apache-sedona`
+To publish Sedona pythons, you have to use GitHub actions since we release
wheels for different platforms. Please use this repo:
https://github.com/jiayuasu/sedona-publish-python
+
```bash
#!/bin/bash
@@ -423,7 +425,6 @@ cp -r sedona-{{ sedona_create_release.current_git_tag}}/*
apache-sedona-{{ sedon
rm -rf sedona-{{ sedona_create_release.current_git_tag}}
-cd apache-sedona-{{ sedona_create_release.current_version }}-src/python &&
python3 setup.py sdist bdist_wheel && twine upload dist/* && cd ..
cd zeppelin && npm publish && cd ..
rm -rf apache-sedona-{{ sedona_create_release.current_version }}-src
```
diff --git a/docs/download.md b/docs/download.md
index 776e5c18..9fc26e7d 100644
--- a/docs/download.md
+++ b/docs/download.md
@@ -6,7 +6,7 @@ Old GeoSpark releases: [GitHub
releases](https://github.com/apache/sedona/releas
Automatically generated binary JARs (per each Master branch commit): [GitHub
Action](https://github.com/apache/sedona/actions/workflows/java.yml)
-## Verify the integrity
+## Verify the integ1.4.0rity
[Public keys](https://downloads.apache.org/sedona/KEYS)
@@ -14,19 +14,19 @@ Automatically generated binary JARs (per each Master branch
commit): [GitHub Act
## Versions
-### 1.3.1-incubating
+### 1.4.0
| | Download from ASF | Checksum | Signature |
|:-----------------:|:--------:|:--------:|:---------:|
-| Source code |
[src](https://www.apache.org/dyn/closer.lua/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-src.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-src.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-src.tar.gz.asc)
|
-| Binary |
[bin](https://www.apache.org/dyn/closer.lua/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-bin.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-bin.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-bin.tar.gz.asc)
+| Source code |
[src](https://www.apache.org/dyn/closer.lua/sedona/1.4.0/apache-sedona-1.4.0-src.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.4.0/apache-sedona-1.4.0-src.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.4.0/apache-sedona-1.4.0-src.tar.gz.asc)
|
+| Binary |
[bin](https://www.apache.org/dyn/closer.lua/sedona/1.4.0/apache-sedona-1.4.0-bin.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.4.0/apache-sedona-1.4.0-bin.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.4.0/apache-sedona-1.4.0-bin.tar.gz.asc)
-### 1.2.1-incubating
+### 1.3.1-incubating
| | Download from ASF | Checksum | Signature |
|:-----------------:|:--------:|:--------:|:---------:|
-| Source code |
[src](https://www.apache.org/dyn/closer.lua/sedona/1.2.1-incubating/apache-sedona-1.2.1-incubating-src.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.2.1-incubating/apache-sedona-1.2.1-incubating-src.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.2.1-incubating/apache-sedona-1.2.1-incubating-src.tar.gz.asc)
|
-| Binary |
[bin](https://www.apache.org/dyn/closer.lua/sedona/1.2.1-incubating/apache-sedona-1.2.1-incubating-bin.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.2.1-incubating/apache-sedona-1.2.1-incubating-bin.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.2.1-incubating/apache-sedona-1.2.1-incubating-bin.tar.gz.asc)
+| Source code |
[src](https://www.apache.org/dyn/closer.lua/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-src.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-src.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-src.tar.gz.asc)
|
+| Binary |
[bin](https://www.apache.org/dyn/closer.lua/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-bin.tar.gz)
|
[sha512](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-bin.tar.gz.sha512)
|
[asc](https://downloads.apache.org/sedona/1.3.1-incubating/apache-sedona-1.3.1-incubating-bin.tar.gz.asc)
### Past releases
diff --git a/docs/index.md b/docs/index.md
index 495f5a05..5b940a44 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,6 +1,5 @@
+### 03/19/2023: Sedona 1.4.0 released. It provides GeoParquet filter pushdown
(10X less memory footprint), faster serialization (3X speed), S2-based fast
approximate join and enhanced R language support
### 01/2023: Apache Sedona graduated to an Apache Top Level Project!
### 12/23/2022: Sedona 1.3.1-incubating is released. It adds native support of
GeoParquet, DataFrame style API, Scala 2.13, Python 3.10, spatial aggregation
on Flink. Please check Sedona release notes.
### 08/30/2022: Sedona 1.2.1-incubating is released. It supports Spark 2.4 -
3.3. and Flink 1.12+.
-### 04/16/2022: Sedona 1.2.0-incubating is released. Sedona now supports
geospatial stream processing in Apache Flink.
-### 11/23/2021: Sedona 1.1.1-incubating is released. It now supports Spark 3.2.
-### 10/06/2021: Sedona 1.1.0-incubating is released. R lang API is available
on CRAN. Raster data and map algebra SQL functions are now supported.
\ No newline at end of file
+### 04/16/2022: Sedona 1.2.0-incubating is released. Sedona now supports
geospatial stream processing in Apache Flink.
\ No newline at end of file
diff --git a/docs/setup/release-notes.md b/docs/setup/release-notes.md
index 99179c5a..8e6eb3f0 100644
--- a/docs/setup/release-notes.md
+++ b/docs/setup/release-notes.md
@@ -1,14 +1,18 @@
!!!warning
Support of Spark 2.X and Scala 2.11 was removed in Sedona 1.3.0+
although some parts of the source code might still be compatible. Sedona 1.3.0+
releases binary for both Scala 2.12 and 2.13.
+!!!danger
+ Sedona Python currently only works with Shapely 1.x. If you use
GeoPandas, please use <= GeoPandas `0.11.1`. GeoPandas > 0.11.1 will
automatically installe Shapely 2.0. If you use Shapely, please use <= `1.8.4`.
+
## Sedona 1.4.0
Sedona 1.4.0 is compiled against, Spark 3.3 / Flink 1.12, Java 8.
### Highlights
-* [X] **Sedona Spark** Pushdown spatial predicate on GeoParquet to reduce
memory consumption by 10X: see
[explanation](../../api/sql/Optimizer/#geoparquet)
* [X] **Sedona Spark & Flink** Serialize and deserialize geometries 3 - 7X
faster
+* [X] **Sedona Spark & Flink** Google S2 based spatial join for fast
approximate point-in-polygon join. See [Join query in
Spark](../../api/sql/Optimizer/#google-s2-based-approximate-equi-join) and
[Join query in Flink](../../tutorial/flink/sql/#join-query)
+* [X] **Sedona Spark** Pushdown spatial predicate on GeoParquet to reduce
memory consumption by 10X: see
[explanation](../../api/sql/Optimizer/#geoparquet)
* [X] **Sedona Spark** Automatically use broadcast index spatial join for
small datasets
* [X] **Sedona Spark** New RasterUDT added to Sedona GeoTiff reader.
* [X] **Sedona Spark** A number of bug fixes and improvement to the Sedona R
module.
diff --git a/docs/tutorial/demo.md b/docs/tutorial/demo.md
index c710216e..d2102e36 100644
--- a/docs/tutorial/demo.md
+++ b/docs/tutorial/demo.md
@@ -1,15 +1,16 @@
# Scala and Java Examples
-[Scala and Java
Examples](https://github.com/apache/sedona/tree/master/examples) contains
template projects for RDD, SQL and Viz. The template projects have been
configured properly.
+[Scala and Java
Examples](https://github.com/apache/sedona/tree/master/examples) contains
template projects for Sedona Spark (RDD, SQL and Viz) and Sedona Flink. The
template projects have been configured properly.
Note that, although the template projects are written in Scala, the same APIs
can be used in Java as well.
## Folder structure
The folder structure of this repository is as follows.
-* rdd-colocation-mining: a scala template shows how to use Sedona RDD API in
Spatial Data Mining
-* sql: a scala template shows how to use Sedona DataFrame and SQL API
-* viz: a scala template shows how to use Sedona Viz RDD and SQL API
+* spark-rdd-colocation-mining: a scala template shows how to use Sedona RDD
API in Spatial Data Mining in Apache Spark
+* spark-sql: a scala template shows how to use Sedona DataFrame and SQL API in
Apache Spark
+* spark-viz: a scala template shows how to use Sedona Viz RDD and SQL API in
Apache Spark
+* flink-sql: a Java template shows how to use Sedona SQL in Apache Flink
## Compile and package
@@ -24,19 +25,23 @@ Please make sure you have the following software installed
on your local machine
Run a terminal command `sbt assembly` within the folder of each template
-### Submit your fat jar to Spark
-After running the command mentioned above, you are able to see a fat jar in
`./target` folder. Please take it and use `./bin/spark-submit` to submit this
jar.
+### Submit your fat jar to Spark or Flink
+After running the command mentioned above, you are able to see a fat jar in
`./target` folder. Please take it and use `./bin/spark-submit` or
`/bin/flink`to submit this jar.
To run the jar in this way, you need to:
-* Either change Spark Master Address in template projects or simply delete it.
Currently, they are hard coded to `local[*]` which means run locally with all
cores.
+* For Spark: either change Spark Master Address in template projects or simply
delete it. Currently, they are hard coded to `local[*]` which means run locally
with all cores.
* Change the dependency packaging scope of Apache Spark from "compile" to
"provided". This is a common packaging strategy in Maven and SBT which means do
not package Spark into your fat jar. Otherwise, this may lead to a huge jar and
version conflicts!
-* Make sure the dependency versions in build.sbt are consistent with your
Spark version.
+* Make sure the dependency versions in build.sbt/pom.xml are consistent with
your Spark/Flink version.
## Run template projects locally
+
We highly suggest you use IDEs to run template projects on your local machine.
For Scala, we recommend IntelliJ IDEA with Scala plug-in. For Java, we
recommend IntelliJ IDEA and Eclipse. With the help of IDEs, **you don't have to
prepare anything** (even don't need to download and set up Spark!). As long as
you have Scala and Java, everything works properly!
### Scala
-Import the Scala template project as SBT project. Then run the Main file in
this project.
\ No newline at end of file
+Import the Scala template project as SBT project. Then run the Main file in
this project.
+
+### Java
+Import the Java template project as Maven project. Then run the Main file in
this project.
\ No newline at end of file
diff --git a/docs/tutorial/flink/sql.md b/docs/tutorial/flink/sql.md
index facdfaec..720d1aca 100644
--- a/docs/tutorial/flink/sql.md
+++ b/docs/tutorial/flink/sql.md
@@ -12,11 +12,12 @@ Detailed SedonaSQL APIs are available here: [SedonaSQL
API](../../../api/flink/O
1. Read [Sedona Maven Central coordinates](../../../setup/maven-coordinates)
2. Add Sedona dependencies in build.sbt or pom.xml.
3. Add [Flink
dependencies](https://nightlies.apache.org/flink/flink-docs-master/docs/dev/configuration/overview/)
in build.sbt or pom.xml.
+4. Please see [SQL example project](../../demo/)
## Initiate Stream Environment
Use the following code to initiate your `StreamExecutionEnvironment` at the
beginning:
```java
-StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment()
+StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment();
EnvironmentSettings settings =
EnvironmentSettings.newInstance().inStreamingMode().build();
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env, settings);
```
@@ -63,7 +64,7 @@ You can create a Table with a Geometry type column as follows:
```java
tableEnv.createTemporaryView("myTable", tbl)
-Table geomTbl = tableEnv.sql("SELECT ST_GeomFromWKT(geom_polygon) as
geom_polygon, name_polygon FROM myTable")
+Table geomTbl = tableEnv.sqlQuery("SELECT ST_GeomFromWKT(geom_polygon) as
geom_polygon, name_polygon FROM myTable")
geomTbl.execute().print()
```
@@ -166,12 +167,9 @@ After the transformation:
+----+--------------------------------+--------------------------------+
```
-
-## Run spatial queries
-
After creating a Geometry type column, you are able to run spatial queries.
-### Range query
+## Range query
Use ==ST_Contains==, ==ST_Intersects== and so on to run a range query over a
single column.
@@ -190,7 +188,7 @@ geomTable.execute().print()
!!!note
Read [SedonaSQL Predicate API](../../../api/flink/Predicate) to learn
different spatial query predicates.
-### KNN query
+## KNN query
Use ==ST_Distance== to calculate the distance and rank the distance.
@@ -207,6 +205,102 @@ geomTable = tableEnv.sqlQuery(
geomTable.execute().print()
```
+## Join query
+
+This equi-join leverages Flink's internal equi-join algorithm. You can opt to
skip the Sedona refinement step by sacrificing query accuracy. A running
example is in [SQL example project](../../demo/).
+
+Please use the following steps:
+
+### 1. Generate S2 ids for both tables
+
+Use [ST_S2CellIds](../../../api/flink/Function/#st_s2cellids) to generate cell
IDs. Each geometry may produce one or more IDs.
+
+```sql
+SELECT id, geom, name, ST_S2CellIDs(geom, 15) as idarray
+FROM lefts
+```
+
+```sql
+SELECT id, geom, name, ST_S2CellIDs(geom, 15) as idarray
+FROM rights
+```
+
+### 2. Explode id array
+
+The produced S2 ids are arrays of integers. We need to explode these Ids to
multiple rows so later we can join two tables by ids.
+
+```
+SELECT id, geom, name, cellId
+FROM lefts CROSS JOIN UNNEST(lefts.idarray) AS tmpTbl1(cellId)
+```
+
+```
+SELECT id, geom, name, cellId
+FROM rights CROSS JOIN UNNEST(rights.idarray) AS tmpTbl2(cellId)
+```
+
+### 3. Perform equi-join
+
+Join the two tables by their S2 cellId
+
+```sql
+SELECT lcs.id as lcs_id, lcs.geom as lcs_geom, lcs.name as lcs_name, rcs.id as
rcs_id, rcs.geom as rcs_geom, rcs.name as rcs_name
+FROM lcs JOIN rcs ON lcs.cellId = rcs.cellId
+```
+
+### 4. Optional: Refine the result
+
+Due to the nature of S2 Cellid, the equi-join results might have a few
false-positives depending on the S2 level you choose. A smaller level indicates
bigger cells, less exploded rows, but more false positives.
+
+To ensure the correctness, you can use one of the [Spatial
Predicates](../../../api/Predicate/) to filter out them. Use this query as the
query in Step 3.
+
+```sql
+SELECT lcs.id as lcs_id, lcs.geom as lcs_geom, lcs.name as lcs_name, rcs.id as
rcs_id, rcs.geom as rcs_geom, rcs.name as rcs_name
+FROM lcs, rcs
+WHERE lcs.cellId = rcs.cellId AND ST_Contains(lcs.geom, rcs.geom)
+```
+
+As you see, compared to the query in Step 2, we added one more filter, which
is `ST_Contains`, to remove false positives. You can also use `ST_Intersects`
and so on.
+
+!!!tip
+ You can skip this step if you don't need 100% accuracy and want faster
query speed.
+
+### 5. Optional: De-duplcate
+
+Due to the explode function used when we generate S2 Cell Ids, the resulting
DataFrame may have several duplicate <lcs_geom, rcs_geom> matches. You can
remove them by performing a GroupBy query.
+
+```sql
+SELECT lcs_id, rcs_id , FIRST_VALUE(lcs_geom), FIRST_VALUE(lcs_name),
first(rcs_geom), first(rcs_name)
+FROM joinresult
+GROUP BY (lcs_id, rcs_id)
+```
+
+The `FIRST_VALUE` function is to take the first value from a number of
duplicate values.
+
+If you don't have a unique id for each geometry, you can also group by
geometry itself. See below:
+
+```sql
+SELECT lcs_geom, rcs_geom, first(lcs_name), first(rcs_name)
+FROM joinresult
+GROUP BY (lcs_geom, rcs_geom)
+```
+
+!!!note
+ If you are doing point-in-polygon join, this is not a problem and you
can safely discard this issue. This issue only happens when you do
polygon-polygon, polygon-linestring, linestring-linestring join.
+
+### S2 for distance join
+
+This also works for distance join. You first need to use `ST_Buffer(geometry,
distance)` to wrap one of your original geometry column. If your original
geometry column contains points, this `ST_Buffer` will make them become circles
with a radius of `distance`.
+
+For example. run this query first on the left table before Step 1.
+
+```sql
+SELECT id, ST_Buffer(geom, DISTANCE), name
+FROM lefts
+```
+
+Since the coordinates are in the longitude and latitude system, so the unit of
`distance` should be degree instead of meter or mile. You will have to estimate
the corresponding degrees based on your meter values. Please use [this
calculator](https://lucidar.me/en/online-unit-converter-length-to-angle/convert-degrees-to-meters/#online-converter).
+
## Convert Spatial Table to Spatial DataStream
### Get DataStream
diff --git a/docs/tutorial/geopandas-shapely.md
b/docs/tutorial/geopandas-shapely.md
index 4b1251fa..96f9774b 100644
--- a/docs/tutorial/geopandas-shapely.md
+++ b/docs/tutorial/geopandas-shapely.md
@@ -1,5 +1,8 @@
# Work with GeoPandas and Shapely
+!!!danger
+ Sedona Python currently only works with Shapely 1.x. If you use
GeoPandas, please use <= GeoPandas `0.11.1`. GeoPandas > 0.11.1 will
automatically installe Shapely 2.0. If you use Shapely, please use <= `1.8.4`.
+
## Interoperate with GeoPandas
Sedona Python has implemented serializers and deserializers which allows to
convert Sedona Geometry objects into Shapely BaseGeometry objects. Based on
that it is possible to load the data with geopandas from file (look at Fiona
possible drivers) and create Spark DataFrame based on GeoDataFrame object.
diff --git a/docs/tutorial/rdd.md b/docs/tutorial/rdd.md
index 99880f38..077415dc 100644
--- a/docs/tutorial/rdd.md
+++ b/docs/tutorial/rdd.md
@@ -928,6 +928,8 @@ The index should be built on either one of two SpatialRDDs.
In general, you shou
A distance join query takes as input two Spatial RDD A and B and a distance.
For each geometry in A, finds the geometries (from B) are within the given
distance to it. A and B can be any geometry type and are not necessary to have
the same geometry type. The unit of the distance is explained
[here](#transform-the-coordinate-reference-system).
+If you don't want to transform your data and are ok with sacrificing the query
accuracy, you can use an approximate degree value for distance. Please use
[this
calculator](https://lucidar.me/en/online-unit-converter-length-to-angle/convert-degrees-to-meters/#online-converter).
+
Assume you now have two SpatialRDDs (typed or generic). You can use the
following code to issue an Distance Join Query on them.
=== "Scala"
diff --git a/examples/sql/.gitignore b/examples/flink-sql/.gitignore
similarity index 100%
copy from examples/sql/.gitignore
copy to examples/flink-sql/.gitignore
diff --git a/examples/flink-sql/pom.xml b/examples/flink-sql/pom.xml
new file mode 100644
index 00000000..75b8b4c3
--- /dev/null
+++ b/examples/flink-sql/pom.xml
@@ -0,0 +1,222 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-flink-example</artifactId>
+ <version>1.0.0</version>
+
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <geotools.version>1.4.0-28.2</geotools.version>
+ <geotools.scope>compile</geotools.scope>
+ <scala.compat.version>2.12</scala.compat.version>
+ <sedona.version>1.4.0</sedona.version>
+ <flink.version>1.14.3</flink.version>
+ <flink.scope>compile</flink.scope>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+
<artifactId>sedona-flink-shaded_${scala.compat.version}</artifactId>
+ <version>${sedona.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.datasyslab</groupId>
+ <artifactId>geotools-wrapper</artifactId>
+ <version>${geotools.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-flink_${scala.compat.version}</artifactId>
+ <version>${sedona.version}</version>
+ <scope>${sedona.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-core</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <!-- For Flink DataStream API-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+
<artifactId>flink-streaming-java_${scala.compat.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <!-- Flink Kafka connector-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+
<artifactId>flink-connector-kafka_${scala.compat.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <!-- For playing flink in IDE-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-clients_${scala.compat.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <!-- For Flink flink api, planner, udf/udt, csv-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+
<artifactId>flink-table-api-java-bridge_${scala.compat.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <!-- Starting Flink 14, Blink planner has been renamed to the
official Flink planner-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+
<artifactId>flink-table-planner_${scala.compat.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-table-common</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-csv</artifactId>
+ <version>${flink.version}</version>
+ <scope>${flink.scope}</scope>
+ </dependency>
+ <!-- For Flink Web Ui in test-->
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-runtime-web_${scala.compat.version}</artifactId>
+ <version>${flink.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <repositories>
+ <repository>
+ <id>maven2-repository.dev.java.net</id>
+ <name>Java.net repository</name>
+ <url>https://download.java.net/maven/2</url>
+ </repository>
+ <repository>
+ <id>osgeo</id>
+ <name>OSGeo Release Repository</name>
+ <url>https://repo.osgeo.org/repository/release/</url>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ </repository>
+ </repositories>
+ <build>
+ <sourceDirectory>src/main/java</sourceDirectory>
+ <plugins>
+ <plugin>
+ <!-- Build an executable JAR -->
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.3.0</version>
+ <configuration>
+ <archive>
+ <manifest>
+ <!-- <addClasspath>true</addClasspath> -->
+ <mainClass>FlinkExample</mainClass>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <!-- use transformer to handle merge of
META-INF/services - see
http://java.net/jira/browse/JERSEY-440?focusedCommentId=14822&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#action_14822
-->
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ <transformer
+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <manifestEntries>
+ <Specification-Title>Java Advanced
Imaging Image I/O Tools</Specification-Title>
+
<Specification-Version>1.1</Specification-Version>
+ <Specification-Vendor>Sun
Microsystems, Inc.</Specification-Vendor>
+
<Implementation-Title>com.sun.media.imageio</Implementation-Title>
+
<Implementation-Version>1.1</Implementation-Version>
+ <Implementation-Vendor>Sun
Microsystems, Inc.</Implementation-Vendor>
+
<Extension-Name>com.sun.media.imageio</Extension-Name>
+ </manifestEntries>
+ </transformer>
+ </transformers>
+ <filters>
+ <!-- filter to address "Invalid signature
file" issue - see http://stackoverflow.com/a/6743609/589215-->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.jacoco</groupId>
+ <artifactId>jacoco-maven-plugin</artifactId>
+ <version>0.8.7</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>prepare-agent</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>report</id>
+ <phase>test</phase>
+ <goals>
+ <goal>report</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <resources>
+ <resource>
+ <directory>src/test/resources/</directory>
+ </resource>
+ </resources>
+ </build>
+</project>
+
diff --git a/examples/flink-sql/src/main/java/FlinkExample.java
b/examples/flink-sql/src/main/java/FlinkExample.java
new file mode 100644
index 00000000..d39c34b7
--- /dev/null
+++ b/examples/flink-sql/src/main/java/FlinkExample.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.sedona.flink.SedonaFlinkRegistrator;
+import org.apache.sedona.flink.expressions.Constructors;
+
+import static org.apache.flink.table.api.Expressions.$;
+import static org.apache.flink.table.api.Expressions.call;
+
+public class FlinkExample
+{
+ static String[] pointColNames = {"geom_point", "name_point", "event_time",
"proc_time"};
+
+ static String[] polygonColNames = {"geom_polygon", "name_polygon",
"event_time", "proc_time"};
+
+ public static void main(String[] args) {
+ int testDataSize = 10;
+ StreamExecutionEnvironment env =
StreamExecutionEnvironment.getExecutionEnvironment();
+ EnvironmentSettings settings =
EnvironmentSettings.newInstance().inStreamingMode().build();
+ StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env,
settings);
+ SedonaFlinkRegistrator.registerType(env);
+ SedonaFlinkRegistrator.registerFunc(tableEnv);
+
+ // Create a fake WKT string table source
+ Table pointWktTable = Utils.createTextTable(env, tableEnv,
Utils.createPointWKT(testDataSize), pointColNames);
+ // Create a geometry column
+ Table pointTable =
pointWktTable.select(call(Constructors.ST_GeomFromWKT.class.getSimpleName(),
+ $(pointColNames[0])).as(pointColNames[0]),
+ $(pointColNames[1]));
+ // Create S2CellID
+ pointTable = pointTable.select($(pointColNames[0]),
$(pointColNames[1]),
+ call("ST_S2CellIDs", $(pointColNames[0]), 6).as("s2id_array"));
+ // Explode s2id array
+ tableEnv.createTemporaryView("pointTable", pointTable);
+ pointTable = tableEnv.sqlQuery("SELECT geom_point, name_point,
s2id_point FROM pointTable CROSS JOIN UNNEST(pointTable.s2id_array) AS
tmpTbl1(s2id_point)");
+ pointTable.execute().print();
+
+
+ // Create a fake WKT string table source
+ Table polygonWktTable = Utils.createTextTable(env, tableEnv,
Utils.createPolygonWKT(testDataSize), polygonColNames);
+ // Create a geometry column
+ Table polygonTable =
polygonWktTable.select(call(Constructors.ST_GeomFromWKT.class.getSimpleName(),
+ $(polygonColNames[0])).as(polygonColNames[0]),
+ $(polygonColNames[1]));
+ // Create S2CellID
+ polygonTable = polygonTable.select($(polygonColNames[0]),
$(polygonColNames[1]),
+ call("ST_S2CellIDs", $(polygonColNames[0]),
6).as("s2id_array"));
+ // Explode s2id array
+ tableEnv.createTemporaryView("polygonTable", polygonTable);
+ polygonTable = tableEnv.sqlQuery("SELECT geom_polygon, name_polygon,
s2id_polygon FROM polygonTable CROSS JOIN UNNEST(polygonTable.s2id_array) AS
tmpTbl2(s2id_polygon)");
+ polygonTable.execute().print();
+
+ // Join two tables by their S2 ids
+ Table joinResult =
pointTable.join(polygonTable).where($("s2id_point").isEqual($("s2id_polygon")));
+ // Optional: remove false positives
+ joinResult = joinResult.where("ST_Contains(geom_polygon, geom_point)");
+ joinResult.execute().print();
+ }
+
+}
diff --git a/examples/flink-sql/src/main/java/Utils.java
b/examples/flink-sql/src/main/java/Utils.java
new file mode 100644
index 00000000..0a95ab1b
--- /dev/null
+++ b/examples/flink-sql/src/main/java/Utils.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.types.Row;
+
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.flink.table.api.Expressions.$;
+
+public class Utils
+{
+ static Long timestamp_base = new
Timestamp(System.currentTimeMillis()).getTime();
+ static Long time_interval = 1L; // Generate a record per this interval.
Unit is second
+
+ static List<Row> createPointText(int size){
+ List<Row> data = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ // Create a number of points (1, 1) (2, 2) ...
+ data.add(Row.of(i + "," + i, "point" + i, timestamp_base +
time_interval * 1000 * i));
+ }
+ return data;
+ }
+
+ static List<Row> createPolygonText(int size) {
+ List<Row> data = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ // Create polygons each of which only has 1 match in points
+ // Each polygon is an envelope like (-0.5, -0.5, 0.5, 0.5)
+ String minX = String.valueOf(i - 0.5);
+ String minY = String.valueOf(i - 0.5);
+ String maxX = String.valueOf(i + 0.5);
+ String maxY = String.valueOf(i + 0.5);
+ List<String> polygon = new ArrayList<>();
+ polygon.add(minX);polygon.add(minY);
+ polygon.add(minX);polygon.add(maxY);
+ polygon.add(maxX);polygon.add(maxY);
+ polygon.add(maxX);polygon.add(minY);
+ polygon.add(minX);polygon.add(minY);
+ data.add(Row.of(String.join(",", polygon), "polygon" + i,
timestamp_base + time_interval * 1000 * i));
+ }
+ return data;
+ }
+
+ static List<Row> createPointWKT(int size){
+ List<Row> data = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ // Create a number of points (1, 1) (2, 2) ...
+ data.add(Row.of("POINT (" + i + " " + i +")", "point" + i,
timestamp_base + time_interval * 1000 * i));
+ }
+ return data;
+ }
+
+ static List<Row> createPolygonWKT(int size) {
+ List<Row> data = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ // Create polygons each of which only has 1 match in points
+ // Each polygon is an envelope like (-0.5, -0.5, 0.5, 0.5)
+ String minX = String.valueOf(i - 0.5);
+ String minY = String.valueOf(i - 0.5);
+ String maxX = String.valueOf(i + 0.5);
+ String maxY = String.valueOf(i + 0.5);
+ List<String> polygon = new ArrayList<>();
+ polygon.add(minX + " " + minY);
+ polygon.add(minX + " " + maxY);
+ polygon.add(maxX + " " + maxY);
+ polygon.add(maxX + " " + minY);
+ polygon.add(minX + " " + minY);
+ data.add(Row.of("POLYGON ((" + String.join(", ", polygon) + "))",
"polygon" + i, timestamp_base + time_interval * 1000 * i));
+ }
+ return data;
+ }
+
+ static Table createTextTable(StreamExecutionEnvironment env,
StreamTableEnvironment tableEnv, List<Row> data, String[] colNames){
+ TypeInformation<?>[] colTypes = {
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO,
+ BasicTypeInfo.LONG_TYPE_INFO
+ };
+ RowTypeInfo typeInfo = new RowTypeInfo(colTypes,
Arrays.copyOfRange(colNames, 0, 3));
+ DataStream<Row> ds = env.fromCollection(data).returns(typeInfo);
+ // Generate Time Attribute
+ WatermarkStrategy<Row> wmStrategy =
+ WatermarkStrategy
+ .<Row>forMonotonousTimestamps()
+ .withTimestampAssigner((event, timestamp) ->
event.getFieldAs(2));
+ return
tableEnv.fromDataStream(ds.assignTimestampsAndWatermarks(wmStrategy),
$(colNames[0]), $(colNames[1]), $(colNames[2]).rowtime(),
$(colNames[3]).proctime());
+ }
+
+
+}
diff --git a/examples/sql/src/test/resources/.gitignore
b/examples/flink-sql/src/test/resources/.gitignore
similarity index 100%
copy from examples/sql/src/test/resources/.gitignore
copy to examples/flink-sql/src/test/resources/.gitignore
diff --git a/examples/sql/src/test/resources/raster/test1.tiff
b/examples/flink-sql/src/test/resources/raster/test1.tiff
similarity index 100%
copy from examples/sql/src/test/resources/raster/test1.tiff
copy to examples/flink-sql/src/test/resources/raster/test1.tiff
diff --git a/examples/sql/src/test/resources/raster/test2.tiff
b/examples/flink-sql/src/test/resources/raster/test2.tiff
similarity index 100%
copy from examples/sql/src/test/resources/raster/test2.tiff
copy to examples/flink-sql/src/test/resources/raster/test2.tiff
diff --git a/examples/sql/src/test/resources/scalastyle_config.xml
b/examples/flink-sql/src/test/resources/scalastyle_config.xml
similarity index 100%
copy from examples/sql/src/test/resources/scalastyle_config.xml
copy to examples/flink-sql/src/test/resources/scalastyle_config.xml
diff --git a/examples/sql/src/test/resources/shapefiles/dbf/map.dbf
b/examples/flink-sql/src/test/resources/shapefiles/dbf/map.dbf
similarity index 100%
copy from examples/sql/src/test/resources/shapefiles/dbf/map.dbf
copy to examples/flink-sql/src/test/resources/shapefiles/dbf/map.dbf
diff --git a/examples/sql/src/test/resources/shapefiles/dbf/map.shp
b/examples/flink-sql/src/test/resources/shapefiles/dbf/map.shp
similarity index 100%
copy from examples/sql/src/test/resources/shapefiles/dbf/map.shp
copy to examples/flink-sql/src/test/resources/shapefiles/dbf/map.shp
diff --git a/examples/sql/src/test/resources/shapefiles/dbf/map.shx
b/examples/flink-sql/src/test/resources/shapefiles/dbf/map.shx
similarity index 100%
copy from examples/sql/src/test/resources/shapefiles/dbf/map.shx
copy to examples/flink-sql/src/test/resources/shapefiles/dbf/map.shx
diff --git a/examples/sql/src/test/resources/testenvelope.csv
b/examples/flink-sql/src/test/resources/testenvelope.csv
similarity index 100%
copy from examples/sql/src/test/resources/testenvelope.csv
copy to examples/flink-sql/src/test/resources/testenvelope.csv
diff --git a/examples/sql/src/test/resources/testpoint.csv
b/examples/flink-sql/src/test/resources/testpoint.csv
similarity index 100%
copy from examples/sql/src/test/resources/testpoint.csv
copy to examples/flink-sql/src/test/resources/testpoint.csv
diff --git a/examples/rdd-colocation-mining/.gitignore
b/examples/spark-rdd-colocation-mining/.gitignore
similarity index 100%
rename from examples/rdd-colocation-mining/.gitignore
rename to examples/spark-rdd-colocation-mining/.gitignore
diff --git a/examples/rdd-colocation-mining/README.md
b/examples/spark-rdd-colocation-mining/README.md
similarity index 100%
rename from examples/rdd-colocation-mining/README.md
rename to examples/spark-rdd-colocation-mining/README.md
diff --git a/examples/rdd-colocation-mining/build.sbt
b/examples/spark-rdd-colocation-mining/build.sbt
similarity index 93%
rename from examples/rdd-colocation-mining/build.sbt
rename to examples/spark-rdd-colocation-mining/build.sbt
index c6d134e4..0b570035 100644
--- a/examples/rdd-colocation-mining/build.sbt
+++ b/examples/spark-rdd-colocation-mining/build.sbt
@@ -39,14 +39,14 @@ val SparkCompatibleVersion = "3.0"
val HadoopVersion = "2.7.2"
-val SedonaVersion = "1.3.1-incubating"
+val SedonaVersion = "1.4.0"
val ScalaCompatibleVersion = "2.12"
// Change the dependency scope to "provided" when you run "sbt assembly"
val dependencyScope = "compile"
-val geotoolsVersion = "1.3.0-27.2"
+val geotoolsVersion = "1.4.0-28.2"
//val jacksonVersion = "2.10.0"
@@ -62,7 +62,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % HadoopVersion %
dependencyScope,
"org.apache.hadoop" % "hadoop-common" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-hdfs" % HadoopVersion % dependencyScope,
- "org.apache.sedona" %
"sedona-python-adapter-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
+ "org.apache.sedona" %
"sedona-spark-shaded-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
"org.apache.sedona" %
"sedona-viz-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
// The following GeoTools packages are only required if you need CRS
transformation. Under GNU Lesser General Public License (LGPL) license
"org.datasyslab" % "geotools-wrapper" % geotoolsVersion % "compile"
diff --git a/examples/rdd-colocation-mining/colocationMap.png
b/examples/spark-rdd-colocation-mining/colocationMap.png
similarity index 100%
rename from examples/rdd-colocation-mining/colocationMap.png
rename to examples/spark-rdd-colocation-mining/colocationMap.png
diff --git a/examples/rdd-colocation-mining/project/.gitignore
b/examples/spark-rdd-colocation-mining/project/.gitignore
similarity index 100%
rename from examples/rdd-colocation-mining/project/.gitignore
rename to examples/spark-rdd-colocation-mining/project/.gitignore
diff --git a/examples/rdd-colocation-mining/project/build.properties
b/examples/spark-rdd-colocation-mining/project/build.properties
similarity index 100%
rename from examples/rdd-colocation-mining/project/build.properties
rename to examples/spark-rdd-colocation-mining/project/build.properties
diff --git a/examples/rdd-colocation-mining/project/plugins.sbt
b/examples/spark-rdd-colocation-mining/project/plugins.sbt
similarity index 100%
rename from examples/rdd-colocation-mining/project/plugins.sbt
rename to examples/spark-rdd-colocation-mining/project/plugins.sbt
diff --git a/examples/rdd-colocation-mining/src/main/scala/ScalaExample.scala
b/examples/spark-rdd-colocation-mining/src/main/scala/ScalaExample.scala
similarity index 100%
rename from examples/rdd-colocation-mining/src/main/scala/ScalaExample.scala
rename to examples/spark-rdd-colocation-mining/src/main/scala/ScalaExample.scala
diff --git a/examples/rdd-colocation-mining/src/test/resources/arealandmark.png
b/examples/spark-rdd-colocation-mining/src/test/resources/arealandmark.png
similarity index 100%
rename from examples/rdd-colocation-mining/src/test/resources/arealandmark.png
rename to
examples/spark-rdd-colocation-mining/src/test/resources/arealandmark.png
diff --git a/examples/rdd-colocation-mining/src/test/resources/colocation.png
b/examples/spark-rdd-colocation-mining/src/test/resources/colocation.png
similarity index 100%
rename from examples/rdd-colocation-mining/src/test/resources/colocation.png
rename to examples/spark-rdd-colocation-mining/src/test/resources/colocation.png
diff --git
a/examples/rdd-colocation-mining/src/test/resources/colocationMap.png
b/examples/spark-rdd-colocation-mining/src/test/resources/colocationMap.png
similarity index 100%
rename from examples/rdd-colocation-mining/src/test/resources/colocationMap.png
rename to
examples/spark-rdd-colocation-mining/src/test/resources/colocationMap.png
diff --git
a/examples/rdd-colocation-mining/src/test/resources/colocationResult.png
b/examples/spark-rdd-colocation-mining/src/test/resources/colocationResult.png
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/colocationResult.png
rename to
examples/spark-rdd-colocation-mining/src/test/resources/colocationResult.png
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.cst
b/examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.cst
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.cst
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.cst
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.dbf
b/examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.dbf
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.dbf
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.dbf
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.prj
b/examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.prj
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.prj
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.prj
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shp
b/examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shp
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shp
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shp
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shx
b/examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shx
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shx
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/nyu_2451_34514.shx
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/wfsrequest.txt
b/examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/wfsrequest.txt
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/wfsrequest.txt
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyc-area-landmark-shapefile/wfsrequest.txt
diff --git
a/examples/rdd-colocation-mining/src/test/resources/nyccolocation.png
b/examples/spark-rdd-colocation-mining/src/test/resources/nyccolocation.png
similarity index 100%
rename from examples/rdd-colocation-mining/src/test/resources/nyccolocation.png
rename to
examples/spark-rdd-colocation-mining/src/test/resources/nyccolocation.png
diff --git
a/examples/rdd-colocation-mining/src/test/resources/yellow_tripdata_2009-01-subset.csv
b/examples/spark-rdd-colocation-mining/src/test/resources/yellow_tripdata_2009-01-subset.csv
similarity index 100%
rename from
examples/rdd-colocation-mining/src/test/resources/yellow_tripdata_2009-01-subset.csv
rename to
examples/spark-rdd-colocation-mining/src/test/resources/yellow_tripdata_2009-01-subset.csv
diff --git a/examples/sql/.gitignore b/examples/spark-sql/.gitignore
similarity index 100%
rename from examples/sql/.gitignore
rename to examples/spark-sql/.gitignore
diff --git a/examples/sql/build.sbt b/examples/spark-sql/build.sbt
similarity index 93%
rename from examples/sql/build.sbt
rename to examples/spark-sql/build.sbt
index f7ebd529..9b24f806 100644
--- a/examples/sql/build.sbt
+++ b/examples/spark-sql/build.sbt
@@ -39,14 +39,14 @@ val SparkCompatibleVersion = "3.0"
val HadoopVersion = "2.7.2"
-val SedonaVersion = "1.3.1-incubating"
+val SedonaVersion = "1.4.0"
val ScalaCompatibleVersion = "2.12"
// Change the dependency scope to "provided" when you run "sbt assembly"
val dependencyScope = "compile"
-val geotoolsVersion = "1.3.0-27.2"
+val geotoolsVersion = "1.4.0-28.2"
//val jacksonVersion = "2.10.0"
@@ -62,7 +62,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % HadoopVersion %
dependencyScope,
"org.apache.hadoop" % "hadoop-common" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-hdfs" % HadoopVersion % dependencyScope,
- "org.apache.sedona" %
"sedona-python-adapter-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
+ "org.apache.sedona" %
"sedona-spark-shaded-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
"org.apache.sedona" %
"sedona-viz-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
// The following GeoTools packages are only required if you need CRS
transformation. Under GNU Lesser General Public License (LGPL) license
"org.datasyslab" % "geotools-wrapper" % geotoolsVersion % "compile"
diff --git a/examples/sql/project/.gitignore
b/examples/spark-sql/project/.gitignore
similarity index 100%
rename from examples/sql/project/.gitignore
rename to examples/spark-sql/project/.gitignore
diff --git a/examples/sql/project/build.properties
b/examples/spark-sql/project/build.properties
similarity index 100%
rename from examples/sql/project/build.properties
rename to examples/spark-sql/project/build.properties
diff --git a/examples/sql/project/plugins.sbt
b/examples/spark-sql/project/plugins.sbt
similarity index 100%
rename from examples/sql/project/plugins.sbt
rename to examples/spark-sql/project/plugins.sbt
diff --git a/examples/sql/src/main/scala/ScalaExample.scala
b/examples/spark-sql/src/main/scala/ScalaExample.scala
similarity index 100%
rename from examples/sql/src/main/scala/ScalaExample.scala
rename to examples/spark-sql/src/main/scala/ScalaExample.scala
diff --git a/examples/sql/src/test/resources/.gitignore
b/examples/spark-sql/src/test/resources/.gitignore
similarity index 100%
rename from examples/sql/src/test/resources/.gitignore
rename to examples/spark-sql/src/test/resources/.gitignore
diff --git a/examples/sql/src/test/resources/raster/test1.tiff
b/examples/spark-sql/src/test/resources/raster/test1.tiff
similarity index 100%
rename from examples/sql/src/test/resources/raster/test1.tiff
rename to examples/spark-sql/src/test/resources/raster/test1.tiff
diff --git a/examples/sql/src/test/resources/raster/test2.tiff
b/examples/spark-sql/src/test/resources/raster/test2.tiff
similarity index 100%
rename from examples/sql/src/test/resources/raster/test2.tiff
rename to examples/spark-sql/src/test/resources/raster/test2.tiff
diff --git a/examples/sql/src/test/resources/scalastyle_config.xml
b/examples/spark-sql/src/test/resources/scalastyle_config.xml
similarity index 100%
rename from examples/sql/src/test/resources/scalastyle_config.xml
rename to examples/spark-sql/src/test/resources/scalastyle_config.xml
diff --git a/examples/sql/src/test/resources/shapefiles/dbf/map.dbf
b/examples/spark-sql/src/test/resources/shapefiles/dbf/map.dbf
similarity index 100%
rename from examples/sql/src/test/resources/shapefiles/dbf/map.dbf
rename to examples/spark-sql/src/test/resources/shapefiles/dbf/map.dbf
diff --git a/examples/sql/src/test/resources/shapefiles/dbf/map.shp
b/examples/spark-sql/src/test/resources/shapefiles/dbf/map.shp
similarity index 100%
rename from examples/sql/src/test/resources/shapefiles/dbf/map.shp
rename to examples/spark-sql/src/test/resources/shapefiles/dbf/map.shp
diff --git a/examples/sql/src/test/resources/shapefiles/dbf/map.shx
b/examples/spark-sql/src/test/resources/shapefiles/dbf/map.shx
similarity index 100%
rename from examples/sql/src/test/resources/shapefiles/dbf/map.shx
rename to examples/spark-sql/src/test/resources/shapefiles/dbf/map.shx
diff --git a/examples/sql/src/test/resources/testenvelope.csv
b/examples/spark-sql/src/test/resources/testenvelope.csv
similarity index 100%
rename from examples/sql/src/test/resources/testenvelope.csv
rename to examples/spark-sql/src/test/resources/testenvelope.csv
diff --git a/examples/sql/src/test/resources/testpoint.csv
b/examples/spark-sql/src/test/resources/testpoint.csv
similarity index 100%
rename from examples/sql/src/test/resources/testpoint.csv
rename to examples/spark-sql/src/test/resources/testpoint.csv
diff --git a/examples/viz/.gitignore b/examples/spark-viz/.gitignore
similarity index 100%
rename from examples/viz/.gitignore
rename to examples/spark-viz/.gitignore
diff --git a/examples/viz/build.sbt b/examples/spark-viz/build.sbt
similarity index 93%
rename from examples/viz/build.sbt
rename to examples/spark-viz/build.sbt
index bfff3fba..6b3e1d37 100644
--- a/examples/viz/build.sbt
+++ b/examples/spark-viz/build.sbt
@@ -39,14 +39,14 @@ val SparkCompatibleVersion = "3.0"
val HadoopVersion = "2.7.2"
-val SedonaVersion = "1.3.1-incubating"
+val SedonaVersion = "1.4.0"
val ScalaCompatibleVersion = "2.12"
// Change the dependency scope to "provided" when you run "sbt assembly"
val dependencyScope = "compile"
-val geotoolsVersion = "1.3.0-27.2"
+val geotoolsVersion = "1.4.0-28.2"
//val jacksonVersion = "2.10.0"
@@ -62,7 +62,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % HadoopVersion %
dependencyScope,
"org.apache.hadoop" % "hadoop-common" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-hdfs" % HadoopVersion % dependencyScope,
- "org.apache.sedona" %
"sedona-python-adapter-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
+ "org.apache.sedona" %
"sedona-spark-shaded-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
"org.apache.sedona" %
"sedona-viz-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion)
% SedonaVersion changing(),
// The following GeoTools packages are only required if you need CRS
transformation. Under GNU Lesser General Public License (LGPL) license
"org.datasyslab" % "geotools-wrapper" % geotoolsVersion % "compile"
diff --git a/examples/viz/project/.gitignore
b/examples/spark-viz/project/.gitignore
similarity index 100%
rename from examples/viz/project/.gitignore
rename to examples/spark-viz/project/.gitignore
diff --git a/examples/viz/project/build.properties
b/examples/spark-viz/project/build.properties
similarity index 100%
rename from examples/viz/project/build.properties
rename to examples/spark-viz/project/build.properties
diff --git a/examples/viz/project/plugins.sbt
b/examples/spark-viz/project/plugins.sbt
similarity index 100%
rename from examples/viz/project/plugins.sbt
rename to examples/spark-viz/project/plugins.sbt
diff --git a/examples/viz/src/main/scala/ScalaExample.scala
b/examples/spark-viz/src/main/scala/ScalaExample.scala
similarity index 100%
rename from examples/viz/src/main/scala/ScalaExample.scala
rename to examples/spark-viz/src/main/scala/ScalaExample.scala
diff --git a/examples/viz/src/test/resources/arealm.csv
b/examples/spark-viz/src/test/resources/arealm.csv
similarity index 100%
rename from examples/viz/src/test/resources/arealm.csv
rename to examples/spark-viz/src/test/resources/arealm.csv
diff --git a/examples/viz/src/test/resources/babylon.point.properties
b/examples/spark-viz/src/test/resources/babylon.point.properties
similarity index 100%
rename from examples/viz/src/test/resources/babylon.point.properties
rename to examples/spark-viz/src/test/resources/babylon.point.properties
diff --git a/examples/viz/src/test/resources/babylon.polygon.properties
b/examples/spark-viz/src/test/resources/babylon.polygon.properties
similarity index 100%
rename from examples/viz/src/test/resources/babylon.polygon.properties
rename to examples/spark-viz/src/test/resources/babylon.polygon.properties
diff --git a/examples/viz/src/test/resources/babylon.rectangle.properties
b/examples/spark-viz/src/test/resources/babylon.rectangle.properties
similarity index 100%
rename from examples/viz/src/test/resources/babylon.rectangle.properties
rename to examples/spark-viz/src/test/resources/babylon.rectangle.properties
diff --git a/examples/viz/src/test/resources/primaryroads-polygon.csv
b/examples/spark-viz/src/test/resources/primaryroads-polygon.csv
similarity index 100%
rename from examples/viz/src/test/resources/primaryroads-polygon.csv
rename to examples/spark-viz/src/test/resources/primaryroads-polygon.csv
diff --git a/examples/viz/src/test/resources/zcta510.csv
b/examples/spark-viz/src/test/resources/zcta510.csv
similarity index 100%
rename from examples/viz/src/test/resources/zcta510.csv
rename to examples/spark-viz/src/test/resources/zcta510.csv
diff --git a/mkdocs.yml b/mkdocs.yml
index b5981fcb..233a325d 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -127,13 +127,14 @@ extra:
- icon: fontawesome/brands/twitter
link: 'https://twitter.com/ApacheSedona'
sedona:
- current_version: 1.3.1-incubating
- current_geotools: 1.3.0-27.2
+ current_version: 1.4.0
+ current_geotools: 1.4.0-28.2
sedona_create_release:
current_version: 1.4.0
current_git_tag: sedona-1.4.0-rc1
current_rc: 1.4.0-rc1
- current_snapshot: 1.4.0-SNAPSHOT
+ current_snapshot: 1.4.1-SNAPSHOT
+ next_version: 1.4.1
copyright: Copyright © 2023 The Apache Software Foundation
markdown_extensions:
- admonition
