This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new d2261e3e6 [SEDONA-636] Parser extension should fall back to delegated
parser when parsing failed (#1547)
d2261e3e6 is described below
commit d2261e3e6b0717ddaa9104d85e548cdcc89de650
Author: Kristin Cowalcijk <[email protected]>
AuthorDate: Fri Aug 16 01:26:23 2024 +0800
[SEDONA-636] Parser extension should fall back to delegated parser when
parsing failed (#1547)
---
docs/api/sql/Parameter.md | 4 ++++
docs/tutorial/sql-pure-sql.md | 15 +++++++++++++
.../org/apache/sedona/spark/SedonaContext.scala | 4 +++-
.../apache/sedona/sql/parser/SedonaSqlParser.scala | 16 +++++++++-----
.../org/sedona/sql/parser/SedonaSqlParser.scala | 16 +++++++++-----
.../apache/sedona/sql/parser/SedonaSqlParser.scala | 16 +++++++++-----
.../src/main/java/org/apache/sedona/Main.java | 25 ----------------------
.../apache/sedona/sql/parser/SedonaSqlParser.scala | 16 +++++++++-----
.../apache/sedona/sql/parser/SedonaSqlParser.scala | 16 +++++++++-----
.../apache/sedona/sql/parser/SedonaSqlParser.scala | 16 +++++++++-----
10 files changed, 88 insertions(+), 56 deletions(-)
diff --git a/docs/api/sql/Parameter.md b/docs/api/sql/Parameter.md
index b84b7cdee..81decf608 100644
--- a/docs/api/sql/Parameter.md
+++ b/docs/api/sql/Parameter.md
@@ -63,3 +63,7 @@ sparkSession.conf.set("sedona.global.index","false")
* all: Always optimize spatial join queries, even for
equi-joins.
* none: Disable optimization for spatial joins.
* nonequi: Optimize spatial join queries that are not
equi-joins.
+* spark.sedona.enableParserExtension
+ * Enable the parser extension to parse GEOMETRY data type in SQL DDL
statements
+ * Default: true
+ * Possible values: true, false
diff --git a/docs/tutorial/sql-pure-sql.md b/docs/tutorial/sql-pure-sql.md
index dd95a26a5..833d8e385 100644
--- a/docs/tutorial/sql-pure-sql.md
+++ b/docs/tutorial/sql-pure-sql.md
@@ -74,3 +74,18 @@ WHERE ST_Contains(polygondata.polygonshape,
pointdata.pointshape)
AND ST_Contains(ST_PolygonFromEnvelope(1.0,101.0,501.0,601.0),
polygondata.polygonshape)
LIMIT 5;
```
+
+## `GEOMETRY` data type support
+
+Sedona has a Spark SQL parser extension to support `GEOMETRY` data type in DDL
statements. For example, you can specify a schema with a geometry column when
creating the table:
+
+```sql
+CREATE TABLE geom_table (id STRING, version INT, geometry GEOMETRY)
+USING geoparquet
+LOCATION '/path/to/geoparquet_geom_table';
+
+SELECT * FROM geom_table LIMIT 10;
+```
+
+The SQL parser extension is enabled by default. If you find it conflicting
with other extensions and want to disable it,
+please specify `--conf spark.sedona.enableParserExtension=false` when starting
`spark-sql`.
diff --git
a/spark/common/src/main/scala/org/apache/sedona/spark/SedonaContext.scala
b/spark/common/src/main/scala/org/apache/sedona/spark/SedonaContext.scala
index 692fe5f72..4c8fcab69 100644
--- a/spark/common/src/main/scala/org/apache/sedona/spark/SedonaContext.scala
+++ b/spark/common/src/main/scala/org/apache/sedona/spark/SedonaContext.scala
@@ -65,7 +65,9 @@ object SedonaContext {
RasterRegistrator.registerAll(sparkSession)
UdtRegistrator.registerAll()
UdfRegistrator.registerAll(sparkSession)
- ParserRegistrator.register(sparkSession)
+ if (sparkSession.conf.get("spark.sedona.enableParserExtension",
"true").toBoolean) {
+ ParserRegistrator.register(sparkSession)
+ }
sparkSession
}
diff --git
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
index 2d270eaa4..80653d40d 100644
---
a/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
+++
b/spark/spark-3.0/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
@@ -33,11 +33,17 @@ class SedonaSqlParser(conf: SQLConf, delegate:
ParserInterface) extends SparkSql
* @param sqlText
* @return
*/
- override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) {
parser =>
- parserBuilder.visit(parser.singleStatement()) match {
- case plan: LogicalPlan => plan
- case _ =>
+ override def parsePlan(sqlText: String): LogicalPlan =
+ try {
+ parse(sqlText) { parser =>
+ parserBuilder.visit(parser.singleStatement()) match {
+ case plan: LogicalPlan => plan
+ case _ =>
+ delegate.parsePlan(sqlText)
+ }
+ }
+ } catch {
+ case _: Exception =>
delegate.parsePlan(sqlText)
}
- }
}
diff --git
a/spark/spark-3.1/src/main/scala/org/sedona/sql/parser/SedonaSqlParser.scala
b/spark/spark-3.1/src/main/scala/org/sedona/sql/parser/SedonaSqlParser.scala
index 420949ab9..6c7041912 100644
--- a/spark/spark-3.1/src/main/scala/org/sedona/sql/parser/SedonaSqlParser.scala
+++ b/spark/spark-3.1/src/main/scala/org/sedona/sql/parser/SedonaSqlParser.scala
@@ -32,11 +32,17 @@ class SedonaSqlParser(delegate: ParserInterface) extends
SparkSqlParser {
* @param sqlText
* @return
*/
- override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) {
parser =>
- parserBuilder.visit(parser.singleStatement()) match {
- case plan: LogicalPlan => plan
- case _ =>
+ override def parsePlan(sqlText: String): LogicalPlan =
+ try {
+ parse(sqlText) { parser =>
+ parserBuilder.visit(parser.singleStatement()) match {
+ case plan: LogicalPlan => plan
+ case _ =>
+ delegate.parsePlan(sqlText)
+ }
+ }
+ } catch {
+ case _: Exception =>
delegate.parsePlan(sqlText)
}
- }
}
diff --git
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
index 420949ab9..6c7041912 100644
---
a/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
+++
b/spark/spark-3.2/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
@@ -32,11 +32,17 @@ class SedonaSqlParser(delegate: ParserInterface) extends
SparkSqlParser {
* @param sqlText
* @return
*/
- override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) {
parser =>
- parserBuilder.visit(parser.singleStatement()) match {
- case plan: LogicalPlan => plan
- case _ =>
+ override def parsePlan(sqlText: String): LogicalPlan =
+ try {
+ parse(sqlText) { parser =>
+ parserBuilder.visit(parser.singleStatement()) match {
+ case plan: LogicalPlan => plan
+ case _ =>
+ delegate.parsePlan(sqlText)
+ }
+ }
+ } catch {
+ case _: Exception =>
delegate.parsePlan(sqlText)
}
- }
}
diff --git a/spark/spark-3.3/src/main/java/org/apache/sedona/Main.java
b/spark/spark-3.3/src/main/java/org/apache/sedona/Main.java
deleted file mode 100644
index acb8e3005..000000000
--- a/spark/spark-3.3/src/main/java/org/apache/sedona/Main.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sedona;
-
-public class Main {
- public static void main(String[] args) {
- System.out.println("Hello world!");
- }
-}
diff --git
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
index 420949ab9..6c7041912 100644
---
a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
+++
b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
@@ -32,11 +32,17 @@ class SedonaSqlParser(delegate: ParserInterface) extends
SparkSqlParser {
* @param sqlText
* @return
*/
- override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) {
parser =>
- parserBuilder.visit(parser.singleStatement()) match {
- case plan: LogicalPlan => plan
- case _ =>
+ override def parsePlan(sqlText: String): LogicalPlan =
+ try {
+ parse(sqlText) { parser =>
+ parserBuilder.visit(parser.singleStatement()) match {
+ case plan: LogicalPlan => plan
+ case _ =>
+ delegate.parsePlan(sqlText)
+ }
+ }
+ } catch {
+ case _: Exception =>
delegate.parsePlan(sqlText)
}
- }
}
diff --git
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
index 420949ab9..6c7041912 100644
---
a/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
+++
b/spark/spark-3.4/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
@@ -32,11 +32,17 @@ class SedonaSqlParser(delegate: ParserInterface) extends
SparkSqlParser {
* @param sqlText
* @return
*/
- override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) {
parser =>
- parserBuilder.visit(parser.singleStatement()) match {
- case plan: LogicalPlan => plan
- case _ =>
+ override def parsePlan(sqlText: String): LogicalPlan =
+ try {
+ parse(sqlText) { parser =>
+ parserBuilder.visit(parser.singleStatement()) match {
+ case plan: LogicalPlan => plan
+ case _ =>
+ delegate.parsePlan(sqlText)
+ }
+ }
+ } catch {
+ case _: Exception =>
delegate.parsePlan(sqlText)
}
- }
}
diff --git
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
index 420949ab9..6c7041912 100644
---
a/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
+++
b/spark/spark-3.5/src/main/scala/org/apache/sedona/sql/parser/SedonaSqlParser.scala
@@ -32,11 +32,17 @@ class SedonaSqlParser(delegate: ParserInterface) extends
SparkSqlParser {
* @param sqlText
* @return
*/
- override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) {
parser =>
- parserBuilder.visit(parser.singleStatement()) match {
- case plan: LogicalPlan => plan
- case _ =>
+ override def parsePlan(sqlText: String): LogicalPlan =
+ try {
+ parse(sqlText) { parser =>
+ parserBuilder.visit(parser.singleStatement()) match {
+ case plan: LogicalPlan => plan
+ case _ =>
+ delegate.parsePlan(sqlText)
+ }
+ }
+ } catch {
+ case _: Exception =>
delegate.parsePlan(sqlText)
}
- }
}
