This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 14cce8847 [spark] Fix PaimonParseException compatibility with spark3.3
(#2849)
14cce8847 is described below
commit 14cce88475fe21f30fa63172d17ae0fda760146c
Author: Zouxxyy <[email protected]>
AuthorDate: Tue Feb 6 14:01:18 2024 +0800
[spark] Fix PaimonParseException compatibility with spark3.3 (#2849)
---
.../paimon/spark/procedure/ProcedureTest.scala} | 17 +----------------
.../extensions/PaimonSparkSqlExtensionsParser.scala | 19 +++++--------------
.../extensions/PaimonSqlExtensionsAstBuilder.scala | 2 --
.../paimon/spark/procedure/ProcedureTestBase.scala | 14 +++++++++++++-
4 files changed, 19 insertions(+), 33 deletions(-)
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala
similarity index 60%
copy from
paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
copy to
paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala
index 9dd7124b9..2d582c5ac 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
+++
b/paimon-spark/paimon-spark-3.4/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTest.scala
@@ -17,19 +17,4 @@
*/
package org.apache.paimon.spark.procedure
-import org.apache.paimon.spark.PaimonSparkTestBase
-import org.apache.paimon.spark.analysis.NoSuchProcedureException
-
-import org.assertj.core.api.Assertions.assertThatThrownBy
-
-abstract class ProcedureTestBase extends PaimonSparkTestBase {
-
- test(s"test call unknown procedure") {
- spark.sql(s"""
- |CREATE TABLE T (id INT, name STRING, dt STRING)
- |""".stripMargin)
-
- assertThatThrownBy(() => spark.sql("CALL unknown_procedure(table =>
'test.T')"))
- .isInstanceOf(classOf[NoSuchProcedureException])
- }
-}
+class ProcedureTest extends ProcedureTestBase {}
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala
index f42cbab55..188917330 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSparkSqlExtensionsParser.scala
@@ -28,7 +28,6 @@ import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface}
import
org.apache.spark.sql.catalyst.parser.extensions.PaimonSqlExtensionsParser.{NonReservedContext,
QuotedIdentifierContext}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.catalyst.trees.Origin
import org.apache.spark.sql.internal.VariableSubstitution
import org.apache.spark.sql.types.{DataType, StructType}
@@ -220,23 +219,15 @@ case object PaimonParseErrorListener extends
BaseErrorListener {
}
/**
- * Copied from Apache Spark A [[ParseException]] is an [[AnalysisException]]
that is thrown during
- * the parse process. It contains fields and an extended error message that
make reporting and
- * diagnosing errors easier.
+ * Copied from Apache Spark [[ParseException]], it contains fields and an
extended error message
+ * that make reporting and diagnosing errors easier.
*/
class PaimonParseException(
val command: Option[String],
message: String,
- val start: Origin,
- val stop: Origin)
- extends AnalysisException(message, start.line, start.startPosition) {
-
- def this(message: String, ctx: ParserRuleContext) =
- this(
- Option(PaimonParserUtils.command(ctx)),
- message,
- PaimonParserUtils.position(ctx.getStart),
- PaimonParserUtils.position(ctx.getStop))
+ start: Origin,
+ stop: Origin)
+ extends Exception {
override def getMessage: String = {
val builder = new StringBuilder
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
index bb8fd9a33..ff6b15d2c 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonSqlExtensionsAstBuilder.scala
@@ -23,14 +23,12 @@ import
org.apache.paimon.spark.catalyst.plans.logical.{PaimonCallArgument, Paimo
import org.antlr.v4.runtime._
import org.antlr.v4.runtime.misc.Interval
import org.antlr.v4.runtime.tree.{ParseTree, TerminalNode}
-import org.apache.spark.QueryContext
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.parser.ParserInterface
import
org.apache.spark.sql.catalyst.parser.extensions.PaimonParserUtils.withOrigin
import
org.apache.spark.sql.catalyst.parser.extensions.PaimonSqlExtensionsParser._
import org.apache.spark.sql.catalyst.plans.logical._
-import org.apache.spark.sql.catalyst.trees.SQLQueryContext
import scala.collection.JavaConverters._
diff --git
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
index 9dd7124b9..8d1717cb0 100644
---
a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
+++
b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/procedure/ProcedureTestBase.scala
@@ -20,6 +20,7 @@ package org.apache.paimon.spark.procedure
import org.apache.paimon.spark.PaimonSparkTestBase
import org.apache.paimon.spark.analysis.NoSuchProcedureException
+import org.apache.spark.sql.catalyst.parser.extensions.PaimonParseException
import org.assertj.core.api.Assertions.assertThatThrownBy
abstract class ProcedureTestBase extends PaimonSparkTestBase {
@@ -29,7 +30,18 @@ abstract class ProcedureTestBase extends PaimonSparkTestBase
{
|CREATE TABLE T (id INT, name STRING, dt STRING)
|""".stripMargin)
- assertThatThrownBy(() => spark.sql("CALL unknown_procedure(table =>
'test.T')"))
+ assertThatThrownBy(() => spark.sql("CALL sys.unknown_procedure(table =>
'test.T')"))
.isInstanceOf(classOf[NoSuchProcedureException])
}
+
+ test(s"test parse exception") {
+ spark.sql(s"""
+ |CREATE TABLE T (id INT, name STRING, dt STRING)
+ |""".stripMargin)
+
+ // Using Chinese comma to simulate parser exception
+ assertThatThrownBy(
+ () => spark.sql("CALL sys.expire_snapshots(table => 'test.T'，retain_max
=> 1)"))
+ .isInstanceOf(classOf[PaimonParseException])
+ }
}
