This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d414535cb [KYUUBI #6582] [KYUUBI-6581] Zorder clause syntax does not
support special characters
d414535cb is described below
commit d414535cb60de471aff4862c2153444ae1feb716
Author: xorsum <[email protected]>
AuthorDate: Tue Aug 6 13:39:25 2024 +0800
[KYUUBI #6582] [KYUUBI-6581] Zorder clause syntax does not support special
characters
# :mag: Description
## Issue References ๐
This pull request fixes #6581
## Describe Your Solution ๐ง
Please include a summary of the change and which issue is fixed. Please
also include relevant motivation and context. List any dependencies that are
required for this change.
I modified `KyuubiSparkSQLAstBuilder#visitMultipartIdentifier` and
implemented `KyuubiSparkSQLAstBuilder#visitQuotedIdentifier` to process the
quoted identifiers.
## Types of changes :bookmark:
- [x] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
#### Behavior With This Pull Request :tada:
#### Related Unit Tests
```
extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
test("optimize sort by backquoted column name")
```
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6582 from XorSum/features/zorder-backquote.
Closes #6582
16ffa1238 [xorsum] zorder by support quote
Authored-by: xorsum <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../kyuubi/sql/KyuubiSparkSQLAstBuilder.scala | 20 +++++++++-
.../org/apache/spark/sql/ZorderSuiteBase.scala | 43 ++++++++++++++++++++++
.../kyuubi/sql/KyuubiSparkSQLAstBuilder.scala | 20 +++++++++-
.../org/apache/spark/sql/ZorderSuiteBase.scala | 43 ++++++++++++++++++++++
.../kyuubi/sql/KyuubiSparkSQLAstBuilder.scala | 20 +++++++++-
.../org/apache/spark/sql/ZorderSuiteBase.scala | 43 ++++++++++++++++++++++
6 files changed, 183 insertions(+), 6 deletions(-)
diff --git
a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
index a7d91ba8b..083623d4e 100644
---
a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
@@ -131,14 +131,30 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
override def visitMultipartIdentifier(ctx: MultipartIdentifierContext):
Seq[String] =
withOrigin(ctx) {
- ctx.parts.asScala.map(_.getText).toSeq
+ ctx.parts.asScala.map(typedVisit[String]).toSeq
}
+ override def visitIdentifier(ctx: IdentifierContext): String = {
+ withOrigin(ctx) {
+ ctx.strictIdentifier() match {
+ case quotedContext: QuotedIdentifierAlternativeContext =>
+ typedVisit[String](quotedContext)
+ case _ => ctx.getText
+ }
+ }
+ }
+
+ override def visitQuotedIdentifier(ctx: QuotedIdentifierContext): String = {
+ withOrigin(ctx) {
+
ctx.BACKQUOTED_IDENTIFIER().getText.stripPrefix("`").stripSuffix("`").replace("``",
"`")
+ }
+ }
+
override def visitZorderClause(ctx: ZorderClauseContext):
Seq[UnresolvedAttribute] =
withOrigin(ctx) {
val res = ListBuffer[UnresolvedAttribute]()
ctx.multipartIdentifier().forEach { identifier =>
- res +=
UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq)
+ res +=
UnresolvedAttribute(identifier.parts.asScala.map(typedVisit[String]).toSeq)
}
res.toSeq
}
diff --git
a/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
index 82ca8d4d2..2e6e1933a 100644
---
a/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
@@ -771,6 +771,49 @@ trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest
with ExpressionEvalHel
}
}
+ test("optimize sort by backquoted column name") {
+ withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+ withTable("up") {
+ sql(s"DROP TABLE IF EXISTS up")
+ val target = Seq(
+ Seq(0, 0),
+ Seq(1, 0),
+ Seq(0, 1),
+ Seq(1, 1),
+ Seq(2, 0),
+ Seq(3, 0),
+ Seq(2, 1),
+ Seq(3, 1),
+ Seq(0, 2),
+ Seq(1, 2),
+ Seq(0, 3),
+ Seq(1, 3),
+ Seq(2, 2),
+ Seq(3, 2),
+ Seq(2, 3),
+ Seq(3, 3))
+ sql(s"CREATE TABLE up (c1 INT, `@c2` INT, c3 INT)")
+ sql(s"INSERT INTO TABLE up VALUES" +
+ "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," +
+ "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," +
+ "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," +
+ "(3,0,3),(3,1,4),(3,2,9),(3,3,0)")
+
+ sql("OPTIMIZE up ZORDER BY c1, `@c2`")
+ val res = sql("SELECT c1, `@c2` FROM up").collect()
+
+ assert(res.length == 16)
+
+ for (i <- target.indices) {
+ val t = target(i)
+ val r = res(i)
+ assert(t(0) == r.getInt(0))
+ assert(t(1) == r.getInt(1))
+ }
+ }
+ }
+ }
+
def createParser: ParserInterface
}
diff --git
a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
index cc00bf88e..7ee439a43 100644
---
a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
@@ -131,14 +131,30 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
override def visitMultipartIdentifier(ctx: MultipartIdentifierContext):
Seq[String] =
withOrigin(ctx) {
- ctx.parts.asScala.map(_.getText).toSeq
+ ctx.parts.asScala.map(typedVisit[String]).toSeq
}
+ override def visitIdentifier(ctx: IdentifierContext): String = {
+ withOrigin(ctx) {
+ ctx.strictIdentifier() match {
+ case quotedContext: QuotedIdentifierAlternativeContext =>
+ typedVisit[String](quotedContext)
+ case _ => ctx.getText
+ }
+ }
+ }
+
+ override def visitQuotedIdentifier(ctx: QuotedIdentifierContext): String = {
+ withOrigin(ctx) {
+
ctx.BACKQUOTED_IDENTIFIER().getText.stripPrefix("`").stripSuffix("`").replace("``",
"`")
+ }
+ }
+
override def visitZorderClause(ctx: ZorderClauseContext):
Seq[UnresolvedAttribute] =
withOrigin(ctx) {
val res = ListBuffer[UnresolvedAttribute]()
ctx.multipartIdentifier().forEach { identifier =>
- res +=
UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq)
+ res +=
UnresolvedAttribute(identifier.parts.asScala.map(typedVisit[String]).toSeq)
}
res.toSeq
}
diff --git
a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
index 26c8ebd1d..b8e01fa83 100644
---
a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
@@ -772,6 +772,49 @@ trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest
with ExpressionEvalHel
}
}
+ test("optimize sort by backquoted column name") {
+ withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+ withTable("up") {
+ sql(s"DROP TABLE IF EXISTS up")
+ val target = Seq(
+ Seq(0, 0),
+ Seq(1, 0),
+ Seq(0, 1),
+ Seq(1, 1),
+ Seq(2, 0),
+ Seq(3, 0),
+ Seq(2, 1),
+ Seq(3, 1),
+ Seq(0, 2),
+ Seq(1, 2),
+ Seq(0, 3),
+ Seq(1, 3),
+ Seq(2, 2),
+ Seq(3, 2),
+ Seq(2, 3),
+ Seq(3, 3))
+ sql(s"CREATE TABLE up (c1 INT, `@c2` INT, c3 INT)")
+ sql(s"INSERT INTO TABLE up VALUES" +
+ "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," +
+ "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," +
+ "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," +
+ "(3,0,3),(3,1,4),(3,2,9),(3,3,0)")
+
+ sql("OPTIMIZE up ZORDER BY c1, `@c2`")
+ val res = sql("SELECT c1, `@c2` FROM up").collect()
+
+ assert(res.length == 16)
+
+ for (i <- target.indices) {
+ val t = target(i)
+ val r = res(i)
+ assert(t(0) == r.getInt(0))
+ assert(t(1) == r.getInt(1))
+ }
+ }
+ }
+ }
+
def createParser: ParserInterface
}
diff --git
a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
index cc00bf88e..7ee439a43 100644
---
a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
@@ -131,14 +131,30 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
override def visitMultipartIdentifier(ctx: MultipartIdentifierContext):
Seq[String] =
withOrigin(ctx) {
- ctx.parts.asScala.map(_.getText).toSeq
+ ctx.parts.asScala.map(typedVisit[String]).toSeq
}
+ override def visitIdentifier(ctx: IdentifierContext): String = {
+ withOrigin(ctx) {
+ ctx.strictIdentifier() match {
+ case quotedContext: QuotedIdentifierAlternativeContext =>
+ typedVisit[String](quotedContext)
+ case _ => ctx.getText
+ }
+ }
+ }
+
+ override def visitQuotedIdentifier(ctx: QuotedIdentifierContext): String = {
+ withOrigin(ctx) {
+
ctx.BACKQUOTED_IDENTIFIER().getText.stripPrefix("`").stripSuffix("`").replace("``",
"`")
+ }
+ }
+
override def visitZorderClause(ctx: ZorderClauseContext):
Seq[UnresolvedAttribute] =
withOrigin(ctx) {
val res = ListBuffer[UnresolvedAttribute]()
ctx.multipartIdentifier().forEach { identifier =>
- res +=
UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq)
+ res +=
UnresolvedAttribute(identifier.parts.asScala.map(typedVisit[String]).toSeq)
}
res.toSeq
}
diff --git
a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
index 26c8ebd1d..b8e01fa83 100644
---
a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala
@@ -772,6 +772,49 @@ trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest
with ExpressionEvalHel
}
}
+ test("optimize sort by backquoted column name") {
+ withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") {
+ withTable("up") {
+ sql(s"DROP TABLE IF EXISTS up")
+ val target = Seq(
+ Seq(0, 0),
+ Seq(1, 0),
+ Seq(0, 1),
+ Seq(1, 1),
+ Seq(2, 0),
+ Seq(3, 0),
+ Seq(2, 1),
+ Seq(3, 1),
+ Seq(0, 2),
+ Seq(1, 2),
+ Seq(0, 3),
+ Seq(1, 3),
+ Seq(2, 2),
+ Seq(3, 2),
+ Seq(2, 3),
+ Seq(3, 3))
+ sql(s"CREATE TABLE up (c1 INT, `@c2` INT, c3 INT)")
+ sql(s"INSERT INTO TABLE up VALUES" +
+ "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," +
+ "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," +
+ "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," +
+ "(3,0,3),(3,1,4),(3,2,9),(3,3,0)")
+
+ sql("OPTIMIZE up ZORDER BY c1, `@c2`")
+ val res = sql("SELECT c1, `@c2` FROM up").collect()
+
+ assert(res.length == 16)
+
+ for (i <- target.indices) {
+ val t = target(i)
+ val r = res(i)
+ assert(t(0) == r.getInt(0))
+ assert(t(1) == r.getInt(1))
+ }
+ }
+ }
+ }
+
def createParser: ParserInterface
}
