This is an automated email from the ASF dual-hosted git repository.
bowenliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new ac624b9ff [KYUUBI #5191] Make Spark extension plugin compilable on
Scala 2.13
ac624b9ff is described below
commit ac624b9ff9d2cf18c3d959bc7b39b211939db75e
Author: liangbowen <[email protected]>
AuthorDate: Wed Aug 23 08:43:11 2023 +0800
[KYUUBI #5191] Make Spark extension plugin compilable on Scala 2.13
### _Why are the changes needed?_
- to make Spark extension plugin compilable on Scala 2.13 with Spark
3.2/3.3/3.4 (Spark 3.1 does not support Scala 2.13)
```
mvn clean install -DskipTests -Pflink-provided,hive-provided,spark-provided
-Pscala-2.13 -pl :kyuubi-extension-spark-3-2_2.13 -Pspark-3.2 -am
mvn clean install -DskipTests -Pflink-provided,hive-provided,spark-provided
-Pscala-2.13 -pl :kyuubi-extension-spark-3-3_2.13 -Pspark-3.3 -am
mvn clean install -DskipTests -Pflink-provided,hive-provided,spark-provided
-Pscala-2.13 -pl :kyuubi-extension-spark-3-4_2.13 -Pspark-3.4 -am
```
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
Closes #5191 from bowenliang123/scala213-sparkext.
Closes #5191
156ae26c8 [liangbowen] adapt spark ext plugin to Scala 2.13
Authored-by: liangbowen <[email protected]>
Signed-off-by: liangbowen <[email protected]>
---
extensions/spark/kyuubi-extension-spark-3-4/pom.xml | 2 +-
.../scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala | 7 ++++---
.../src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala | 2 +-
.../scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala | 7 ++++---
.../src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala | 2 +-
5 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/extensions/spark/kyuubi-extension-spark-3-4/pom.xml
b/extensions/spark/kyuubi-extension-spark-3-4/pom.xml
index 20db5d12f..fcd5026da 100644
--- a/extensions/spark/kyuubi-extension-spark-3-4/pom.xml
+++ b/extensions/spark/kyuubi-extension-spark-3-4/pom.xml
@@ -25,7 +25,7 @@
<relativePath>../../../pom.xml</relativePath>
</parent>
- <artifactId>kyuubi-extension-spark-3-4_2.12</artifactId>
+ <artifactId>kyuubi-extension-spark-3-4_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<name>Kyuubi Dev Spark Extensions (for Spark 3.4)</name>
<url>https://kyuubi.apache.org/</url>
diff --git
a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
index c937bd575..cc00bf88e 100644
---
a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
@@ -116,6 +116,7 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
val zorderCols = ctx.zorderClause().order.asScala
.map(visitMultipartIdentifier)
.map(UnresolvedAttribute(_))
+ .toSeq
val orderExpr =
if (zorderCols.length == 1) {
@@ -130,16 +131,16 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
override def visitMultipartIdentifier(ctx: MultipartIdentifierContext):
Seq[String] =
withOrigin(ctx) {
- ctx.parts.asScala.map(_.getText)
+ ctx.parts.asScala.map(_.getText).toSeq
}
override def visitZorderClause(ctx: ZorderClauseContext):
Seq[UnresolvedAttribute] =
withOrigin(ctx) {
val res = ListBuffer[UnresolvedAttribute]()
ctx.multipartIdentifier().forEach { identifier =>
- res += UnresolvedAttribute(identifier.parts.asScala.map(_.getText))
+ res +=
UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq)
}
- res
+ res.toSeq
}
private def typedVisit[T](ctx: ParseTree): T = {
diff --git
a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
index 6254829f2..a202e813c 100644
---
a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
+++
b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
@@ -376,7 +376,7 @@ trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest
{
|ORDER BY a
|DESC
|""".stripMargin)
- .collect().head.get(0).equals(10))
+ .collect().head.get(0) === 10)
}
}
}
diff --git
a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
index c937bd575..cc00bf88e 100644
---
a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
+++
b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala
@@ -116,6 +116,7 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
val zorderCols = ctx.zorderClause().order.asScala
.map(visitMultipartIdentifier)
.map(UnresolvedAttribute(_))
+ .toSeq
val orderExpr =
if (zorderCols.length == 1) {
@@ -130,16 +131,16 @@ class KyuubiSparkSQLAstBuilder extends
KyuubiSparkSQLBaseVisitor[AnyRef] with SQ
override def visitMultipartIdentifier(ctx: MultipartIdentifierContext):
Seq[String] =
withOrigin(ctx) {
- ctx.parts.asScala.map(_.getText)
+ ctx.parts.asScala.map(_.getText).toSeq
}
override def visitZorderClause(ctx: ZorderClauseContext):
Seq[UnresolvedAttribute] =
withOrigin(ctx) {
val res = ListBuffer[UnresolvedAttribute]()
ctx.multipartIdentifier().forEach { identifier =>
- res += UnresolvedAttribute(identifier.parts.asScala.map(_.getText))
+ res +=
UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq)
}
- res
+ res.toSeq
}
private def typedVisit[T](ctx: ParseTree): T = {
diff --git
a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
index 6254829f2..a202e813c 100644
---
a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
+++
b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala
@@ -376,7 +376,7 @@ trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest
{
|ORDER BY a
|DESC
|""".stripMargin)
- .collect().head.get(0).equals(10))
+ .collect().head.get(0) === 10)
}
}
}
