This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 1ab68974b [KYUUBI #2471] Fix the bug of dynamically loading external
packages
1ab68974b is described below
commit 1ab68974b71b89c086b032e3861e789072a96d9b
Author: odone <[email protected]>
AuthorDate: Thu May 12 16:57:02 2022 +0800
[KYUUBI #2471] Fix the bug of dynamically loading external packages
### _Why are the changes needed?_
In scala code mode implementation, we need to call
`repl.addUrlsToClassPath` to set the classpath of the current thread to repl
before entering `repl`
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #2475 from iodone/dev-1.
Closes #2471
21b92cc4 [odone] [KYUUBI #2471] add dynamic jar generator
3542786e [odone] [KYUUBI #2471] Fixed
3db2a7df [odone] [KYUUBI #2471] Fixed
65303683 [odone] [KYUUBI #2471] Fixed
Authored-by: odone <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
bin/docker-image-tool.sh | 2 +-
.../engine/spark/operation/ExecuteScala.scala | 3 +
.../src/test/resources/test-function.jar | Bin 0 -> 1673 bytes
integration-tests/kyuubi-kubernetes-it/pom.xml | 12 ++++
kyuubi-common/pom.xml | 6 ++
.../apache/kyuubi/operation/SparkQueryTests.scala | 59 ++++++++++++++++++
.../apache/kyuubi/operation/UserJarTestUtils.scala | 69 +++++++++++++++++++++
kyuubi-server/pom.xml | 6 ++
8 files changed, 156 insertions(+), 1 deletion(-)
diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh
index 69a77217a..ac29ee261 100755
--- a/bin/docker-image-tool.sh
+++ b/bin/docker-image-tool.sh
@@ -85,7 +85,7 @@ function resolve_file {
function create_dev_build_context {(
set -e
local BASE_CTX="$CTX_DIR/base"
- mkdir -r "$BASE_CTX/docker"
+ mkdir -p "$BASE_CTX/docker"
cp -r "docker/" "$BASE_CTX/docker"
cp -r "kyuubi-assembly/target/scala-${KYUUBI_SCALA_VERSION}/jars"
"$BASE_CTX/jars"
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala
index 30f86db70..df3da5c43 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala
@@ -64,6 +64,9 @@ class ExecuteScala(
if (legacyOutput.nonEmpty) {
warn(s"Clearing legacy output from last interpreting:\n $legacyOutput")
}
+ val jars = spark.sharedState.jarClassLoader.getURLs
+ repl.addUrlsToClassPath(jars: _*)
+
repl.interpretWithRedirectOutError(statement) match {
case Success =>
iter = {
diff --git
a/externals/kyuubi-spark-sql-engine/src/test/resources/test-function.jar
b/externals/kyuubi-spark-sql-engine/src/test/resources/test-function.jar
new file mode 100644
index 000000000..d240f4658
Binary files /dev/null and
b/externals/kyuubi-spark-sql-engine/src/test/resources/test-function.jar differ
diff --git a/integration-tests/kyuubi-kubernetes-it/pom.xml
b/integration-tests/kyuubi-kubernetes-it/pom.xml
index 908bb83d2..1833823f2 100644
--- a/integration-tests/kyuubi-kubernetes-it/pom.xml
+++ b/integration-tests/kyuubi-kubernetes-it/pom.xml
@@ -80,6 +80,18 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<!-- for hive driver related dependency -->
<dependency>
<groupId>org.apache.kyuubi</groupId>
diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml
index 9b16527a4..020825cdc 100644
--- a/kyuubi-common/pom.xml
+++ b/kyuubi-common/pom.xml
@@ -127,6 +127,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>failureaccess</artifactId>
diff --git
a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
index 1cc55554f..ead80d98d 100644
---
a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
+++
b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
@@ -18,9 +18,11 @@
package org.apache.kyuubi.operation
import java.sql.{Date, SQLException, SQLTimeoutException, Timestamp}
+import java.util.UUID
import scala.collection.JavaConverters._
+import org.apache.commons.io.FileUtils
import org.apache.commons.lang3.StringUtils
import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq,
TFetchResultsReq, TOpenSessionReq, TStatusCode}
@@ -608,6 +610,63 @@ trait SparkQueryTests extends HiveJDBCTestHelper {
}
}
+ test("scala code with loading external package at runtime ") {
+ val jarDir = Utils.createTempDir().toFile
+
+ withJdbcStatement() { statement =>
+ statement.execute("SET kyuubi.operation.language=scala")
+ val udfCode =
+ """
+ |package test.utils
+ |
+ |object Math {
+ |def add(x: Int, y: Int): Int = x + y
+ |}
+ |
+ |""".stripMargin
+
+ val jarFile = UserJarTestUtils.createJarFile(
+ udfCode,
+ "test",
+ s"test-function-${UUID.randomUUID}.jar",
+ jarDir.toString)
+ val jarBytes = FileUtils.readFileToByteArray(jarFile)
+ val jarStr = new String(java.util.Base64.getEncoder().encode(jarBytes))
+ val jarName = s"test-function-${UUID.randomUUID}.jar"
+
+ val code0 = """spark.sql("SET kyuubi.operation.language").show(false)"""
+ statement.execute(code0)
+
+ // Generate a jar package in spark engine
+ val batchCode =
+ s"""
+ |import java.io.{BufferedOutputStream, File, FileOutputStream}
+ |val dir =
spark.sparkContext.getConf.get("spark.repl.class.outputDir")
+ |val jarFile = new File(dir, "$jarName")
+ |val bos = new BufferedOutputStream(new FileOutputStream(jarFile))
+ |val path = "$jarStr"
+ |bos.write(java.util.Base64.getDecoder.decode(path))
+ |bos.close()
+ |val jarPath = jarFile.getAbsolutePath
+ |val fileSize = jarFile.length
+ |""".stripMargin
+ batchCode.split("\n").filter(_.nonEmpty).foreach { code =>
+ val rs = statement.executeQuery(code)
+ rs.next()
+ // scalastyle:off
+ println(rs.getString(1))
+ // scalastyle:on
+ }
+
+ val code1 = s"""spark.sql("add jar " + jarPath)"""
+ val code2 = """val x = test.utils.Math.add(1,2)"""
+ statement.execute(code1)
+ val rs = statement.executeQuery(code2)
+ rs.next()
+ assert(rs.getString(1) == "x: Int = 3")
+ }
+ }
+
def sparkEngineMajorMinorVersion: (Int, Int) = {
var sparkRuntimeVer = ""
withJdbcStatement() { stmt =>
diff --git
a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/UserJarTestUtils.scala
b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/UserJarTestUtils.scala
new file mode 100644
index 000000000..cb709dcd2
--- /dev/null
+++
b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/UserJarTestUtils.scala
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.operation
+
+import java.io.{File, FileOutputStream}
+import java.nio.file.Files
+import java.util.UUID
+import java.util.jar.{JarEntry, JarOutputStream}
+
+import scala.tools.nsc.{Global, Settings}
+
+import org.apache.commons.io.FileUtils
+
+object UserJarTestUtils {
+
+ def createJarFile(
+ codeText: String,
+ packageRootPath: String,
+ jarName: String,
+ outputDir: String): File = {
+ val codeFile = new File(outputDir, s"test-${UUID.randomUUID}.scala")
+ FileUtils.writeStringToFile(codeFile, codeText, "UTF-8")
+
+ val settings = new Settings
+ settings.outputDirs.setSingleOutput(outputDir)
+ settings.usejavacp.value = true
+ val global = new Global(settings)
+ val runner = new global.Run
+ runner.compile(List(codeFile.getAbsolutePath))
+ val jarFile = new File(outputDir, jarName)
+ val targetJar = new JarOutputStream(new FileOutputStream(jarFile))
+ add(s"$outputDir/$packageRootPath", targetJar, outputDir + "/")
+ targetJar.close()
+ jarFile
+ }
+
+ private def add(folder: String, target: JarOutputStream, replacement:
String): Unit = {
+ val source = new File(folder)
+ if (source.isDirectory) {
+ for (nestedFile <- source.listFiles) {
+ add(nestedFile.getAbsolutePath, target, replacement)
+ }
+ } else {
+ val entry = new JarEntry(source.getPath
+ .replace("\\", "/")
+ .replace(replacement, ""))
+ entry.setTime(source.lastModified)
+ target.putNextEntry(entry)
+ val byteArray = Files.readAllBytes(source.toPath)
+ target.write(byteArray)
+ target.closeEntry()
+ }
+ }
+}
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index e79b36c3b..b8d8f11fc 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -428,6 +428,12 @@
<artifactId>mysql-connector-java</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
