This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new fe431b2a3 [KYUUBI #3635] Delete temp directories on exit in test
suites with Utils.createTempDir
fe431b2a3 is described below
commit fe431b2a3698dc27f167f3d142752e6b5b8afa58
Author: Bowen Liang <[email protected]>
AuthorDate: Tue Oct 18 22:19:07 2022 +0800
[KYUUBI #3635] Delete temp directories on exit in test suites with
Utils.createTempDir
### _Why are the changes needed?_
to close #3635 .
- change test suits to to use Utils.createTempDir of kyuubi-common module
to create temp dir with delete on exit to prevent disk usage leaking
- change signature Utils.createTempDir by making `prefix` param from second
place to the first, to make it friendly to use without repeating param name
itself. The name`prefix` is more closer to Java's style in
`Files.createTempDirectory`
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #3660 from bowenliang123/3635-deletetmp.
Closes #3635
8947fb0f [Bowen Liang] remove unused imports
c06769d4 [Bowen Liang] nit
dac266c9 [Bowen Liang] - change test suits to to use Utils.createTempDir of
kyuubi-common module to create temp dir with delete on exit to prevent disk
usage leaking - change signature Utils.createTempDir by making prefix param
from second place to the first, to make it friendly to use without repeating
param name itself. The nameprefix is more closer to Java's style in
Files.createTempDirectory
Authored-by: Bowen Liang <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
extensions/spark/kyuubi-spark-authz/pom.xml | 7 +++++++
.../apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala | 5 +++--
.../authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala | 4 ++--
.../scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala | 6 ++----
.../apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala | 4 ++--
.../main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala | 2 +-
.../engine/hive/operation/HiveCatalogDatabaseOperationSuite.scala | 2 +-
.../apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala | 2 +-
.../main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala | 2 +-
.../operation/KyuubiOperationHiveEnginePerConnectionSuite.scala | 2 +-
.../it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala | 2 +-
kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala | 6 +++---
.../src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala | 5 ++---
.../src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala | 4 ++--
.../org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala | 6 +++---
15 files changed, 32 insertions(+), 27 deletions(-)
diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml
b/extensions/spark/kyuubi-spark-authz/pom.xml
index cf61281d4..45513a83b 100644
--- a/extensions/spark/kyuubi-spark-authz/pom.xml
+++ b/extensions/spark/kyuubi-spark-authz/pom.xml
@@ -253,6 +253,13 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.kyuubi</groupId>
+ <artifactId>kyuubi-common_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala
index eb2b77959..06ab61f38 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala
@@ -21,6 +21,7 @@ import java.nio.file.Files
import org.apache.spark.sql.{DataFrame, SparkSession, SparkSessionExtensions}
+import org.apache.kyuubi.Utils
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
trait SparkSessionProvider {
@@ -36,7 +37,7 @@ trait SparkSessionProvider {
protected lazy val spark: SparkSession = {
val metastore = {
- val path = Files.createTempDirectory("hms")
+ val path = Utils.createTempDir(prefix = "hms")
Files.delete(path)
path
}
@@ -47,7 +48,7 @@ trait SparkSessionProvider {
.config("spark.sql.catalogImplementation", catalogImpl)
.config(
"spark.sql.warehouse.dir",
- Files.createTempDirectory("spark-warehouse").toString)
+ Utils.createTempDir("spark-warehouse").toString)
.config("spark.sql.extensions", sqlExtensions)
.withExtensions(extension)
.getOrCreate()
diff --git
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala
index fe2902ada..531af4d3e 100644
---
a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala
+++
b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala
@@ -17,8 +17,8 @@
package org.apache.kyuubi.plugin.spark.authz.ranger
// scalastyle:off
-import java.nio.file.Files
+import org.apache.kyuubi.Utils
import org.apache.kyuubi.plugin.spark.authz.AccessControlException
/**
@@ -45,7 +45,7 @@ class IcebergCatalogRangerSparkExtensionSuite extends
RangerSparkExtensionSuite
spark.conf.set(s"spark.sql.catalog.$catalogV2.type", "hadoop")
spark.conf.set(
s"spark.sql.catalog.$catalogV2.warehouse",
- Files.createTempDirectory("iceberg-hadoop").toString)
+ Utils.createTempDir("iceberg-hadoop").toString)
super.beforeAll()
diff --git
a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala
b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala
index 67a9618b6..fbfb8df29 100644
---
a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala
+++
b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala
@@ -17,8 +17,6 @@
package org.apache.kyuubi.engine.flink
-import java.nio.file.Files
-
import scala.collection.JavaConverters._
import org.apache.flink.client.cli.{CustomCommandLine, DefaultCLI}
@@ -26,7 +24,7 @@ import org.apache.flink.configuration.{Configuration,
RestOptions}
import org.apache.flink.runtime.minicluster.{MiniCluster,
MiniClusterConfiguration}
import org.apache.flink.table.client.gateway.context.DefaultContext
-import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.{KyuubiFunSuite, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.flink.util.TestUserClassLoaderJar
@@ -70,7 +68,7 @@ trait WithFlinkSQLEngine extends KyuubiFunSuite {
kyuubiConf.set(k, v)
}
val udfJar = TestUserClassLoaderJar.createJarFile(
- Files.createTempDirectory("test-jar").toFile,
+ Utils.createTempDir("test-jar").toFile,
"test-classloader-udf.jar",
GENERATED_UDF_CLASS,
GENERATED_UDF_CODE)
diff --git
a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala
b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala
index 660c9b0e7..a7bc2428e 100644
---
a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala
+++
b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala
@@ -17,7 +17,6 @@
package org.apache.kyuubi.engine.flink.operation
-import java.nio.file.Files
import java.sql.DatabaseMetaData
import java.util.UUID
@@ -29,6 +28,7 @@ import org.apache.hive.service.rpc.thrift._
import org.scalatest.concurrent.PatienceConfiguration.Timeout
import org.scalatest.time.SpanSugar._
+import org.apache.kyuubi.Utils
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.engine.flink.FlinkEngineUtils._
import org.apache.kyuubi.engine.flink.WithFlinkSQLEngine
@@ -999,7 +999,7 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with
HiveJDBCTestHelper {
test("execute statement - add/remove/show jar") {
val jarName = s"newly-added-${UUID.randomUUID()}.jar"
val newJar = TestUserClassLoaderJar.createJarFile(
- Files.createTempDirectory("add-jar-test").toFile,
+ Utils.createTempDir("add-jar-test").toFile,
jarName,
GENERATED_UDF_CLASS,
GENERATED_UDF_CODE).toPath
diff --git
a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala
b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala
index 73132d5fc..839da710e 100644
---
a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala
+++
b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala
@@ -92,7 +92,7 @@ object HiveSQLEngine extends Logging {
hiveConf.setBoolean("datanucleus.schema.autoCreateAll", true)
hiveConf.set(
"hive.metastore.warehouse.dir",
- Utils.createTempDir(namePrefix = "kyuubi_hive_warehouse").toString)
+ Utils.createTempDir(prefix = "kyuubi_hive_warehouse").toString)
hiveConf.set("hive.metastore.fastpath", "true")
}
diff --git
a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveCatalogDatabaseOperationSuite.scala
b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveCatalogDatabaseOperationSuite.scala
index aa95c4df6..a63de20c7 100644
---
a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveCatalogDatabaseOperationSuite.scala
+++
b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveCatalogDatabaseOperationSuite.scala
@@ -27,7 +27,7 @@ import org.apache.kyuubi.operation.HiveJDBCTestHelper
class HiveCatalogDatabaseOperationSuite extends HiveJDBCTestHelper {
override def beforeAll(): Unit = {
- val metastore = Utils.createTempDir(namePrefix = getClass.getSimpleName)
+ val metastore = Utils.createTempDir(prefix = getClass.getSimpleName)
metastore.toFile.delete()
val args = Array(
"--conf",
diff --git
a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala
b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala
index 3595b35c2..f949ec37a 100644
---
a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala
+++
b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala
@@ -26,7 +26,7 @@ import org.apache.kyuubi.jdbc.hive.KyuubiStatement
class HiveOperationSuite extends HiveEngineTests {
override def beforeAll(): Unit = {
- val metastore = Utils.createTempDir(namePrefix = getClass.getSimpleName)
+ val metastore = Utils.createTempDir(prefix = getClass.getSimpleName)
metastore.toFile.delete()
val args = Array(
"--conf",
diff --git
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
index 3462c67ce..ff96ef4f9 100644
---
a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
+++
b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
@@ -143,7 +143,7 @@ object SparkSQLEngine extends Logging {
_sparkConf = new SparkConf()
_kyuubiConf = KyuubiConf()
val rootDir =
_sparkConf.getOption("spark.repl.classdir").getOrElse(getLocalDir(_sparkConf))
- val outputDir = Utils.createTempDir(root = rootDir, namePrefix = "repl")
+ val outputDir = Utils.createTempDir(prefix = "repl", root = rootDir)
_sparkConf.setIfMissing("spark.sql.execution.topKSortFallbackThreshold",
"10000")
_sparkConf.setIfMissing("spark.sql.legacy.castComplexTypesToString.enabled",
"true")
_sparkConf.setIfMissing("spark.master", "local")
diff --git
a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerConnectionSuite.scala
b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerConnectionSuite.scala
index 08ce79e42..ae160f0c7 100644
---
a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerConnectionSuite.scala
+++
b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerConnectionSuite.scala
@@ -29,7 +29,7 @@ class KyuubiOperationHiveEnginePerConnectionSuite extends
WithKyuubiServer with
val kyuubiHome: String =
Utils.getCodeSourceLocation(getClass).split("integration-tests").head
override protected val conf: KyuubiConf = {
- val metastore = Utils.createTempDir(namePrefix = getClass.getSimpleName)
+ val metastore = Utils.createTempDir(prefix = getClass.getSimpleName)
metastore.toFile.delete()
val currentUser = Utils.currentUser
KyuubiConf()
diff --git
a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala
b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala
index de4b9f5cb..556dbea01 100644
---
a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala
+++
b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala
@@ -28,7 +28,7 @@ class KyuubiOperationHiveEnginePerUserSuite extends
WithKyuubiServer with HiveEn
val kyuubiHome: String =
Utils.getCodeSourceLocation(getClass).split("integration-tests").head
override protected val conf: KyuubiConf = {
- val metastore = Utils.createTempDir(namePrefix = getClass.getSimpleName)
+ val metastore = Utils.createTempDir(prefix = getClass.getSimpleName)
metastore.toFile.delete()
KyuubiConf()
.set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
index b79038b2f..19b84d7d6 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
@@ -136,9 +136,9 @@ object Utils extends Logging {
* automatically deleted when the VM shuts down.
*/
def createTempDir(
- root: String = System.getProperty("java.io.tmpdir"),
- namePrefix: String = "kyuubi"): Path = {
- val dir = createDirectory(root, namePrefix)
+ prefix: String = "kyuubi",
+ root: String = System.getProperty("java.io.tmpdir")): Path = {
+ val dir = createDirectory(root, prefix)
dir.toFile.deleteOnExit()
dir
}
diff --git
a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
index 6bca3244a..d877f1d52 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KerberizedTestHelper.scala
@@ -34,9 +34,8 @@ import org.ietf.jgss.{GSSContext, GSSException, GSSManager,
GSSName}
import org.scalatest.time.SpanSugar._
trait KerberizedTestHelper extends KyuubiFunSuite {
- val baseDir: File = Utils.createTempDir(
- Utils.getCodeSourceLocation(getClass),
- "kyuubi-kdc").toFile
+ val baseDir: File =
+ Utils.createTempDir("kyuubi-kdc",
Utils.getCodeSourceLocation(getClass)).toFile
val kdcConf = MiniKdc.createConf()
val hostName = "localhost"
kdcConf.setProperty(MiniKdc.INSTANCE, this.getClass.getSimpleName)
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
index 65c457601..7dc295ce5 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/ProcBuilder.scala
@@ -125,10 +125,10 @@ trait ProcBuilder {
if (Files.isDirectory(working)) {
working
} else {
- Utils.createTempDir(rootAbs, proxyUser)
+ Utils.createTempDir(proxyUser, rootAbs)
}
}.getOrElse {
- Utils.createTempDir(namePrefix = proxyUser)
+ Utils.createTempDir(prefix = proxyUser)
}
}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
index a4f33dde1..5ae03f545 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
@@ -151,7 +151,7 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper
with MockitoSugar {
test(s"sub process log should be overwritten") {
def atomicTest(): Unit = {
val pool = Executors.newFixedThreadPool(3)
- val fakeWorkDir = Files.createTempDirectory("fake")
+ val fakeWorkDir = Utils.createTempDir("fake")
val dir = fakeWorkDir.toFile
try {
assert(dir.list().length == 0)
@@ -197,7 +197,7 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper
with MockitoSugar {
}
test("overwrite log file should cleanup before write") {
- val fakeWorkDir = Files.createTempDirectory("fake")
+ val fakeWorkDir = Utils.createTempDir("fake")
val conf = KyuubiConf()
conf.set(ENGINE_LOG_TIMEOUT, Duration.ofDays(1).toMillis)
val builder1 = new FakeSparkProcessBuilder(conf) {
@@ -220,7 +220,7 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper
with MockitoSugar {
}
test("main resource jar should not check when is not a local file") {
- val workDir = Files.createTempDirectory("resource")
+ val workDir = Utils.createTempDir("resource")
val jarPath = Paths.get(workDir.toString, "test.jar")
val hdfsPath = s"hdfs://$jarPath"
