This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d375eb0 [KYUUBI #1010] Rename engine.share.level.sub.domain to
engine.share.level.subdomain
d375eb0 is described below
commit d375eb0cfc9b26815ad8f2ee1f37d801b3289468
Author: Kent Yao <[email protected]>
AuthorDate: Thu Sep 2 11:46:07 2021 +0800
[KYUUBI #1010] Rename engine.share.level.sub.domain to
engine.share.level.subdomain
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines:
https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in
https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your
PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g.,
'[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
`subdomain` is a widely used term in computer science
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #1010 from yaooqinn/subdomain.
Closes #1010
72ed19a3 [Kent Yao] address comments
4dcae414 [Kent Yao] fix test
c3bb788d [Kent Yao] Rename engine.share.level.sub.domain to
engine.share.level.subdomain
b7762357 [Kent Yao] Rename engine.share.level.sub.domain to
engine.share.level.subdomain
Authored-by: Kent Yao <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
---
docs/deployment/settings.md | 5 ++-
.../org/apache/kyuubi/config/KyuubiConf.scala | 19 ++++++---
.../scala/org/apache/kyuubi/engine/EngineRef.scala | 12 +++---
.../org/apache/kyuubi/engine/EngineRefSuite.scala | 48 +++++++++++++---------
.../kyuubi/engine/spark/SparkSqlEngineSuite.scala | 8 ++--
.../operation/KyuubiOperationPerUserSuite.scala | 2 +-
6 files changed, 55 insertions(+), 39 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 6237227..be9a37f 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -172,8 +172,9 @@ kyuubi\.engine<br>\.initialize\.sql|<div style='width:
65pt;word-wrap: break-wor
kyuubi\.engine\.pool<br>\.size|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'>-1</div>|<div style='width: 170pt;word-wrap:
break-word;white-space: normal'>The size of engine pool. Note that, if the size
is less than 1, the engine pool will not be enabled; otherwise, the size of the
engine pool will be min(this, kyuubi.engine.pool.size.threshold).</div>|<div
style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
kyuubi\.engine\.pool<br>\.size\.threshold|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'>9</div>|<div style='width: 170pt;word-wrap:
break-word;white-space: normal'>This parameter is introduced as a server-side
parameter, and controls the upper limit of the engine pool.</div>|<div
style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
kyuubi\.engine\.session<br>\.initialize\.sql|<div style='width:
65pt;word-wrap: break-word;white-space: normal'>SHOW DATABASES</div>|<div
style='width: 170pt;word-wrap: break-word;white-space:
normal'>SemiColon-separated list of SQL statements to be initialized in the
newly created engine session before queries. This configuration can not be used
in JDBC url due to the limitation of Beeline/JDBC driver.</div>|<div
style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.3.0</div>
-kyuubi\.engine\.share<br>\.level|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'>USER</div>|<div style='width: 170pt;word-wrap:
break-word;white-space: normal'>Engines will be shared in different levels,
available configs are: <ul> <li>CONNECTION: engine will not be shared but only
used by the current client connection</li> <li>USER: engine will be shared by
all sessions created by a unique username, see also
kyuubi.engine.share.level.sub.domain</li> <li>SERVER: the Ap [...]
-kyuubi\.engine\.share<br>\.level\.sub\.domain|<div style='width:
65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div
style='width: 170pt;word-wrap: break-word;white-space: normal'>Allow end-users
to create a sub-domain for the share level of an engine. A sub-domain is a
case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form. For example, for
`USER` share level, an end-user can share a certain engine within a sub-domain,
not for all of its clients. End-users [...]
+kyuubi\.engine\.share<br>\.level|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'>USER</div>|<div style='width: 170pt;word-wrap:
break-word;white-space: normal'>Engines will be shared in different levels,
available configs are: <ul> <li>CONNECTION: engine will not be shared but only
used by the current client connection</li> <li>USER: engine will be shared by
all sessions created by a unique username, see also
kyuubi.engine.share.level.subdomain</li> <li>SERVER: the App [...]
+kyuubi\.engine\.share<br>\.level\.sub\.domain|<div style='width:
65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div
style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) -
Using kyuubi.engine.share.level.subdomain instead</div>|<div style='width:
30pt'>string</div>|<div style='width: 20pt'>1.2.0</div>
+kyuubi\.engine\.share<br>\.level\.subdomain|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'><undefined></div>|<div style='width:
170pt;word-wrap: break-word;white-space: normal'>Allow end-users to create a
subdomain for the share level of an engine. A subdomain is a case-insensitive
string values in `^[a-zA-Z_-]{1,14}$` form. For example, for `USER` share
level, an end-user can share a certain engine within a subdomain, not for all
of its clients. End-users are f [...]
kyuubi\.engine\.single<br>\.spark\.session|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'>false</div>|<div style='width: 170pt;word-wrap:
break-word;white-space: normal'>When set to true, this engine is running in a
single session mode. All the JDBC/ODBC connections share the temporary views,
function registries, SQL configuration and the current database.</div>|<div
style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.3.0</div>
kyuubi\.engine\.ui\.stop<br>\.enabled|<div style='width: 65pt;word-wrap:
break-word;white-space: normal'>true</div>|<div style='width: 170pt;word-wrap:
break-word;white-space: normal'>When true, allows Kyuubi engine to be killed
from the Spark Web UI.</div>|<div style='width: 30pt'>boolean</div>|<div
style='width: 20pt'>1.3.0</div>
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index ae5cd22..48da7ea 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -574,13 +574,10 @@ object KyuubiConf {
private val validEngineSubDomain: Pattern = "^[a-zA-Z_-]{1,14}$".r.pattern
+ @deprecated(s"using kyuubi.engine.share.level.subdomain instead", "1.4.0")
val ENGINE_SHARE_LEVEL_SUB_DOMAIN: OptionalConfigEntry[String] =
buildConf("engine.share.level.sub.domain")
- .doc("Allow end-users to create a sub-domain for the share level of an
engine. A" +
- " sub-domain is a case-insensitive string values in
`^[a-zA-Z_-]{1,14}$` form." +
- " For example, for `USER` share level, an end-user can share a certain
engine within" +
- " a sub-domain, not for all of its clients. End-users are free to
create multiple" +
- " engines in the `USER` share level")
+ .doc("(deprecated) - Using kyuubi.engine.share.level.subdomain instead")
.version("1.2.0")
.stringConf
.transform(_.toLowerCase(Locale.ROOT))
@@ -588,6 +585,16 @@ object KyuubiConf {
"must be [1, 14] length alphabet string, e.g. 'abc', 'apache'")
.createOptional
+ val ENGINE_SHARE_LEVEL_SUBDOMAIN: ConfigEntry[Option[String]] =
+ buildConf("engine.share.level.subdomain")
+ .doc("Allow end-users to create a subdomain for the share level of an
engine. A" +
+ " subdomain is a case-insensitive string values in
`^[a-zA-Z_-]{1,14}$` form." +
+ " For example, for `USER` share level, an end-user can share a certain
engine within" +
+ " a subdomain, not for all of its clients. End-users are free to
create multiple" +
+ " engines in the `USER` share level")
+ .version("1.4.0")
+ .fallbackConf(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+
val ENGINE_CONNECTION_URL_USE_HOSTNAME: ConfigEntry[Boolean] =
buildConf("engine.connection.url.use.hostname")
.doc("When true, engine register with hostname to zookeeper. When spark
run on k8s" +
@@ -601,7 +608,7 @@ object KyuubiConf {
" <li>CONNECTION: engine will not be shared but only used by the current
client" +
" connection</li>" +
" <li>USER: engine will be shared by all sessions created by a unique
username," +
- s" see also ${ENGINE_SHARE_LEVEL_SUB_DOMAIN.key}</li>" +
+ s" see also ${ENGINE_SHARE_LEVEL_SUBDOMAIN.key}</li>" +
" <li>SERVER: the App will be shared by Kyuubi servers</li></ul>")
.version("1.2.0")
.fallbackConf(LEGACY_ENGINE_SHARE_LEVEL)
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
index 0eecccb..219dcf8 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
@@ -62,7 +62,7 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf,
user: String, sessionI
private val poolThreshold: Int = conf.get(ENGINE_POOL_SIZE_THRESHOLD)
@VisibleForTesting
- private[kyuubi] val subDomain: Option[String] =
conf.get(ENGINE_SHARE_LEVEL_SUB_DOMAIN).orElse {
+ private[kyuubi] val subdomain: Option[String] =
conf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN).orElse {
val clientPoolSize: Int = conf.get(ENGINE_POOL_SIZE)
if (clientPoolSize > 0) {
@@ -94,7 +94,7 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf,
user: String, sessionI
@VisibleForTesting
private[kyuubi] val defaultEngineName: String = shareLevel match {
case CONNECTION => s"kyuubi_${shareLevel}_${appUser}_$sessionId"
- case _ => subDomain match {
+ case _ => subdomain match {
case Some(domain) =>
s"kyuubi_${shareLevel}_${appUser}_${domain}_$sessionId"
case _ => s"kyuubi_${shareLevel}_${appUser}_$sessionId"
}
@@ -104,15 +104,15 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf,
user: String, sessionI
* The EngineSpace used to expose itself to the KyuubiServers in
`serverSpace`
*
* For `CONNECTION` share level:
- * /$serverSpace_CONNECTION/$user/$sessionId
+ * /`serverSpace_CONNECTION`/`user`/`sessionId`
* For `USER` share level:
- * /$serverSpace_USER/$user[/$subDomain]
+ * /`serverSpace_USER`/`user`[/`subdomain`]
*
*/
@VisibleForTesting
private[kyuubi] lazy val engineSpace: String = shareLevel match {
case CONNECTION => ZKPaths.makePath(s"${serverSpace}_$shareLevel",
appUser, sessionId)
- case _ => subDomain match {
+ case _ => subdomain match {
case Some(domain) => ZKPaths.makePath(s"${serverSpace}_$shareLevel",
appUser, domain)
case None => ZKPaths.makePath(s"${serverSpace}_$shareLevel", appUser)
}
@@ -126,7 +126,7 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf,
user: String, sessionI
case CONNECTION => f
case _ =>
val lockPath =
- ZKPaths.makePath(s"${serverSpace}_$shareLevel", "lock", appUser,
subDomain.orNull)
+ ZKPaths.makePath(s"${serverSpace}_$shareLevel", "lock", appUser,
subdomain.orNull)
var lock: InterProcessSemaphoreMutex = null
try {
try {
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
index 78d0127..066d496 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
@@ -51,11 +51,17 @@ class EngineRefSuite extends KyuubiFunSuite {
super.afterAll()
}
- test(s"${CONNECTION} shared level engine name") {
+ override def beforeEach(): Unit = {
+ conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN)
+ conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ super.beforeEach()
+ }
+
+ test("CONNECTION shared level engine name") {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
Seq(None, Some("suffix")).foreach { domain =>
conf.set(KyuubiConf.ENGINE_SHARE_LEVEL, CONNECTION.toString)
- domain.foreach(conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key, _))
+ domain.foreach(conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key, _))
val engine = EngineRef(conf, user, id)
assert(engine.engineSpace ===
ZKPaths.makePath(s"kyuubi_$CONNECTION", user, id.identifier.toString))
@@ -63,31 +69,33 @@ class EngineRefSuite extends KyuubiFunSuite {
}
}
- test(s"${USER} shared level engine name") {
+ test("USER shared level engine name") {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
conf.set(KyuubiConf.ENGINE_SHARE_LEVEL, USER.toString)
- conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN)
val appName = EngineRef(conf, user, id)
assert(appName.engineSpace === ZKPaths.makePath(s"kyuubi_$USER", user))
assert(appName.defaultEngineName ===
s"kyuubi_${USER}_${user}_${id.identifier}")
- conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key, "abc")
- val appName2 = EngineRef(conf, user, id)
- assert(appName2.engineSpace ===
- ZKPaths.makePath(s"kyuubi_$USER", user, "abc"))
- assert(appName2.defaultEngineName ===
s"kyuubi_${USER}_${user}_abc_${id.identifier}")
+ Seq(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN,
+ KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN).foreach { k =>
+ conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN)
+ conf.set(k.key, "abc")
+ val appName2 = EngineRef(conf, user, id)
+ assert(appName2.engineSpace ===
+ ZKPaths.makePath(s"kyuubi_$USER", user, "abc"))
+ assert(appName2.defaultEngineName ===
s"kyuubi_${USER}_${user}_abc_${id.identifier}")
+ }
}
- test(s"${SERVER} shared level engine name") {
+ test("SERVER shared level engine name") {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
conf.set(KyuubiConf.ENGINE_SHARE_LEVEL, SERVER.toString)
- conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN)
val appName = EngineRef(conf, user, id)
assert(appName.engineSpace ===
ZKPaths.makePath(s"kyuubi_$SERVER", user))
assert(appName.defaultEngineName ===
s"kyuubi_${SERVER}_${user}_${id.identifier}")
- conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key, "abc")
+ conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "abc")
val appName2 = EngineRef(conf, user, id)
assert(appName2.engineSpace ===
ZKPaths.makePath(s"kyuubi_$SERVER", user, "abc"))
@@ -98,26 +106,26 @@ class EngineRefSuite extends KyuubiFunSuite {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
// test subdomain
- conf.set(ENGINE_SHARE_LEVEL_SUB_DOMAIN, "abc")
+ conf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "abc")
val engine1 = EngineRef(conf, user, id)
- assert(engine1.subDomain === Some("abc"))
+ assert(engine1.subdomain === Some("abc"))
// unset domain
- conf.unset(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ conf.unset(ENGINE_SHARE_LEVEL_SUBDOMAIN)
val engine2 = EngineRef(conf, user, id)
- assert(engine2.subDomain === None)
+ assert(engine2.subdomain === None)
// 1 <= engine pool size < threshold
- conf.unset(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ conf.unset(ENGINE_SHARE_LEVEL_SUBDOMAIN)
conf.set(ENGINE_POOL_SIZE, 3)
val engine3 = EngineRef(conf, user, id)
- assert(engine3.subDomain.get.startsWith("engine-pool-"))
+ assert(engine3.subdomain.get.startsWith("engine-pool-"))
// engine pool size > threshold
- conf.unset(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ conf.unset(ENGINE_SHARE_LEVEL_SUBDOMAIN)
conf.set(ENGINE_POOL_SIZE, 100)
val engine4 = EngineRef(conf, user, id)
- val engineNumber = Integer.parseInt(engine4.subDomain.get.substring(12))
+ val engineNumber = Integer.parseInt(engine4.subdomain.get.substring(12))
val threshold = ENGINE_POOL_SIZE_THRESHOLD.defaultVal.get
assert(engineNumber <= threshold)
}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
index 57a4554..7cd1004 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
@@ -75,7 +75,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with
JDBCTestUtils {
Seq("1", ",", "", "a" * 15, "abc.xyz").foreach { invalid =>
val sparkHiveConfigs = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> invalid)
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> invalid)
withSessionConf(Map.empty)(sparkHiveConfigs)(Map.empty) {
assertJDBCConnectionFail()
}
@@ -85,7 +85,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with
JDBCTestUtils {
test("Engine isolation with sub domain configurations") {
val sparkHiveConfigs = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "spark",
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "spark",
"spark.driver.memory" -> "1000M")
var mem: String = null
withSessionConf(Map.empty)(sparkHiveConfigs)(Map.empty) {
@@ -99,7 +99,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with
JDBCTestUtils {
val sparkHiveConfigs2 = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "spark",
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "spark",
"spark.driver.memory" -> "1001M")
withSessionConf(Map.empty)(sparkHiveConfigs2)(Map.empty) {
withJdbcStatement() { statement =>
@@ -112,7 +112,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with
JDBCTestUtils {
val sparkHiveConfigs3 = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "kyuubi",
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "kyuubi",
"spark.driver.memory" -> "1002M")
withSessionConf(Map.empty)(sparkHiveConfigs3)(Map.empty) {
withJdbcStatement() { statement =>
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
index 4e6de55..bdd41d2 100644
---
a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
+++
b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
@@ -59,7 +59,7 @@ class KyuubiOperationPerUserSuite extends WithKyuubiServer
with JDBCTests {
test("ensure two connections share the same engine when specifying
subDomain.") {
withSessionConf()(
Map(
- KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "abc"
+ KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "abc"
))(Map.empty) {
var r1: String = null
