This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 5a20267db [KYUUBI #6523] Support configuration with `--conf` options
5a20267db is described below
commit 5a20267dbc115a8848e5ab167fa7c400dd3a07e2
Author: dnskr <[email protected]>
AuthorDate: Thu Jul 18 11:33:17 2024 +0800
[KYUUBI #6523] Support configuration with `--conf` options
# :mag: Description
## Issue References ๐
This pull request fixes https://github.com/apache/kyuubi/issues/6523
## Describe Your Solution ๐ง
`KyuubiServer` respects configuration properties provided as `--conf` cli
options with highest priority.
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
#### Behavior Without This Pull Request :coffin:
REST endpoint starts at `10099` port:
```shell
bin/kyuubi run --conf kyuubi.frontend.rest.bind.port=9999
...
2024-07-10 17:57:24.998 INFO main
org.apache.kyuubi.server.KyuubiRestFrontendService:
Service[KyuubiRestFrontendService] is started.
2024-07-10 17:57:24.999 INFO main
org.apache.kyuubi.server.KyuubiRestFrontendService: Exposing REST endpoint at:
http://0.0.0.0:10099
2024-07-10 17:57:24.999 INFO main org.apache.kyuubi.server.KyuubiServer:
Service[KyuubiServer] is started.
```
#### Behavior With This Pull Request :tada:
REST endpoint starts at `9999` port:
```shell
bin/kyuubi run --conf kyuubi.frontend.rest.bind.port=9999
...
2024-07-10 17:49:41.659 INFO main
org.apache.kyuubi.server.KyuubiRestFrontendService:
Service[KyuubiRestFrontendService] is started.
2024-07-10 17:49:41.660 INFO main
org.apache.kyuubi.server.KyuubiRestFrontendService: Exposing REST endpoint at:
http://0.0.0.0:9999
2024-07-10 17:49:41.660 INFO main org.apache.kyuubi.server.KyuubiServer:
Service[KyuubiServer] is started.
```
#### Related Unit Tests
`kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala`
---
# Checklist ๐
- [ ] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6530 from dnskr/kyuubi-supports-conf-args.
Closes #6523
c7dbb305b [dnskr] Parse command and shift args
4977af168 [dnskr] [KYUUBI #6523] Support configuration with --conf options
Authored-by: dnskr <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
bin/kyuubi | 9 +++++++--
.../scala/org/apache/kyuubi/config/KyuubiConf.scala | 5 +++++
.../org/apache/kyuubi/config/KyuubiConfSuite.scala | 9 +++++++++
.../org/apache/kyuubi/server/KyuubiServer.scala | 21 ++++++++++++++-------
4 files changed, 35 insertions(+), 9 deletions(-)
diff --git a/bin/kyuubi b/bin/kyuubi
index 17ab717e5..76666a276 100755
--- a/bin/kyuubi
+++ b/bin/kyuubi
@@ -102,7 +102,12 @@ if [[ -n ${YARN_CONF_DIR} ]]; then
KYUUBI_CLASSPATH="${KYUUBI_CLASSPATH}:${YARN_CONF_DIR}"
fi
-cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS"
+if [[ "$1" =~ ^(start|restart|run|stop|status)$ ]]; then
+ command=$1
+ shift
+fi
+
+cmd="${RUNNER} ${KYUUBI_JAVA_OPTS} -cp ${KYUUBI_CLASSPATH} $CLASS $@"
pid="${KYUUBI_PID_DIR}/kyuubi-$USER-$CLASS.pid"
@@ -218,7 +223,7 @@ function check_kyuubi() {
fi
}
-case $1 in
+case $command in
(start | "")
start_kyuubi
;;
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index fcbd1c9e6..05940aee7 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -53,6 +53,11 @@ case class KyuubiConf(loadSysDefault: Boolean = true)
extends Logging {
this
}
+ def loadFromArgs(args: Array[String]): KyuubiConf = {
+ Utils.fromCommandLineArgs(args, this)
+ this
+ }
+
def set[T](entry: ConfigEntry[T], value: T): KyuubiConf = {
require(entry != null, "entry cannot be null")
require(value != null, s"value cannot be null for key: ${entry.key}")
diff --git
a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
index 39e68f0ec..e5c8b901a 100644
---
a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
+++
b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala
@@ -45,6 +45,15 @@ class KyuubiConfSuite extends KyuubiFunSuite {
assert(conf.getOption("spark.kyuubi.yes").get === "no")
}
+ test("load config from --conf arguments") {
+ val conf = KyuubiConf()
+ assert(conf.get(KINIT_MAX_ATTEMPTS) === 10)
+
+ val args: Array[String] = Array("--conf", "kyuubi.kinit.max.attempts=15")
+ conf.loadFromArgs(args)
+ assert(conf.get(KINIT_MAX_ATTEMPTS) === 15)
+ }
+
test("set and unset conf") {
val conf = new KyuubiConf(false)
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
index b5f135339..ace7ba9d4 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiServer.scala
@@ -40,6 +40,7 @@ import org.apache.kyuubi.zookeeper.EmbeddedZookeeper
object KyuubiServer extends Logging {
private[kyuubi] var kyuubiServer: KyuubiServer = _
@volatile private[kyuubi] var hadoopConf: Configuration = _
+ private var commandArgs: Array[String] = Array.empty[String]
def startServer(conf: KyuubiConf): KyuubiServer = {
hadoopConf = KyuubiHadoopUtils.newHadoopConf(conf)
@@ -93,9 +94,11 @@ object KyuubiServer extends Logging {
s" ${Properties.javaVersion}")
SignalRegister.registerLogger(logger)
+ commandArgs = args
+
// register conf entries
JDBCMetadataStoreConf
- val conf = new KyuubiConf().loadFileDefaults()
+ val conf = createKyuubiConf()
UserGroupInformation.setConfiguration(KyuubiHadoopUtils.newHadoopConf(conf))
startServer(conf)
}
@@ -105,13 +108,13 @@ object KyuubiServer extends Logging {
}
private[kyuubi] def reloadHadoopConf(): Unit = synchronized {
- val _hadoopConf = KyuubiHadoopUtils.newHadoopConf(new
KyuubiConf().loadFileDefaults())
+ val _hadoopConf = KyuubiHadoopUtils.newHadoopConf(createKyuubiConf())
hadoopConf = _hadoopConf
}
private[kyuubi] def refreshUserDefaultsConf(): Unit =
kyuubiServer.conf.synchronized {
val existedUserDefaults = kyuubiServer.conf.getAllUserDefaults
- val refreshedUserDefaults =
KyuubiConf().loadFileDefaults().getAllUserDefaults
+ val refreshedUserDefaults = createKyuubiConf().getAllUserDefaults
refreshConfig("user defaults", existedUserDefaults, refreshedUserDefaults)
}
@@ -119,7 +122,7 @@ object KyuubiServer extends Logging {
val existedKubernetesConf =
kyuubiServer.conf.getAll.filter(_._1.startsWith(KYUUBI_KUBERNETES_CONF_PREFIX))
val refreshedKubernetesConf =
-
KyuubiConf().loadFileDefaults().getAll.filter(_._1.startsWith(KYUUBI_KUBERNETES_CONF_PREFIX))
+
createKyuubiConf().getAll.filter(_._1.startsWith(KYUUBI_KUBERNETES_CONF_PREFIX))
refreshConfig("kubernetes", existedKubernetesConf, refreshedKubernetesConf)
}
@@ -149,7 +152,7 @@ object KyuubiServer extends Logging {
private[kyuubi] def refreshUnlimitedUsers(): Unit = synchronized {
val sessionMgr =
kyuubiServer.backendService.sessionManager.asInstanceOf[KyuubiSessionManager]
val existingUnlimitedUsers = sessionMgr.getUnlimitedUsers
- sessionMgr.refreshUnlimitedUsers(KyuubiConf().loadFileDefaults())
+ sessionMgr.refreshUnlimitedUsers(createKyuubiConf())
val refreshedUnlimitedUsers = sessionMgr.getUnlimitedUsers
info(s"Refreshed unlimited users from $existingUnlimitedUsers to
$refreshedUnlimitedUsers")
}
@@ -157,7 +160,7 @@ object KyuubiServer extends Logging {
private[kyuubi] def refreshDenyUsers(): Unit = synchronized {
val sessionMgr =
kyuubiServer.backendService.sessionManager.asInstanceOf[KyuubiSessionManager]
val existingDenyUsers = sessionMgr.getDenyUsers
- sessionMgr.refreshDenyUsers(KyuubiConf().loadFileDefaults())
+ sessionMgr.refreshDenyUsers(createKyuubiConf())
val refreshedDenyUsers = sessionMgr.getDenyUsers
info(s"Refreshed deny users from $existingDenyUsers to
$refreshedDenyUsers")
}
@@ -165,10 +168,14 @@ object KyuubiServer extends Logging {
private[kyuubi] def refreshDenyIps(): Unit = synchronized {
val sessionMgr =
kyuubiServer.backendService.sessionManager.asInstanceOf[KyuubiSessionManager]
val existingDenyIps = sessionMgr.getDenyIps
- sessionMgr.refreshDenyIps(KyuubiConf().loadFileDefaults())
+ sessionMgr.refreshDenyIps(createKyuubiConf())
val refreshedDenyIps = sessionMgr.getDenyIps
info(s"Refreshed deny client ips from $existingDenyIps to
$refreshedDenyIps")
}
+
+ private def createKyuubiConf(): KyuubiConf = {
+ KyuubiConf().loadFileDefaults().loadFromArgs(commandArgs)
+ }
}
class KyuubiServer(name: String) extends Serverable(name) {
