This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 2fee652ef [KYUUBI #5201] Allow disabling batch impl v2 on globally
2fee652ef is described below
commit 2fee652efc02ef7c2dee645bb6ba7f16abb2ffcd
Author: Cheng Pan <[email protected]>
AuthorDate: Sun Aug 27 23:16:23 2023 +0800
[KYUUBI #5201] Allow disabling batch impl v2 on globally
### _Why are the changes needed?_
It makes no sense to enable batch impl v2 when
`kyuubi.batch.submitter.enabled` is disabled, to avoid misuse, globally disable
batch impl v2 in such case.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
No.
Closes #5201 from pan3793/batch-v2-switch.
Closes #5201
2cc24032f [Cheng Pan] nit
f5c8bbf44 [Cheng Pan] nit
d280a209b [Cheng Pan] Allow disabling batch impl v2 on globally
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.../org/apache/kyuubi/config/KyuubiConf.scala | 29 +++++++++++-----------
.../kyuubi/server/api/v1/BatchesResource.scala | 2 ++
2 files changed, 17 insertions(+), 14 deletions(-)
diff --git
a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 1e8ce8c8a..c1b2052d9 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -1670,24 +1670,12 @@ object KyuubiConf {
.booleanConf
.createWithDefault(true)
- val BATCH_IMPL_VERSION: ConfigEntry[String] =
- buildConf("kyuubi.batch.impl.version")
- .internal
- .serverOnly
- .doc("Batch API version, candidates: 1, 2. " +
- "Note: Batch API v2 is experimental and under rapid development, this
configuration " +
- "is added to allow explorers conveniently testing the developing Batch
v2 API, not " +
- "intended exposing to end users, it may be removed in anytime.")
- .version("1.8.0")
- .stringConf
- .createWithDefault("1")
-
val BATCH_SUBMITTER_ENABLED: ConfigEntry[Boolean] =
buildConf("kyuubi.batch.submitter.enabled")
.internal
.serverOnly
- .doc("When Batch API v2 is enabled, Kyuubi server requires to pick the
INITIALIZED " +
- "batch job from metastore and submits it to Resource Manager. " +
+ .doc("Batch API v2 requires batch submitter to pick the INITIALIZED
batch job " +
+ "from metastore and submits it to Resource Manager. " +
"Note: Batch API v2 is experimental and under rapid development, this
configuration " +
"is added to allow explorers conveniently testing the developing Batch
v2 API, not " +
"intended exposing to end users, it may be removed in anytime.")
@@ -1705,6 +1693,19 @@ object KyuubiConf {
.intConf
.createWithDefault(100)
+ val BATCH_IMPL_VERSION: ConfigEntry[String] =
+ buildConf("kyuubi.batch.impl.version")
+ .internal
+ .serverOnly
+ .doc("Batch API version, candidates: 1, 2. Only take effect when " +
+ s"${BATCH_SUBMITTER_ENABLED.key} is true, otherwise always use v1
implementation. " +
+ "Note: Batch API v2 is experimental and under rapid development, this
configuration " +
+ "is added to allow explorers conveniently testing the developing Batch
v2 API, not " +
+ "intended exposing to end users, it may be removed in anytime.")
+ .version("1.8.0")
+ .stringConf
+ .createWithDefault("1")
+
val SERVER_EXEC_POOL_SIZE: ConfigEntry[Int] =
buildConf("kyuubi.backend.server.exec.pool.size")
.doc("Number of threads in the operation execution thread pool of Kyuubi
server")
diff --git
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala
index caa768a66..d7e8b615a 100644
---
a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala
+++
b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/api/v1/BatchesResource.scala
@@ -42,6 +42,7 @@ import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.config.KyuubiReservedKeys._
import org.apache.kyuubi.engine.{ApplicationInfo, ApplicationManagerInfo,
KillResponse, KyuubiApplicationManager}
import org.apache.kyuubi.operation.{BatchJobSubmission, FetchOrientation,
OperationState}
+import org.apache.kyuubi.server.KyuubiServer
import org.apache.kyuubi.server.api.ApiRequestContext
import org.apache.kyuubi.server.api.v1.BatchesResource._
import org.apache.kyuubi.server.metadata.MetadataManager
@@ -59,6 +60,7 @@ private[v1] class BatchesResource extends ApiRequestContext
with Logging {
fe.getConf.get(BATCH_INTERNAL_REST_CLIENT_CONNECT_TIMEOUT).toInt
private def batchV2Enabled(reqConf: Map[String, String]): Boolean = {
+ KyuubiServer.kyuubiServer.getConf.get(BATCH_SUBMITTER_ENABLED) &&
reqConf.getOrElse(BATCH_IMPL_VERSION.key,
fe.getConf.get(BATCH_IMPL_VERSION)) == "2"
}
