[incubator-kyuubi] branch master updated: [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on tagging Spark application

[chengpan]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/master by this push:
     new cebbd230b [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on 
tagging Spark application
cebbd230b is described below

commit cebbd230b080db32a8ae995e0cc8826775024ee4
Author: zwangsheng <2213335...@qq.com>
AuthorDate: Thu Oct 20 13:29:21 2022 +0800

    [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on tagging Spark 
application
    
    ### _Why are the changes needed?_
    
    Fix kyuubi application manager tag spark kubernetes application with yarn 
tag.
    
    The reason for this is the need to determine whether resourceManager start 
with K8S instead of being equal to K8S.
    
    Remove duplicate code from spark batch process builder.
    
    ### _How was this patch tested?_
    - [x] Add some test cases that check the changes thoroughly including 
negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [ ] [Run 
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
 locally before make a pull request
    
    Closes #3670 from 
zwangsheng/bugfix/kyuubi_application_manager_tag_application.
    
    Closes #3670
    
    d4e8aa30 [zwangsheng] fix
    7b5419eb [zwangsheng] back
    a976f994 [zwangsheng] fix
    
    Authored-by: zwangsheng <2213335...@qq.com>
    Signed-off-by: Cheng Pan <cheng...@apache.org>
---
 .../apache/kyuubi/engine/KyuubiApplicationManager.scala  |  2 +-
 .../kyuubi/engine/KyuubiApplicationManagerSuite.scala    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git 
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
 
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
index 7f2501ec2..b772afc04 100644
--- 
a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
+++ 
b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
@@ -160,7 +160,7 @@ object KyuubiApplicationManager {
       conf: KyuubiConf): Unit = {
     (applicationType.toUpperCase, resourceManager.map(_.toUpperCase())) match {
       case ("SPARK", Some("YARN")) => setupSparkYarnTag(applicationTag, conf)
-      case ("SPARK", Some("K8S")) => setupSparkK8sTag(applicationTag, conf)
+      case ("SPARK", Some(rm)) if rm.startsWith("K8S") => 
setupSparkK8sTag(applicationTag, conf)
       case ("SPARK", _) =>
         // if the master is not identified ahead, add all tags
         setupSparkYarnTag(applicationTag, conf)
diff --git 
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
 
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
index 93ef02cdc..b01f82a24 100644
--- 
a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
+++ 
b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.engine
 
 import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite}
 import org.apache.kyuubi.config.KyuubiConf
+import 
org.apache.kyuubi.engine.KubernetesApplicationOperation.LABEL_KYUUBI_UNIQUE_KEY
 
 class KyuubiApplicationManagerSuite extends KyuubiFunSuite {
   test("application access path") {
@@ -70,4 +71,19 @@ class KyuubiApplicationManagerSuite extends KyuubiFunSuite {
         localDirLimitConf)
     }
   }
+
+  test("Test kyuubi application Manager tag spark on kubernetes application") {
+    val conf: KyuubiConf = KyuubiConf()
+    val tag = "kyuubi-test-tag"
+    KyuubiApplicationManager.tagApplication(
+      tag,
+      "SPARK",
+      Some("k8s://https://kyuubi-test:8443";),
+      conf)
+
+    val kubernetesTag = conf.getOption("spark.kubernetes.driver.label." + 
LABEL_KYUUBI_UNIQUE_KEY)
+    val yarnTag = conf.getOption("spark.yarn.tags")
+    assert(kubernetesTag.nonEmpty && tag.equals(kubernetesTag.get))
+    assert(yarnTag.isEmpty)
+  }
 }