This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a4c521c69 [KYUUBI #2216] [Test] [K8s] Add Spark Cluster mode on
Kubernetes integration test
a4c521c69 is described below
commit a4c521c69b5ded2112d6eeceac589e352a23b4ba
Author: zwangsheng <[email protected]>
AuthorDate: Tue Apr 12 21:54:04 2022 +0800
[KYUUBI #2216] [Test] [K8s] Add Spark Cluster mode on Kubernetes
integration test
### _Why are the changes needed?_
Close #2216
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests)
locally before make a pull request
Closes #2258 from zwangsheng/feature/add_cluster_k8s_test.
Closes #2216
7eea16b0 [zwangsheng] add linces
8effcf07 [Binjie Yang] Spark On K8s With Cluster
9cd03ad9 [ulysses-you] config
ce7143ee [zwangsheng] fix unused import
0a06521f [zwangsheng] fix
f9293925 [zwangsheng] test
88ec89f9 [zwangsheng] test
0f780f37 [zwangsheng] test
9b27bfa6 [zwangsheng] Test
df43cd76 [zwangsheng] Using Kerberos Submit Spark
99893a64 [zwangsheng] call beforall help set kerbores
0a732d94 [zwangsheng] Spark On K
Lead-authored-by: zwangsheng <[email protected]>
Co-authored-by: ulysses-you <[email protected]>
Co-authored-by: Binjie Yang <[email protected]>
Signed-off-by: ulysses-you <[email protected]>
---
.github/workflows/master.yml | 11 +++++
integration-tests/kyuubi-kubernetes-it/pom.xml | 6 +++
.../src/test/resources/driver.yml | 24 ++++++++++
.../test/spark/SparkOnKubernetesTestsSuite.scala | 53 ++++++++++++++++++++--
.../org/apache/kyuubi/WithSimpleDFSService.scala | 47 +++++++++++++++++++
.../org/apache/kyuubi/server/MiniDFSService.scala | 4 +-
6 files changed, 140 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 35b7f0e5b..677f02692 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -184,6 +184,12 @@ jobs:
kubernetes version: 'v1.23.3'
driver: docker
start args: '--extra-config=kubeadm.ignore-preflight-errors=NumCPU
--force --cpus 2 --memory 4096'
+ # in case:
https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac
+ - name: Create Service Account
+ run: |
+ kubectl create serviceaccount spark
+ kubectl create clusterrolebinding spark-role --clusterrole=edit
--serviceaccount=default:spark --namespace=default
+ kubectl get serviceaccount
- name: integration tests
run: >-
./build/mvn clean install
@@ -195,6 +201,11 @@ jobs:
-Pflink-provided,hive-provided
-Pkubernetes-it
-Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.spark
+ - name: Print Driver Pod logs
+ if: failure()
+ run: |
+ kubectl get pods
+ kubectl get pods | grep driver | awk -F " " '{print$1}' | xargs -I
{} kubectl logs {}
- name: Upload test logs
if: failure()
uses: actions/upload-artifact@v2
diff --git a/integration-tests/kyuubi-kubernetes-it/pom.xml
b/integration-tests/kyuubi-kubernetes-it/pom.xml
index 089e63c53..908bb83d2 100644
--- a/integration-tests/kyuubi-kubernetes-it/pom.xml
+++ b/integration-tests/kyuubi-kubernetes-it/pom.xml
@@ -74,6 +74,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
<!-- for hive driver related dependency -->
<dependency>
<groupId>org.apache.kyuubi</groupId>
diff --git
a/integration-tests/kyuubi-kubernetes-it/src/test/resources/driver.yml
b/integration-tests/kyuubi-kubernetes-it/src/test/resources/driver.yml
new file mode 100644
index 000000000..c71735224
--- /dev/null
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/resources/driver.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersio: v1
+kind: Pod
+metadata:
+ name: driver
+spec:
+ hostNetwork: true
+ dnsPolicy: ClusterFirstWithHostNet
diff --git
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
index 38bc6a112..f2b5efba6 100644
---
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
+++
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@@ -17,12 +17,16 @@
package org.apache.kyuubi.kubernetes.test.spark
-import org.apache.kyuubi.{Logging, WithKyuubiServer}
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.net.NetUtils
+
+import org.apache.kyuubi.{Logging, Utils, WithKyuubiServer,
WithSimpleDFSService}
import org.apache.kyuubi.config.KyuubiConf
+import
org.apache.kyuubi.config.KyuubiConf.{FRONTEND_CONNECTION_URL_USE_HOSTNAME,
FRONTEND_THRIFT_BINARY_BIND_HOST}
import org.apache.kyuubi.kubernetes.test.MiniKube
import org.apache.kyuubi.operation.SparkQueryTests
+import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT_ADDRESS
-// TODO Support Spark Cluster mode
abstract class SparkOnKubernetesSuiteBase
extends WithKyuubiServer with SparkQueryTests with Logging {
private val apiServerAddress = {
@@ -36,16 +40,17 @@ abstract class SparkOnKubernetesSuiteBase
.set("spark.kubernetes.container.image", "apache/spark:v3.2.1")
.set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
.set("spark.executor.instances", "1")
- .set("spark.executor.cores", "1")
.set("spark.executor.memory", "512M")
.set("spark.driver.memory", "512M")
+ .set("spark.kubernetes.driver.request.cores", "250m")
+ .set("spark.kubernetes.executor.request.cores", "250m")
}
override protected def jdbcUrl: String = getJdbcUrl
}
/**
- * This test is for Kyuubi Server with Spark engine on Kubernetes:
+ * This test is for Kyuubi Server with Spark engine Using client deploy-mode
on Kubernetes:
*
* Real World
Kubernetes Pod
* -------------------------------------------------------
---------------------
@@ -59,3 +64,43 @@ class SparkClientModeOnKubernetesSuite extends
SparkOnKubernetesSuiteBase {
sparkOnK8sConf.set("spark.submit.deployMode", "client")
}
}
+
+/**
+ * This test is for Kyuubi Server with Spark engine Using cluster deploy-mode
on Kubernetes:
+ *
+ * Real World Kubernetes Pod
Kubernetes Pod
+ * ---------------------------------- ---------------------
---------------------
+ * | JDBC | | |
| |
+ * | Client ----> Kyuubi Server | ----> | Spark Driver | ---->
| Spark Executors |
+ * | | | |
| |
+ * ---------------------------------- ---------------------
---------------------
+ */
+class SparkClusterModeOnKubernetesSuite
+ extends SparkOnKubernetesSuiteBase with WithSimpleDFSService {
+ private val localhostAddress = Utils.findLocalInetAddress.getHostAddress
+ private val driverTemplate =
+ Thread.currentThread().getContextClassLoader.getResource("driver.yml")
+
+ override val hadoopConf: Configuration = {
+ val hdfsConf: Configuration = new Configuration()
+ hdfsConf.set("dfs.namenode.rpc-bind-host", "0.0.0.0")
+ hdfsConf.set("dfs.namenode.servicerpc-bind-host", "0.0.0.0")
+ hdfsConf.set("dfs.datanode.hostname", localhostAddress)
+ hdfsConf.set("dfs.datanode.address",
s"0.0.0.0:${NetUtils.getFreeSocketPort}")
+ // spark use 185 as userid in docker
+ hdfsConf.set("hadoop.proxyuser.185.groups", "*")
+ hdfsConf.set("hadoop.proxyuser.185.hosts", "*")
+ hdfsConf
+ }
+
+ override protected lazy val conf: KyuubiConf = {
+ sparkOnK8sConf.set("spark.submit.deployMode", "cluster")
+ .set("spark.kubernetes.file.upload.path",
s"hdfs://$localhostAddress:$getDFSPort/spark")
+ .set("spark.hadoop.dfs.client.use.datanode.hostname", "true")
+ .set("spark.kubernetes.authenticate.driver.serviceAccountName", "spark")
+ .set("spark.kubernetes.driver.podTemplateFile", driverTemplate.getPath)
+ .set(ZK_CLIENT_PORT_ADDRESS.key, localhostAddress)
+ .set(FRONTEND_CONNECTION_URL_USE_HOSTNAME.key, "false")
+ .set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, localhostAddress)
+ }
+}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
new file mode 100644
index 000000000..e7d0fbfe1
--- /dev/null
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/WithSimpleDFSService.scala
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi
+
+import org.apache.hadoop.conf.Configuration
+
+import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.server.MiniDFSService
+
+trait WithSimpleDFSService extends KyuubiFunSuite {
+
+ private var miniDFSService: MiniDFSService = _
+
+ def hadoopConf: Configuration = new Configuration()
+
+ override def beforeAll(): Unit = {
+ miniDFSService = new MiniDFSService(hadoopConf)
+ miniDFSService.initialize(new KyuubiConf(false))
+ miniDFSService.start()
+ super.beforeAll()
+ }
+
+ override def afterAll(): Unit = {
+ miniDFSService.stop()
+ super.afterAll()
+ }
+
+ def getHadoopConf: Configuration = miniDFSService.getHadoopConf
+
+ def getDefaultFS: String = miniDFSService.getHadoopConf.get("fs.defaultFS")
+ def getDFSPort: Int = miniDFSService.getDFSPort
+}
diff --git
a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/MiniDFSService.scala
b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/MiniDFSService.scala
index e2f93f47c..d7f01e728 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/server/MiniDFSService.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/server/MiniDFSService.scala
@@ -37,7 +37,7 @@ class MiniDFSService(name: String, hdfsConf: Configuration)
override def initialize(conf: KyuubiConf): Unit = {
// Set bind host to localhost to avoid java.net.BindException
- hdfsConf.set("dfs.namenode.rpc-bind-host", "localhost")
+ hdfsConf.setIfUnset("dfs.namenode.rpc-bind-host", "localhost")
// enable proxy
val currentUser = UserGroupInformation.getCurrentUser.getShortUserName
@@ -49,6 +49,7 @@ class MiniDFSService(name: String, hdfsConf: Configuration)
override def start(): Unit = {
hdfsCluster = new MiniDFSCluster.Builder(hdfsConf)
.checkDataNodeAddrConfig(true)
+ .checkDataNodeHostConfig(true)
.build()
info(
s"NameNode address in configuration is " +
@@ -62,4 +63,5 @@ class MiniDFSService(name: String, hdfsConf: Configuration)
}
def getHadoopConf: Configuration = hdfsConf
+ def getDFSPort: Int = hdfsCluster.getNameNodePort
}
