This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 5068d6e89 [CELEBORN-105][TEST] Kubernetes Integration Test
5068d6e89 is described below
commit 5068d6e89701eebc9caff0e9d32fe909567e9124
Author: zwangsheng <[email protected]>
AuthorDate: Mon Jun 5 12:11:29 2023 +0800
[CELEBORN-105][TEST] Kubernetes Integration Test
### What changes were proposed in this pull request?
Add Kubernetes Integration Test
- [x] test helm install deploy
- [ ] test shuffle
### Why are the changes needed?
Add integration test
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Ci test
Closes #1484 from zwangsheng/CELEBORN-105.
Authored-by: zwangsheng <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/integration.yml | 78 ++++++++++++++
pom.xml | 14 +++
tests/kubernetes-it/docker/helm/values.yaml | 116 +++++++++++++++++++++
tests/kubernetes-it/pom.xml | 70 +++++++++++++
.../src/test/resources/log4j2-test.xml | 37 +++++++
.../DeploySuite.scala | 59 +++++++++++
.../WithMiniKube.scala | 29 ++++++
7 files changed, 403 insertions(+)
diff --git a/.github/workflows/integration.yml
b/.github/workflows/integration.yml
new file mode 100644
index 000000000..6f5c4f2e8
--- /dev/null
+++ b/.github/workflows/integration.yml
@@ -0,0 +1,78 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Integration Test
+
+on:
+ push:
+ branches:
+ - main
+ - branch-*
+ pull_request:
+ branches:
+ - main
+ - branch-*
+env:
+ MINIKUBE_VERSION: v1.29.0
+ KUBERNETES_VERSION: v1.26.1
+
+jobs:
+ celeborn_integration_test:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-java@v2
+ name: Setup JDK with Maven
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: maven
+ check-latest: false
+ - name: Build Celeborn Binary
+ run:
+ ./build/make-distribution.sh -Pspark-3.3 -Pgoogle-mirror
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ - name: Build Celeborn Docker Image
+ uses: docker/build-push-action@v2
+ with:
+ context: ./dist
+ file: ./docker/Dockerfile
+ load: true
+ tags: apache/celeborn:latest
+ - name: Setup Helm
+ uses: azure/setup-helm@v3
+ with:
+ version: v3.10.0
+ - name: Setup Minikube
+ run: |
+ # https://minikube.sigs.k8s.io/docs/start/
+ curl -LO
https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64
+ sudo install minikube-linux-amd64 /usr/local/bin/minikube
+ minikube start --cpus 2 --memory 4096
--kubernetes-version=${KUBERNETES_VERSION} --force
+ - name: Deploy Celeborn
+ run: |
+ #
https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime
+ minikube image load apache/celeborn:latest
+ helm install celeborn charts/celeborn -f
./tests/kubernetes-it/docker/helm/values.yaml
+ - name: Wait Celeborn Ready
+ run: >-
+ ./build/mvn clean install
+ -Pkubernetes-it
+ -Pspark-3.3
+ -Dtest=none
+ -DwildcardSuites=org.apache.celeborn.integration.DeploySuite
diff --git a/pom.xml b/pom.xml
index 43b06de52..93baeffea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -90,6 +90,7 @@
<roaringbitmap.version>0.9.32</roaringbitmap.version>
<snakeyaml.version>1.33</snakeyaml.version>
<zstd-jni.version>1.5.2-1</zstd-jni.version>
+ <kubernetes-client.version>6.7.0</kubernetes-client.version>
<shading.prefix>org.apache.celeborn.shaded</shading.prefix>
@@ -411,6 +412,12 @@
<version>${mockito-scalatest.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <version>${kubernetes-client.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -1121,5 +1128,12 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>kubernetes-it</id>
+ <modules>
+ <module>tests/kubernetes-it</module>
+ </modules>
+ </profile>
</profiles>
</project>
diff --git a/tests/kubernetes-it/docker/helm/values.yaml
b/tests/kubernetes-it/docker/helm/values.yaml
new file mode 100644
index 000000000..f2b88c4d2
--- /dev/null
+++ b/tests/kubernetes-it/docker/helm/values.yaml
@@ -0,0 +1,116 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+image:
+ repository: apache/celeborn
+ pullPolicy: IfNotPresent
+ tag: latest
+
+imagePullSecrets: {}
+
+# For integration test, using single celeborn mater & worker
+masterReplicas: 1
+workerReplicas: 1
+
+securityContext:
+ runAsUser: 10006
+ runAsGroup: 10006
+ fsGroup: 10006
+
+# Current Celeborn support followings volume type:
+# - emptyDir
+# - hostPath
+# Note: size only works in emptyDir type
+# Celeborn Master will pick first volumes for store raft log
+volumes:
+ master:
+ - mountPath: /mnt/rss_ratis
+ type: emptyDir
+ size: 1Gi
+ worker:
+ - mountPath: /mnt/disk1
+ type: emptyDir
+ size: 1Gi
+ - mountPath: /mnt/disk2
+ type: emptyDir
+ size: 1Gi
+
+# celeborn configurations
+celeborn:
+ celeborn.ha.enabled: false
+ celeborn.metrics.enabled: false
+ celeborn.master.metrics.prometheus.port: 9098
+ celeborn.worker.metrics.prometheus.port: 9096
+ celeborn.worker.monitor.disk.enabled: false
+ celeborn.shuffle.chunk.size: 8m
+ celeborn.rpc.io.serverThreads: 64
+ celeborn.rpc.io.numConnectionsPerPeer: 2
+ celeborn.rpc.io.clientThreads: 64
+ celeborn.rpc.dispatcher.numThreads: 4
+ celeborn.worker.flusher.buffer.size: 256K
+ celeborn.worker.fetch.io.threads: 32
+ celeborn.worker.push.io.threads: 32
+ celeborn.push.stageEnd.timeout: 120s
+ celeborn.application.heartbeat.timeout: 120s
+ celeborn.worker.heartbeat.timeout: 120s
+
+environments:
+ CELEBORN_MASTER_MEMORY: 100m
+ CELEBORN_WORKER_MEMORY: 100m
+ CELEBORN_WORKER_OFFHEAP_MEMORY: 100m
+ CELEBORN_NO_DAEMONIZE: 1
+ TZ: "Asia/Shanghai"
+
+podMonitor:
+ enable: false
+ podMetricsEndpoint:
+ scheme: http
+ interval: 5s
+ portName: metrics
+
+service:
+ type: ClusterIP
+ port: 9097
+
+cluster:
+ name: cluster
+
+configmap: celeborn-conf
+
+resources:
+ master:
+ limits:
+ cpu: 100m
+ memory: 800Mi
+ requests:
+ cpu: 100m
+ memory: 800Mi
+ worker:
+ limits:
+ cpu: 100m
+ memory: 1Gi
+ requests:
+ cpu: 100m
+ memory: 1Gi
+
+podAnnotations: {}
+
+affinity: {}
+
+tolerations: []
+
+nodeSelector: {}
diff --git a/tests/kubernetes-it/pom.xml b/tests/kubernetes-it/pom.xml
new file mode 100644
index 000000000..2efbb905d
--- /dev/null
+++ b/tests/kubernetes-it/pom.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-parent_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>kubernetes-it</artifactId>
+ <packaging>jar</packaging>
+ <name>Celeborn Kubernetes Integration Test</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-common_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-client_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-client_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-master_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.celeborn</groupId>
+ <artifactId>celeborn-worker_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.fabric8</groupId>
+ <artifactId>kubernetes-client</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/tests/kubernetes-it/src/test/resources/log4j2-test.xml
b/tests/kubernetes-it/src/test/resources/log4j2-test.xml
new file mode 100644
index 000000000..9bcf7f263
--- /dev/null
+++ b/tests/kubernetes-it/src/test/resources/log4j2-test.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<Configuration status="INFO">
+ <Appenders>
+ <Console name="stdout" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}:
%m%n%ex"/>
+ <Filters>
+ <ThresholdFilter level="FATAL"/>
+ </Filters>
+ </Console>
+ <File name="file" fileName="target/unit-tests.log">
+ <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}:
%m%n%ex"/>
+ </File>
+ </Appenders>
+ <Loggers>
+ <Root level="INFO">
+ <AppenderRef ref="stdout"/>
+ <AppenderRef ref="file"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git
a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
new file mode 100644
index 000000000..7d93d6d06
--- /dev/null
+++
b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.integration
+
+import scala.language.postfixOps
+
+import org.scalatest.concurrent.Eventually.eventually
+import org.scalatest.concurrent.Waiters.{interval, timeout}
+import org.scalatest.time.SpanSugar.convertIntToGrainOfTime
+
+import org.apache.celeborn.CelebornFunSuite
+import org.apache.celeborn.client.WithShuffleClientSuite
+
+// TODO need add shuffle client test
+class DeploySuite extends CelebornFunSuite with WithMiniKube {
+ final val masterStatefulSet =
+ kubernetesClient.apps().statefulSets().withName("celeborn-master").get()
+ assert(masterStatefulSet != null)
+ final val workerStatefulSet =
+ kubernetesClient.apps().statefulSets().withName("celeborn-worker").get()
+ assert(workerStatefulSet != null)
+
+ test("Check master started") {
+ val masterPod = kubernetesClient.pods().withName("celeborn-master-0")
+ // wait for master ready
+ eventually(timeout(5 minutes), interval(10 seconds)) {
+ val log = masterPod.getLog(true)
+ assert(log.contains("Master started."))
+ }
+
+ assert(masterPod.get().getStatus.getPhase == "Running")
+ }
+
+ test("Check worker started") {
+ val workerPod = kubernetesClient.pods().withName("celeborn-worker-0")
+ // wait for worker ready
+ eventually(timeout(5 minutes), interval(10 seconds)) {
+ val log = workerPod.getLog(true)
+ assert(log.contains("Register worker successfully."))
+ }
+
+ assert(workerPod.get().getStatus.getPhase == "Running")
+ }
+}
diff --git
a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
new file mode 100644
index 000000000..8676d7174
--- /dev/null
+++
b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.integration
+
+import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder}
+import io.fabric8.kubernetes.client.Config
+
+trait WithMiniKube {
+
+ lazy val kubernetesClient: KubernetesClient =
+ new KubernetesClientBuilder()
+ .withConfig(Config.autoConfigure("minikube"))
+ .build()
+}
