This is an automated email from the ASF dual-hosted git repository. zhouky pushed a commit to branch branch-0.3 in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 24222f2a20ede7250e3db92ae1af3d643e30d23a Author: zwangsheng <[email protected]> AuthorDate: Mon Jun 5 12:11:29 2023 +0800 [CELEBORN-105][TEST] Kubernetes Integration Test --- .github/workflows/integration.yml | 78 ++++++++++++++++++++++ pom.xml | 14 ++++ tests/kubernetes-it/pom.xml | 70 +++++++++++++++++++ .../src/test/resources/log4j2-test.xml | 37 ++++++++++ .../DeploySuite.scala | 59 ++++++++++++++++ .../WithMiniKube.scala | 29 ++++++++ 6 files changed, 287 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 000000000..6f5c4f2e8 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Integration Test + +on: + push: + branches: + - main + - branch-* + pull_request: + branches: + - main + - branch-* +env: + MINIKUBE_VERSION: v1.29.0 + KUBERNETES_VERSION: v1.26.1 + +jobs: + celeborn_integration_test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v2 + name: Setup JDK with Maven + with: + distribution: zulu + java-version: 8 + cache: maven + check-latest: false + - name: Build Celeborn Binary + run: + ./build/make-distribution.sh -Pspark-3.3 -Pgoogle-mirror + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Build Celeborn Docker Image + uses: docker/build-push-action@v2 + with: + context: ./dist + file: ./docker/Dockerfile + load: true + tags: apache/celeborn:latest + - name: Setup Helm + uses: azure/setup-helm@v3 + with: + version: v3.10.0 + - name: Setup Minikube + run: | + # https://minikube.sigs.k8s.io/docs/start/ + curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force + - name: Deploy Celeborn + run: | + # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime + minikube image load apache/celeborn:latest + helm install celeborn charts/celeborn -f ./tests/kubernetes-it/docker/helm/values.yaml + - name: Wait Celeborn Ready + run: >- + ./build/mvn clean install + -Pkubernetes-it + -Pspark-3.3 + -Dtest=none + -DwildcardSuites=org.apache.celeborn.integration.DeploySuite diff --git a/pom.xml b/pom.xml index 3c080daf9..85f614077 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ <roaringbitmap.version>0.9.32</roaringbitmap.version> <snakeyaml.version>1.33</snakeyaml.version> <zstd-jni.version>1.5.2-1</zstd-jni.version> + <kubernetes-client.version>6.7.0</kubernetes-client.version> <shading.prefix>org.apache.celeborn.shaded</shading.prefix> @@ -405,6 +406,12 @@ <version>${mockito-scalatest.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>io.fabric8</groupId> + <artifactId>kubernetes-client</artifactId> + <version>${kubernetes-client.version}</version> + <scope>test</scope> + </dependency> </dependencies> </dependencyManagement> @@ -1134,5 +1141,12 @@ </plugins> </build> </profile> + + <profile> + <id>kubernetes-it</id> + <modules> + <module>tests/kubernetes-it</module> + </modules> + </profile> </profiles> </project> diff --git a/tests/kubernetes-it/pom.xml b/tests/kubernetes-it/pom.xml new file mode 100644 index 000000000..2efbb905d --- /dev/null +++ b/tests/kubernetes-it/pom.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.celeborn</groupId> + <artifactId>celeborn-parent_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <relativePath>../../pom.xml</relativePath> + </parent> + + <artifactId>kubernetes-it</artifactId> + <packaging>jar</packaging> + <name>Celeborn Kubernetes Integration Test</name> + + <dependencies> + <dependency> + <groupId>org.apache.celeborn</groupId> + <artifactId>celeborn-common_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.celeborn</groupId> + <artifactId>celeborn-client_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.celeborn</groupId> + <artifactId>celeborn-client_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.celeborn</groupId> + <artifactId>celeborn-master_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.celeborn</groupId> + <artifactId>celeborn-worker_${scala.binary.version}</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>io.fabric8</groupId> + <artifactId>kubernetes-client</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> diff --git a/tests/kubernetes-it/src/test/resources/log4j2-test.xml b/tests/kubernetes-it/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000..9bcf7f263 --- /dev/null +++ b/tests/kubernetes-it/src/test/resources/log4j2-test.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<Configuration status="INFO"> + <Appenders> + <Console name="stdout" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n%ex"/> + <Filters> + <ThresholdFilter level="FATAL"/> + </Filters> + </Console> + <File name="file" fileName="target/unit-tests.log"> + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss,SSS} %p [%t] %c{1}: %m%n%ex"/> + </File> + </Appenders> + <Loggers> + <Root level="INFO"> + <AppenderRef ref="stdout"/> + <AppenderRef ref="file"/> + </Root> + </Loggers> +</Configuration> diff --git a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala new file mode 100644 index 000000000..7d93d6d06 --- /dev/null +++ b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/DeploySuite.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.integration + +import scala.language.postfixOps + +import org.scalatest.concurrent.Eventually.eventually +import org.scalatest.concurrent.Waiters.{interval, timeout} +import org.scalatest.time.SpanSugar.convertIntToGrainOfTime + +import org.apache.celeborn.CelebornFunSuite +import org.apache.celeborn.client.WithShuffleClientSuite + +// TODO need add shuffle client test +class DeploySuite extends CelebornFunSuite with WithMiniKube { + final val masterStatefulSet = + kubernetesClient.apps().statefulSets().withName("celeborn-master").get() + assert(masterStatefulSet != null) + final val workerStatefulSet = + kubernetesClient.apps().statefulSets().withName("celeborn-worker").get() + assert(workerStatefulSet != null) + + test("Check master started") { + val masterPod = kubernetesClient.pods().withName("celeborn-master-0") + // wait for master ready + eventually(timeout(5 minutes), interval(10 seconds)) { + val log = masterPod.getLog(true) + assert(log.contains("Master started.")) + } + + assert(masterPod.get().getStatus.getPhase == "Running") + } + + test("Check worker started") { + val workerPod = kubernetesClient.pods().withName("celeborn-worker-0") + // wait for worker ready + eventually(timeout(5 minutes), interval(10 seconds)) { + val log = workerPod.getLog(true) + assert(log.contains("Register worker successfully.")) + } + + assert(workerPod.get().getStatus.getPhase == "Running") + } +} diff --git a/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala new file mode 100644 index 000000000..8676d7174 --- /dev/null +++ b/tests/kubernetes-it/src/test/scala/org.apache.celeborn.integration/WithMiniKube.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.celeborn.integration + +import io.fabric8.kubernetes.client.{KubernetesClient, KubernetesClientBuilder} +import io.fabric8.kubernetes.client.Config + +trait WithMiniKube { + + lazy val kubernetesClient: KubernetesClient = + new KubernetesClientBuilder() + .withConfig(Config.autoConfigure("minikube")) + .build() +}
