This is an automated email from the ASF dual-hosted git repository. gyfora pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/flink-kubernetes-operator.git
commit 8d8e7a18b2780f7856bd57162275e47941872abc Author: Matyas Orhidi <[email protected]> AuthorDate: Mon Jun 13 17:07:12 2022 +0200 [hotfix] e2e test --- .github/workflows/ci.yml | 9 +- e2e-tests/data/multi-sessionjob.yaml | 186 +++++++++++++++++++++++++++++++++++ e2e-tests/test_multi_sessionjob.sh | 50 ++++++++++ e2e-tests/utils.sh | 15 +++ 4 files changed, 257 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d12c0c..c0b4608 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,9 +95,11 @@ jobs: - description: 'Default configuration' namespace: default extraArgs: '' + tests: "test_application_kubernetes_ha.sh test_application_operations.sh test_sessionjob_kubernetes_ha.sh test_sessionjob_operations.sh" - description: 'WatchNamespaces enabled' namespace: flink extraArgs: '--create-namespace --set "watchNamespaces={default,flink}"' + tests: "test_application_kubernetes_ha.sh test_application_operations.sh test_sessionjob_kubernetes_ha.sh test_sessionjob_operations.sh test_multi_sessionjob.sh" versions: - image: flink:1.13 flinkVersion: v1_13 @@ -153,9 +155,10 @@ jobs: sed -i "s/image: flink:.*/image: ${{ matrix.versions.image }}/" e2e-tests/data/*.yaml sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.versions.flinkVersion }}/" e2e-tests/data/*.yaml git diff HEAD - ls e2e-tests/test_*.sh | while read script_test;do \ - echo "Running $script_test" - bash $script_test || exit 1 + TESTS="${{ matrix.config.tests }}" + for test in $TESTS;do \ + echo "Running e2e-tests/$test" + bash e2e-tests/$test || exit 1 done git reset --hard - name: Stop the operator diff --git a/e2e-tests/data/multi-sessionjob.yaml b/e2e-tests/data/multi-sessionjob.yaml new file mode 100644 index 0000000..ed5c892 --- /dev/null +++ b/e2e-tests/data/multi-sessionjob.yaml @@ -0,0 +1,186 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + namespace: default + name: session-cluster-1 +spec: + image: flink:1.15 + flinkVersion: v1_15 + ingress: + template: "/{{namespace}}/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + high-availability: org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory + high-availability.storageDir: file:///opt/flink/volume/flink-ha + state.checkpoints.dir: file:///opt/flink/volume/flink-cp + state.savepoints.dir: file:///opt/flink/volume/flink-sp + serviceAccount: flink + podTemplate: + apiVersion: v1 + kind: Pod + metadata: + name: pod-template + spec: + containers: + # Do not change the main container name + - name: flink-main-container + resources: + requests: + ephemeral-storage: 2048Mi + limits: + ephemeral-storage: 2048Mi + volumeMounts: + - mountPath: /opt/flink/volume + name: flink-volume + volumes: + - name: flink-volume + persistentVolumeClaim: + claimName: session-cluster-1-pvc + jobManager: + resource: + memory: "1024m" + cpu: 0.25 + taskManager: + resource: + memory: "1024m" + cpu: 0.25 + +--- +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + namespace: flink + name: session-cluster-1 +spec: + image: flink:1.15 + flinkVersion: v1_15 + ingress: + template: "/{{namespace}}/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + high-availability: org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory + high-availability.storageDir: file:///opt/flink/volume/flink-ha + state.checkpoints.dir: file:///opt/flink/volume/flink-cp + state.savepoints.dir: file:///opt/flink/volume/flink-sp + serviceAccount: flink + podTemplate: + apiVersion: v1 + kind: Pod + metadata: + name: pod-template + spec: + containers: + # Do not change the main container name + - name: flink-main-container + resources: + requests: + ephemeral-storage: 2048Mi + limits: + ephemeral-storage: 2048Mi + volumeMounts: + - mountPath: /opt/flink/volume + name: flink-volume + volumes: + - name: flink-volume + persistentVolumeClaim: + claimName: session-cluster-1-pvc + jobManager: + resource: + memory: "1024m" + cpu: 0.25 + taskManager: + resource: + memory: "1024m" + cpu: 0.25 + +--- +apiVersion: flink.apache.org/v1beta1 +kind: FlinkSessionJob +metadata: + namespace: default + name: flink-example-statemachine +spec: + deploymentName: session-cluster-1 + job: + jarURI: https://repo1.maven.org/maven2/org/apache/flink/flink-examples-streaming_2.12/1.14.4/flink-examples-streaming_2.12-1.14.4.jar + parallelism: 2 + upgradeMode: savepoint + entryClass: org.apache.flink.streaming.examples.statemachine.StateMachineExample + +--- +apiVersion: flink.apache.org/v1beta1 +kind: FlinkSessionJob +metadata: + namespace: flink + name: flink-example-statemachine +spec: + deploymentName: session-cluster-1 + job: + jarURI: https://repo1.maven.org/maven2/org/apache/flink/flink-examples-streaming_2.12/1.14.4/flink-examples-streaming_2.12-1.14.4.jar + parallelism: 2 + upgradeMode: savepoint + entryClass: org.apache.flink.streaming.examples.statemachine.StateMachineExample + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: session-cluster-1-pvc + namespace: default +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi + +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: session-cluster-1-pvc + namespace: flink +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 1Gi + +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + annotations: + ingressclass.kubernetes.io/is-default-class: "true" + labels: + app.kubernetes.io/component: controller + name: nginx +spec: + controller: k8s.io/ingress-nginx diff --git a/e2e-tests/test_multi_sessionjob.sh b/e2e-tests/test_multi_sessionjob.sh new file mode 100755 index 0000000..1484e68 --- /dev/null +++ b/e2e-tests/test_multi_sessionjob.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +source "$(dirname "$0")"/utils.sh + +CLUSTER_ID="session-cluster-1" +APPLICATION_YAML="e2e-tests/data/multi-sessionjob.yaml" +TIMEOUT=300 +SESSION_CLUSTER_IDENTIFIER="flinkdep/session-cluster-1" +SESSION_JOB_IDENTIFIER="sessionjob/flink-example-statemachine" + +on_exit cleanup_and_exit $APPLICATION_YAML $TIMEOUT $CLUSTER_ID + +create_namespace flink +retry_times 5 30 "kubectl apply -f $APPLICATION_YAML" || exit 1 + +# Current namespace: default +kubectl config set-context --current --namespace=default +echo "Current namespace is $(kubectl config view --minify | grep namespace | awk '{print $2}')" +wait_for_jobmanager_running $CLUSTER_ID $TIMEOUT +jm_pod_name=$(get_jm_pod_name $CLUSTER_ID) +wait_for_logs $jm_pod_name "Completed checkpoint [0-9]+ for job" ${TIMEOUT} || exit 1 +wait_for_status $SESSION_CLUSTER_IDENTIFIER '.status.jobManagerDeploymentStatus' READY ${TIMEOUT} || exit 1 +wait_for_status $SESSION_JOB_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +echo "Flink Session Job is running properly" + +# Current namespace: flink +kubectl config set-context --current --namespace=flink +echo "Current namespace is $(kubectl config view --minify | grep namespace | awk '{print $2}')" +wait_for_jobmanager_running $CLUSTER_ID $TIMEOUT +jm_pod_name=$(get_jm_pod_name $CLUSTER_ID) +wait_for_logs $jm_pod_name "Completed checkpoint [0-9]+ for job" ${TIMEOUT} || exit 1 +wait_for_status $SESSION_CLUSTER_IDENTIFIER '.status.jobManagerDeploymentStatus' READY ${TIMEOUT} || exit 1 +wait_for_status $SESSION_JOB_IDENTIFIER '.status.jobStatus.state' RUNNING ${TIMEOUT} || exit 1 +echo "Flink Session Job is running properly" diff --git a/e2e-tests/utils.sh b/e2e-tests/utils.sh index 43d5e3e..b8df6a4 100755 --- a/e2e-tests/utils.sh +++ b/e2e-tests/utils.sh @@ -172,6 +172,7 @@ function cleanup_and_exit() { TIMEOUT=$2 CLUSTER_ID=$3 + kubectl config set-context --current --namespace=default kubectl delete -f $APPLICATION_YAML kubectl wait --for=delete pod --timeout=${TIMEOUT}s --selector="app=${CLUSTER_ID}" kubectl delete cm --selector="app=${CLUSTER_ID},configmap-type=high-availability" @@ -202,3 +203,17 @@ function on_exit { # Keep commands in reverse order, so commands would be executed in LIFO order. _on_exit_commands=("${command} `echo "${@:2}"`" "${_on_exit_commands[@]-}") } + +function create_namespace() { + + NAMESPACE_NAME=${1:-default}; + + NS=$(kubectl get namespace $NAMESPACE_NAME --ignore-not-found); + if [[ "$NS" ]]; then + echo "Skipping creation of namespace $NAMESPACE_NAME - already exists"; + else + echo "Creating namespace $NAMESPACE_NAME"; + kubectl create namespace $NAMESPACE_NAME; + fi; + +}
