This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch v2-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit bc49f1f25e2336ff3d457b5989f5c695a42b81f7 Author: Ephraim Anierobi <[email protected]> AuthorDate: Mon May 17 22:50:23 2021 +0100 Use different executors for Helm Chart tests in CI (#15791) closes https://github.com/apache/airflow/issues/14301 (cherry picked from commit 2a7298d6fc8a4f587ce51d9a2f75aa23231343c3) --- .github/workflows/ci.yml | 6 +++++- kubernetes_tests/test_kubernetes_pod_operator.py | 2 ++ kubernetes_tests/test_kubernetes_pod_operator_backcompat.py | 5 ++++- .../kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh | 1 + .../ci_setup_cluster_and_run_kubernetes_tests_single_job.sh | 1 + .../ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh | 1 - scripts/ci/libraries/_initialization.sh | 4 ++-- scripts/ci/libraries/_kind.sh | 4 ---- 8 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 569dc17..4a8a209 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -915,15 +915,19 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-kubernetes: timeout-minutes: 50 - name: Helm Chart + name: Helm Chart; ${{matrix.executor}} runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, prod-images] + strategy: + matrix: + executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} BACKEND: postgres RUN_TESTS: "true" RUNTIME: "kubernetes" KUBERNETES_MODE: "image" + EXECUTOR: ${{matrix.executor}} KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py index 2da5da0..9d32782 100644 --- a/kubernetes_tests/test_kubernetes_pod_operator.py +++ b/kubernetes_tests/test_kubernetes_pod_operator.py @@ -40,6 +40,7 @@ from airflow.providers.cncf.kubernetes.utils.pod_launcher import PodLauncher from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults from airflow.utils import timezone from airflow.version import version as airflow_version +from kubernetes_tests.test_base import EXECUTOR def create_context(task): @@ -62,6 +63,7 @@ def get_kubeconfig_path(): return kubeconfig_path if kubeconfig_path else os.path.expanduser('~/.kube/config') [email protected](EXECUTOR != 'KubernetesExecutor', reason="Only runs on KubernetesExecutor") class TestKubernetesPodOperatorSystem(unittest.TestCase): def get_current_task_name(self): # reverse test name to make pod name unique (it has limited length) diff --git a/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py b/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py index ea9c9ee..f2058c3 100644 --- a/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py +++ b/kubernetes_tests/test_kubernetes_pod_operator_backcompat.py @@ -41,9 +41,11 @@ from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults from airflow.utils import timezone from airflow.utils.state import State from airflow.version import version as airflow_version - +from kubernetes_tests.test_base import EXECUTOR # noinspection DuplicatedCode + + def create_context(task): dag = DAG(dag_id="dag") tzinfo = pendulum.timezone("Europe/Amsterdam") @@ -60,6 +62,7 @@ def create_context(task): # noinspection DuplicatedCode,PyUnusedLocal [email protected](EXECUTOR != 'KubernetesExecutor', reason="Only runs on KubernetesExecutor") class TestKubernetesPodOperatorSystem(unittest.TestCase): def get_current_task_name(self): # reverse test name to make pod name unique (it has limited length) diff --git a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh index 1e0fa36..fce0a4c 100755 --- a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh +++ b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. + export SKIP_BUILDING_PROD_IMAGE="true" # shellcheck source=scripts/ci/libraries/_script_init.sh diff --git a/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh index 9b0d86f..914e426 100755 --- a/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh +++ b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh @@ -30,6 +30,7 @@ fi export PYTHON_MAJOR_MINOR_VERSION=$1 shift + # Requires PARALLEL_JOB_STATUS if [[ -z "${PARALLEL_JOB_STATUS=}" ]]; then diff --git a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh index 88aa2cd..5790b94 100755 --- a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh +++ b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh @@ -101,6 +101,5 @@ parallel::make_sure_kubernetes_versions_are_specified get_maximum_parallel_k8s_jobs run_k8s_tests_in_parallel "${@}" - # this will exit with error code in case some of the tests failed parallel::print_job_summary_and_return_status_code diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 0bc9dc1..034cd37 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -25,6 +25,7 @@ CURRENT_POSTGRES_VERSIONS=() CURRENT_MYSQL_VERSIONS=() CURRENT_KIND_VERSIONS=() CURRENT_HELM_VERSIONS=() +CURRENT_EXECUTOR=() ALL_PYTHON_MAJOR_MINOR_VERSIONS=() INSTALLED_PROVIDERS=() @@ -790,9 +791,8 @@ function initialization::make_constants_read_only() { readonly KUBERNETES_VERSION readonly KIND_VERSION readonly HELM_VERSION - readonly EXECUTOR readonly KUBECTL_VERSION - + readonly EXECUTOR readonly POSTGRES_VERSION readonly MYSQL_VERSION diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index d5adadf..085dfac 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -140,10 +140,6 @@ function kind::perform_kind_cluster_operation() { echo "Kubernetes mode: ${KUBERNETES_MODE}" echo - echo - echo "Executor: ${EXECUTOR}" - echo - if [[ ${operation} == "status" ]]; then if [[ ${all_clusters} == *"${KIND_CLUSTER_NAME}"* ]]; then echo
