This is an automated email from the ASF dual-hosted git repository. ash pushed a commit to branch v2-0-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 14f71164485f48310e37b9127a7fabab618c1ec6 Author: Jarek Potiuk <[email protected]> AuthorDate: Wed Apr 7 20:08:00 2021 +0200 Run kubernetes tests in parallel (#15222) (cherry picked from commit ea0710edc106a9091d666a3be629201ad7cbbcad) --- .github/workflows/ci.yml | 61 ++++++------ .gitignore | 2 +- .rat-excludes | 2 +- TESTING.rst | 6 ++ breeze | 2 +- .../ci/images/ci_wait_for_and_verify_ci_image.sh | 3 +- .../ci/images/ci_wait_for_and_verify_prod_image.sh | 17 ++-- scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 10 +- ...tup_cluster_and_deploy_airflow_to_kubernetes.sh | 6 +- ...cluster_and_run_kubernetes_tests_single_job.sh} | 51 +++++----- ...lusters_and_run_kubernetes_tests_in_parallel.sh | 106 +++++++++++++++++++++ scripts/ci/kubernetes/kind-cluster-conf.yaml | 4 +- scripts/ci/libraries/_build_images.sh | 23 +---- scripts/ci/libraries/_docker_engine_resources.sh | 10 +- scripts/ci/libraries/_initialization.sh | 10 +- scripts/ci/libraries/_kind.sh | 57 +++++------ scripts/ci/libraries/_parallel.sh | 15 ++- scripts/ci/libraries/_testing.sh | 2 +- scripts/ci/libraries/_verbosity.sh | 2 +- scripts/ci/selective_ci_checks.sh | 2 + scripts/ci/testing/ci_run_quarantined_tests.sh | 3 + scripts/in_container/entrypoint_ci.sh | 4 +- 22 files changed, 248 insertions(+), 150 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21a5429..243557f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,6 +130,7 @@ jobs: pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} + kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} defaultKubernetesVersion: ${{ steps.selective-checks.outputs.default-kubernetes-version }} kubernetesModes: ${{ steps.selective-checks.outputs.kubernetes-modes }} defaultKubernetesMode: ${{ steps.selective-checks.outputs.default-kubernetes-mode }} @@ -145,7 +146,6 @@ jobs: postgresExclude: ${{ steps.selective-checks.outputs.postgres-exclude }} mysqlExclude: ${{ steps.selective-checks.outputs.mysql-exclude }} sqliteExclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} - kubernetesExclude: ${{ steps.selective-checks.outputs.kubernetes-exclude }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} @@ -628,7 +628,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" with: name: > coverage-helm - path: "./files/coverage.xml" + path: "./files/coverage*.xml" retention-days: 7 tests-postgres: @@ -686,7 +686,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" with: name: > coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/coverage.xml" + path: "./files/coverage*.xml" retention-days: 7 tests-mysql: @@ -742,7 +742,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" uses: actions/upload-artifact@v2 with: name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/coverage.xml" + path: "./files/coverage*.xml" retention-days: 7 tests-sqlite: @@ -796,7 +796,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" uses: actions/upload-artifact@v2 with: name: coverage-sqlite-${{matrix.python-version}} - path: ./files/coverage.xml + path: ./files/coverage*.xml retention-days: 7 tests-quarantined: @@ -867,7 +867,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" uses: actions/upload-artifact@v2 with: name: coverage-quarantined-${{ matrix.backend }} - path: "./files/coverage.xml" + path: "./files/coverage*.xml" retention-days: 7 upload-coverage: @@ -947,27 +947,22 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" tests-kubernetes: timeout-minutes: 50 - name: K8s ${{matrix.python-version}} ${{matrix.kubernetes-version}} ${{matrix.kubernetes-mode}} + name: K8s tests runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, prod-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - kubernetes-version: ${{ fromJson(needs.build-info.outputs.kubernetesVersions) }} - kubernetes-mode: ${{ fromJson(needs.build-info.outputs.kubernetesModes) }} - exclude: ${{ fromJson(needs.build-info.outputs.kubernetesExclude) }} - fail-fast: false env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} BACKEND: postgres RUN_TESTS: "true" RUNTIME: "kubernetes" - PYTHON_MAJOR_MINOR_VERSION: "${{ matrix.python-version }}" - KUBERNETES_MODE: "${{ matrix.kubernetes-mode }}" - KUBERNETES_VERSION: "${{ matrix.kubernetes-version }}" + KUBERNETES_MODE: "image" KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" GITHUB_REGISTRY: ${{ needs.prod-images.outputs.githubRegistry }} + CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > + ${{needs.build-info.outputs.pythonVersionsListAsString}} + CURRENT_KUBERNETES_VERSIONS_AS_STRING: > + ${{needs.build-info.outputs.kubernetesVersionsListAsString}} if: needs.build-info.outputs.run-kubernetes-tests == 'true' steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -980,45 +975,43 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - name: "Free space" run: ./scripts/ci/tools/ci_free_space_on_ci.sh - - name: "Prepare PROD Image" - run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh - - name: "Setup cluster and deploy Airflow" - id: setp-cluster-deploy-app - run: ./scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh - env: - # We have the right image pulled already by the previous step - SKIP_BUILDING_PROD_IMAGE: "true" + - name: "Get all PROD images" + run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh - name: "Cache virtualenv for kubernetes testing" uses: actions/cache@v2 with: - path: ".build/.kubernetes_venv_ ${{ needs.build-info.outputs.defaultPythonVersion }}" + path: ".build/.kubernetes_venv" key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ +-${{needs.build-info.outputs.kubernetesVersionsListAsString}} +-${{needs.build-info.outputs.pythonVersionsListAsString}} -${{ hashFiles('setup.py','setup.cfg') }}" - restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-" + restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\ +-${{needs.build-info.outputs.kubernetesVersionsListAsString}} +-${{needs.build-info.outputs.pythonVersionsListAsString}}" - name: "Cache bin folder with tools for kubernetes testing" uses: actions/cache@v2 with: - path: ".build/bin" - key: "bin-${{ matrix.kubernetes-version }}\ + path: ".build/kubernetes-bin" + key: "kubernetes-binaries -${{ needs.build-info.outputs.defaultKindVersion }}\ -${{ needs.build-info.outputs.defaultHelmVersion }}" - restore-keys: "bin-${{ matrix.kubernetes-version }}" + restore-keys: "kubernetes-binaries" - name: "Kubernetes Tests" - run: ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh + run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh - name: "Upload KinD logs" uses: actions/upload-artifact@v2 if: failure() with: name: > - kind-logs-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}} + kind-logs- path: /tmp/kind_logs_* retention-days: 7 - name: "Upload artifact for coverage" uses: actions/upload-artifact@v2 with: name: > - coverage-k8s-${{matrix.kubernetes-mode}}-${{matrix.python-version}}-${{matrix.kubernetes-version}} - path: "./files/coverage.xml" + coverage-k8s- + path: "./files/coverage*.xml" retention-days: 7 push-prod-images-to-github-registry: diff --git a/.gitignore b/.gitignore index 5da0b18..0454790 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,7 @@ htmlcov/ .coverage.* .cache nosetests.xml -coverage.xml +coverage*.xml *,cover .hypothesis/ .pytest_cache diff --git a/.rat-excludes b/.rat-excludes index 9f467d8..125ed46 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -71,7 +71,7 @@ node_modules/* coverage/* git_version flake8_diff.sh -coverage.xml +coverage*.xml _sources/* rat-results.txt diff --git a/TESTING.rst b/TESTING.rst index 102cac8..24d1e72 100644 --- a/TESTING.rst +++ b/TESTING.rst @@ -629,6 +629,12 @@ Entering shell with Kubernetes Cluster This shell is prepared to run Kubernetes tests interactively. It has ``kubectl`` and ``kind`` cli tools available in the path, it has also activated virtualenv environment that allows you to run tests via pytest. +The binaries are available in ./.build/kubernetes-bin/``KUBERNETES_VERSION`` path. +The virtualenv is available in ./.build/.kubernetes_venv/``KIND_CLUSTER_NAME``_host_python_``HOST_PYTHON_VERSION`` + +Where ``KIND_CLUSTER_NAME`` is the name of the cluster and ``HOST_PYTHON_VERSION`` is the version of python +in the host. + You can enter the shell via those scripts ./scripts/ci/kubernetes/ci_run_kubernetes_tests.sh [-i|--interactive] - Activates virtual environment ready to run tests and drops you in diff --git a/breeze b/breeze index 302cebb..df24b14 100755 --- a/breeze +++ b/breeze @@ -3418,7 +3418,7 @@ function breeze::run_breeze_command() { set +u local dc_run_file local run_command="breeze::run_command" - if [[ ${DRY_RUN_DOCKER} != "false" ]]; then + if [[ ${DRY_RUN_DOCKER=} != "false" ]]; then run_command="breeze::print_command" fi if [[ ${PRODUCTION_IMAGE} == "true" ]]; then diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh index e83b0d6..29daca7 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh @@ -31,7 +31,6 @@ shift function pull_ci_image() { local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" start_end::group_start "Pulling ${image_name_with_tag} image" - push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}" start_end::group_end @@ -39,7 +38,9 @@ function pull_ci_image() { push_pull_remove_images::check_if_github_registry_wait_for_image_enabled +start_end::group_start "Configure Docker Registry" build_image::configure_docker_registry +start_end::group_end export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" diff --git a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh index b4a482a..84c310e 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh +++ b/scripts/ci/images/ci_wait_for_and_verify_prod_image.sh @@ -28,17 +28,11 @@ shift # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -function pull_prod_image() { - local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - start_end::group_start "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}" - - push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}" - start_end::group_end -} - push_pull_remove_images::check_if_github_registry_wait_for_image_enabled +start_end::group_start "Configure Docker Registry" build_image::configure_docker_registry +start_end::group_end export AIRFLOW_PROD_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}" @@ -48,8 +42,11 @@ push_pull_remove_images::wait_for_github_registry_image \ "${AIRFLOW_PROD_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" start_end::group_end +start_end::group_start "Pulling the PROD Image" build_images::prepare_prod_build - -pull_prod_image +image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" +verbosity::print_info "Pulling the ${image_name_with_tag} image and tagging with ${AIRFLOW_PROD_IMAGE}" +push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_PROD_IMAGE}" "${image_name_with_tag}" +start_end::group_end verify_image::verify_prod_image "${AIRFLOW_PROD_IMAGE}" diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh index 65f885c..fbcf66b 100755 --- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh +++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh @@ -62,7 +62,7 @@ function parse_tests_to_run() { "--durations=100" "--cov=airflow/" "--cov-config=.coveragerc" - "--cov-report=xml:files/coverage.xml" + "--cov-report=xml:files/coverage=${KIND_CLUSTER_NAME}.xml" "--color=yes" "--maxfail=50" "--pythonwarnings=ignore::DeprecationWarning" @@ -73,12 +73,12 @@ function parse_tests_to_run() { } function create_virtualenv() { - start_end::group_start "Creating virtualenv" HOST_PYTHON_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")') readonly HOST_PYTHON_VERSION - local virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv_${HOST_PYTHON_VERSION}" + local virtualenv_path="${BUILD_CACHE_DIR}/.kubernetes_venv/${KIND_CLUSTER_NAME}_host_python_${HOST_PYTHON_VERSION}" + mkdir -pv "${BUILD_CACHE_DIR}/.kubernetes_venv/" if [[ ! -d ${virtualenv_path} ]]; then echo echo "Creating virtualenv at ${virtualenv_path}" @@ -95,14 +95,10 @@ function create_virtualenv() { pip install -e ".[kubernetes]" \ --constraint "https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${HOST_PYTHON_VERSION}.txt"; - - start_end::group_end } function run_tests() { - start_end::group_start "Running K8S tests" pytest "${pytest_args[@]}" "${tests_to_run[@]}" - start_end::group_end } cd "${AIRFLOW_SOURCES}" || exit 1 diff --git a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh index ec493f8..1e0fa36 100755 --- a/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh +++ b/scripts/ci/kubernetes/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh @@ -15,9 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +export SKIP_BUILDING_PROD_IMAGE="true" + # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -set -euo pipefail traps::add_trap "kind::dump_kind_logs" EXIT HUP INT TERM @@ -25,7 +27,7 @@ kind::make_sure_kubernetes_tools_are_installed kind::get_kind_cluster_name kind::perform_kind_cluster_operation "start" build_images::prepare_prod_build -build_images::build_prod_images_with_group +build_images::build_prod_images kind::build_image_for_kubernetes_tests kind::load_image_to_kind_cluster kind::deploy_airflow_with_helm diff --git a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh similarity index 52% copy from scripts/ci/images/ci_wait_for_and_verify_ci_image.sh copy to scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh index e83b0d6..9b0d86f 100755 --- a/scripts/ci/images/ci_wait_for_and_verify_ci_image.sh +++ b/scripts/ci/kubernetes/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh @@ -15,43 +15,40 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - if [[ $1 == "" ]]; then - >&2 echo "Requires python MAJOR/MINOR version as first parameter" + >&2 echo "Requires Kubernetes_version as first parameter" exit 1 fi - -export PYTHON_MAJOR_MINOR_VERSION=$1 +export KUBERNETES_VERSION=$1 shift -# shellcheck source=scripts/ci/libraries/_script_init.sh -. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" - -function pull_ci_image() { - local image_name_with_tag="${GITHUB_REGISTRY_AIRFLOW_CI_IMAGE}:${GITHUB_REGISTRY_PULL_IMAGE_TAG}" - start_end::group_start "Pulling ${image_name_with_tag} image" - - push_pull_remove_images::pull_image_github_dockerhub "${AIRFLOW_CI_IMAGE}" "${image_name_with_tag}" - start_end::group_end - -} - -push_pull_remove_images::check_if_github_registry_wait_for_image_enabled - -build_image::configure_docker_registry +if [[ $1 == "" ]]; then + >&2 echo "Requires Python Major/Minor version as second parameter" + exit 1 +fi +export PYTHON_MAJOR_MINOR_VERSION=$1 +shift -export AIRFLOW_CI_IMAGE_NAME="${BRANCH_NAME}-python${PYTHON_MAJOR_MINOR_VERSION}-ci" +# Requires PARALLEL_JOB_STATUS -start_end::group_start "Waiting for ${AIRFLOW_CI_IMAGE_NAME} image to appear" +if [[ -z "${PARALLEL_JOB_STATUS=}" ]]; then + echo "Needs PARALLEL_JOB_STATUS to be set" + exit 1 +fi -push_pull_remove_images::wait_for_github_registry_image \ - "${AIRFLOW_CI_IMAGE_NAME}${GITHUB_REGISTRY_IMAGE_SUFFIX}" "${GITHUB_REGISTRY_PULL_IMAGE_TAG}" +echo +echo "KUBERNETES_VERSION: ${KUBERNETES_VERSION}" +echo "PYTHON_MAJOR_MINOR_VERSION: ${PYTHON_MAJOR_MINOR_VERSION}" +echo -build_images::prepare_ci_build +# shellcheck source=scripts/ci/libraries/_script_init.sh +. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" -pull_ci_image +kind::get_kind_cluster_name +trap 'echo $? > "${PARALLEL_JOB_STATUS}"; kind::perform_kind_cluster_operation "stop"' EXIT HUP INT TERM -verify_image::verify_ci_image "${AIRFLOW_CI_IMAGE}" +"$( dirname "${BASH_SOURCE[0]}" )/ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh" -start_end::group_end +export CLUSTER_FORWARDED_PORT="${FORWARDED_PORT_NUMBER}" +"$( dirname "${BASH_SOURCE[0]}" )/ci_run_kubernetes_tests.sh" diff --git a/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh new file mode 100755 index 0000000..88aa2cd --- /dev/null +++ b/scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -euo pipefail + +# We cannot perform full initialization because it will be done later in the "single run" scripts +# And some readonly variables are set there, therefore we only selectively reuse parallel lib needed +LIBRARIES_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../libraries/" && pwd) +# shellcheck source=scripts/ci/libraries/_all_libs.sh +source "${LIBRARIES_DIR}/_all_libs.sh" +export SEMAPHORE_NAME="kubernetes-tests" + +function get_maximum_parallel_k8s_jobs() { + docker_engine_resources::get_available_cpus_in_docker + if [[ -n ${RUNS_ON=} && ${RUNS_ON} != *"self-hosted"* ]]; then + echo + echo "${COLOR_YELLOW}This is a Github Public runner - for now we are forcing max parallel K8S tests jobs to 1 for those${COLOR_RESET}" + echo + export MAX_PARALLEL_K8S_JOBS="1" + else + if [[ ${MAX_PARALLEL_K8S_JOBS=} != "" ]]; then + echo + echo "${COLOR_YELLOW}Maximum parallel k8s jobs forced vi MAX_PARALLEL_K8S_JOBS = ${MAX_PARALLEL_K8S_JOBS}${COLOR_RESET}" + echo + else + MAX_PARALLEL_K8S_JOBS=${CPUS_AVAILABLE_FOR_DOCKER} + echo + echo "${COLOR_YELLOW}Maximum parallel k8s jobs set to number of CPUs available for Docker = ${MAX_PARALLEL_K8S_JOBS}${COLOR_RESET}" + echo + fi + fi + export MAX_PARALLEL_K8S_JOBS +} + +# Launches parallel building of images. Redirects output to log set the right directories +# $1 - test_specification +# $2 - bash file to execute in parallel +function run_kubernetes_test() { + local kubernetes_version=$1 + local python_version=$2 + local job="Cluster-${kubernetes_version}-python-${python_version}" + + mkdir -p "${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}" + export JOB_LOG="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}/stdout" + export PARALLEL_JOB_STATUS="${PARALLEL_MONITORED_DIR}/${SEMAPHORE_NAME}/${job}/status" + echo "Starting K8S tests for kubernetes version ${kubernetes_version}, python version: ${python_version}" + parallel --ungroup --bg --semaphore --semaphorename "${SEMAPHORE_NAME}" \ + --jobs "${MAX_PARALLEL_K8S_JOBS}" \ + "$(dirname "${BASH_SOURCE[0]}")/ci_setup_cluster_and_run_kubernetes_tests_single_job.sh" \ + "${kubernetes_version}" "${python_version}" >"${JOB_LOG}" 2>&1 +} + +function run_k8s_tests_in_parallel() { + parallel::cleanup_runner + start_end::group_start "Monitoring k8s tests" + parallel::initialize_monitoring + parallel::monitor_progress + + # In case there are more kubernetes versions than strings, we can reuse python versions so we add it twice here + local repeated_python_versions + # shellcheck disable=SC2206 + repeated_python_versions=(${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING} ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}) + local index=0 + for kubernetes_version in ${CURRENT_KUBERNETES_VERSIONS_AS_STRING} + do + index=$((index + 1)) + python_version=${repeated_python_versions[${index}]} + FORWARDED_PORT_NUMBER=$((38080 + index)) + export FORWARDED_PORT_NUMBER + API_SERVER_PORT=$((19090 + index)) + export API_SERVER_PORT + run_kubernetes_test "${kubernetes_version}" "${python_version}" "${@}" + done + set +e + parallel --semaphore --semaphorename "${SEMAPHORE_NAME}" --wait + parallel::kill_monitor + set -e + start_end::group_end +} + +initialization::set_output_color_variables + +parallel::make_sure_gnu_parallel_is_installed +parallel::make_sure_python_versions_are_specified +parallel::make_sure_kubernetes_versions_are_specified + +get_maximum_parallel_k8s_jobs + +run_k8s_tests_in_parallel "${@}" + +# this will exit with error code in case some of the tests failed +parallel::print_job_summary_and_return_status_code diff --git a/scripts/ci/kubernetes/kind-cluster-conf.yaml b/scripts/ci/kubernetes/kind-cluster-conf.yaml index f03c1b7..4e891f8 100644 --- a/scripts/ci/kubernetes/kind-cluster-conf.yaml +++ b/scripts/ci/kubernetes/kind-cluster-conf.yaml @@ -20,12 +20,12 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: ipv4 apiServerAddress: "127.0.0.1" - apiServerPort: 19090 + apiServerPort: {{API_SERVER_PORT}} nodes: - role: control-plane - role: worker extraPortMappings: - containerPort: 30007 - hostPort: 8080 + hostPort: {{FORWARDED_PORT_NUMBER}} listenAddress: "127.0.0.1" protocol: TCP diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 771a06d..edf9e29 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -462,7 +462,6 @@ function build_images::get_docker_image_names() { # Also enable experimental features of docker (we need `docker manifest` command) function build_image::configure_docker_registry() { if [[ ${USE_GITHUB_REGISTRY} == "true" ]]; then - start_end::group_start "Determine GitHub Registry token" local token="" if [[ "${GITHUB_REGISTRY}" == "ghcr.io" ]]; then # For now ghcr.io can only authenticate using Personal Access Token with package access scope. @@ -482,8 +481,6 @@ function build_image::configure_docker_registry() { echo exit 1 fi - start_end::group_end - start_end::group_start "Logging in to GitHub Registry" if [[ -z "${token}" ]] ; then verbosity::print_info verbosity::print_info "Skip logging in to GitHub Registry. No Token available!" @@ -497,14 +494,9 @@ function build_image::configure_docker_registry() { else verbosity::print_info "Skip Login to GitHub Registry ${GITHUB_REGISTRY} as token is missing" fi - start_end::group_end - - start_end::group_start "Make sure experimental docker features are enabled" local new_config new_config=$(jq '.experimental = "enabled"' "${HOME}/.docker/config.json") echo "${new_config}" > "${HOME}/.docker/config.json" - start_end::group_end - fi } @@ -861,9 +853,10 @@ function build_images::build_prod_images() { build_images::print_build_info if [[ ${SKIP_BUILDING_PROD_IMAGE} == "true" ]]; then - verbosity::print_info - verbosity::print_info "Skip building production image. Assume the one we have is good!" - verbosity::print_info + echo + echo "${COLOR_YELLOW}Skip building production image. Assume the one we have is good!${COLOR_RESET}" + echo "${COLOR_YELLOW}You must run './breeze build-image --production-image before for all python versions!${COLOR_RESET}" + echo return fi @@ -978,12 +971,6 @@ function build_images::build_prod_images() { fi } -function build_images::build_prod_images_with_group() { - start_end::group_start "Build PROD images ${AIRFLOW_PROD_BUILD_IMAGE}" - build_images::build_prod_images - start_end::group_end -} - # Waits for image tag to appear in GitHub Registry, pulls it and tags with the target tag # Parameters: # $1 - image name to wait for @@ -1084,7 +1071,7 @@ function build_images::build_prod_images_from_locally_built_airflow_packages() { build_airflow_packages::build_airflow_packages mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/" - build_images::build_prod_images_with_group + build_images::build_prod_images } # Useful information for people who stumble upon a pip check failure diff --git a/scripts/ci/libraries/_docker_engine_resources.sh b/scripts/ci/libraries/_docker_engine_resources.sh index f5ed3e6..0433359 100644 --- a/scripts/ci/libraries/_docker_engine_resources.sh +++ b/scripts/ci/libraries/_docker_engine_resources.sh @@ -19,10 +19,16 @@ function docker_engine_resources::print_overall_stats() { echo - echo "Overall resource statistics" + echo "Docker statistics" echo docker stats --all --no-stream --no-trunc - docker run --rm --entrypoint /bin/bash "debian:buster-slim" -c "cat /proc/meminfo" + echo + echo "Memory statistics" + echo + docker run --rm --entrypoint /bin/sh "alpine:latest" -c "free -m" + echo + echo "Disk statistics" + echo df -h || true } diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 1098e4a..191baee 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -508,14 +508,18 @@ function initialization::initialize_kubernetes_variables() { # Kubectl version export KUBECTL_VERSION=${KUBERNETES_VERSION:=${DEFAULT_KUBERNETES_VERSION}} # Local Kind path - export KIND_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kind" + export KIND_BINARY_PATH="${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}/kind" readonly KIND_BINARY_PATH # Local Helm path - export HELM_BINARY_PATH="${BUILD_CACHE_DIR}/bin/helm" + export HELM_BINARY_PATH="${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}/helm" readonly HELM_BINARY_PATH # local Kubectl path - export KUBECTL_BINARY_PATH="${BUILD_CACHE_DIR}/bin/kubectl" + export KUBECTL_BINARY_PATH="${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}/kubectl" readonly KUBECTL_BINARY_PATH + FORWARDED_PORT_NUMBER="${FORWARDED_PORT_NUMBER:="8080"}" + readonly FORWARDED_PORT_NUMBER + API_SERVER_PORT="${API_SERVER_PORT:="19090"}" + readonly API_SERVER_PORT } function initialization::initialize_git_variables() { diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index a8deaac..bb883ec 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -21,26 +21,24 @@ function kind::get_kind_cluster_name() { export KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:="airflow-python-${PYTHON_MAJOR_MINOR_VERSION}-${KUBERNETES_VERSION}"} # Name of the KinD cluster to connect to when referred to via kubectl export KUBECTL_CLUSTER_NAME=kind-${KIND_CLUSTER_NAME} - export KUBECONFIG="${BUILD_CACHE_DIR}/.kube/config" - mkdir -pv "${BUILD_CACHE_DIR}/.kube/" + export KUBECONFIG="${BUILD_CACHE_DIR}/${KIND_CLUSTER_NAME}/.kube/config" + mkdir -pv "${BUILD_CACHE_DIR}/${KIND_CLUSTER_NAME}/.kube/" touch "${KUBECONFIG}" } function kind::dump_kind_logs() { - start_end::group_start "Dumping logs from KinD" + verbosity::print_info "Dumping logs from KinD" local DUMP_DIR_NAME DUMP_DIR DUMP_DIR_NAME=kind_logs_$(date "+%Y-%m-%d")_${CI_BUILD_ID}_${CI_JOB_ID} DUMP_DIR="/tmp/${DUMP_DIR_NAME}" kind --name "${KIND_CLUSTER_NAME}" export logs "${DUMP_DIR}" - start_end::group_end } function kind::make_sure_kubernetes_tools_are_installed() { - start_end::group_start "Make sure Kubernetes tools are installed" SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${SYSTEM}-amd64"; - mkdir -pv "${BUILD_CACHE_DIR}/bin" + mkdir -pv "${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION}" if [[ -f "${KIND_BINARY_PATH}" ]]; then DOWNLOADED_KIND_VERSION=v"$(${KIND_BINARY_PATH} --version | awk '{ print $3 }')" echo "Currently downloaded kind version = ${DOWNLOADED_KIND_VERSION}" @@ -87,15 +85,17 @@ function kind::make_sure_kubernetes_tools_are_installed() { echo "Helm version ok" echo fi - PATH=${PATH}:${BUILD_CACHE_DIR}/bin - start_end::group_end + PATH=${PATH}:${BUILD_CACHE_DIR}/kubernetes-bin/${KUBERNETES_VERSION} } function kind::create_cluster() { - kind create cluster \ - --name "${KIND_CLUSTER_NAME}" \ - --config "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/kind-cluster-conf.yaml" \ - --image "kindest/node:${KUBERNETES_VERSION}" + sed "s/{{FORWARDED_PORT_NUMBER}}/${FORWARDED_PORT_NUMBER}/" < \ + "${AIRFLOW_SOURCES}/scripts/ci/kubernetes/kind-cluster-conf.yaml" | \ + sed "s/{{API_SERVER_PORT}}/${API_SERVER_PORT}/" | \ + kind create cluster \ + --name "${KIND_CLUSTER_NAME}" \ + --config - \ + --image "kindest/node:${KUBERNETES_VERSION}" echo echo "Created cluster ${KIND_CLUSTER_NAME}" echo @@ -106,7 +106,7 @@ function kind::delete_cluster() { echo echo "Deleted cluster ${KIND_CLUSTER_NAME}" echo - rm -rf "${HOME}/.kube/*" + rm -rf "${BUILD_CACHE_DIR}/${KIND_CLUSTER_NAME}/.kube/" } function kind::set_current_context() { @@ -122,7 +122,6 @@ function kind::perform_kind_cluster_operation() { echo exit 1 fi - start_end::group_start "Perform KinD cluster operation: ${1}" set -u OPERATION="${1}" @@ -229,7 +228,6 @@ function kind::perform_kind_cluster_operation() { exit 1 fi fi - start_end::group_end } function kind::check_cluster_ready_for_airflow() { @@ -250,7 +248,6 @@ function kind::check_cluster_ready_for_airflow() { } function kind::build_image_for_kubernetes_tests() { - start_end::group_start "Build image for kubernetes tests ${AIRFLOW_PROD_IMAGE_KUBERNETES}" cd "${AIRFLOW_SOURCES}" || exit 1 docker_v build --tag "${AIRFLOW_PROD_IMAGE_KUBERNETES}" . -f - <<EOF FROM ${AIRFLOW_PROD_IMAGE} @@ -261,13 +258,10 @@ COPY airflow/kubernetes_executor_templates/ \${AIRFLOW_HOME}/pod_templates/ EOF echo "The ${AIRFLOW_PROD_IMAGE_KUBERNETES} is prepared for test kubernetes deployment." - start_end::group_end } function kind::load_image_to_kind_cluster() { - start_end::group_start "Loading ${AIRFLOW_PROD_IMAGE_KUBERNETES} to ${KIND_CLUSTER_NAME}" kind load docker-image --name "${KIND_CLUSTER_NAME}" "${AIRFLOW_PROD_IMAGE_KUBERNETES}" - start_end::group_end } MAX_NUM_TRIES_FOR_HEALTH_CHECK=12 @@ -276,12 +270,7 @@ readonly MAX_NUM_TRIES_FOR_HEALTH_CHECK SLEEP_TIME_FOR_HEALTH_CHECK=10 readonly SLEEP_TIME_FOR_HEALTH_CHECK -FORWARDED_PORT_NUMBER=8080 -readonly FORWARDED_PORT_NUMBER - - function kind::wait_for_webserver_healthy() { - start_end::group_start "Waiting for webserver being healthy" num_tries=0 set +e sleep "${SLEEP_TIME_FOR_HEALTH_CHECK}" @@ -300,14 +289,10 @@ function kind::wait_for_webserver_healthy() { echo echo "Connection to 'airflow webserver' established on port ${FORWARDED_PORT_NUMBER}" echo - initialization::ga_env CLUSTER_FORWARDED_PORT "${FORWARDED_PORT_NUMBER}" - export CLUSTER_FORWARDED_PORT="${FORWARDED_PORT_NUMBER}" set -e - start_end::group_end } function kind::deploy_airflow_with_helm() { - start_end::group_start "Deploying Airflow with Helm" echo "Deleting namespace ${HELM_AIRFLOW_NAMESPACE}" kubectl delete namespace "${HELM_AIRFLOW_NAMESPACE}" >/dev/null 2>&1 || true kubectl delete namespace "test-namespace" >/dev/null 2>&1 || true @@ -331,7 +316,14 @@ function kind::deploy_airflow_with_helm() { kubectl -n "${HELM_AIRFLOW_NAMESPACE}" patch serviceaccount default -p '{"imagePullSecrets": [{"name": "regcred"}]}' fi - pushd "${AIRFLOW_SOURCES}/chart" >/dev/null 2>&1 || exit 1 + local chartdir + chartdir=$(mktemp -d) + traps::add_trap "rm -rf ${chartdir}" EXIT INT HUP TERM + # Copy chart to temporary directory to allow chart deployment in parallel + # Otherwise helm deployment will fail on renaming charts to tmpcharts + cp -r "${AIRFLOW_SOURCES}/chart" "${chartdir}" + + pushd "${chartdir}/chart" >/dev/null 2>&1 || exit 1 helm repo add stable https://charts.helm.sh/stable/ helm dep update helm install airflow . --namespace "${HELM_AIRFLOW_NAMESPACE}" \ @@ -343,15 +335,10 @@ function kind::deploy_airflow_with_helm() { --set "config.api.enable_experimental_api=true" echo popd > /dev/null 2>&1|| exit 1 - start_end::group_end } function kind::deploy_test_kubernetes_resources() { - start_end::group_start "Deploying Airflow with Helm" - echo - echo "Deploying Custom kubernetes resources" - echo + verbosity::print_info "Deploying Custom kubernetes resources" kubectl apply -f "scripts/ci/kubernetes/volumes.yaml" --namespace default kubectl apply -f "scripts/ci/kubernetes/nodeport.yaml" --namespace airflow - start_end::group_end } diff --git a/scripts/ci/libraries/_parallel.sh b/scripts/ci/libraries/_parallel.sh index 7239e82..935f465 100644 --- a/scripts/ci/libraries/_parallel.sh +++ b/scripts/ci/libraries/_parallel.sh @@ -194,7 +194,6 @@ function parallel::cleanup_runner() { start_end::group_end } - function parallel::make_sure_python_versions_are_specified() { if [[ -z "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING=}" ]]; then echo @@ -203,6 +202,18 @@ function parallel::make_sure_python_versions_are_specified() { exit 1 fi echo - echo "${COLOR_BLUE}Running parallel builds for those Python versions: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}!${COLOR_RESET}" + echo "${COLOR_BLUE}Running parallel builds for those Python versions: ${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING}${COLOR_RESET}" + echo +} + +function parallel::make_sure_kubernetes_versions_are_specified() { + if [[ -z "${CURRENT_KUBERNETES_VERSIONS_AS_STRING=}" ]]; then + echo + echo "${COLOR_RED}The CURRENT_KUBERNETES_VERSIONS_AS_STRING variable must be set and list K8S versions to use!${COLOR_RESET}" + echo + exit 1 + fi + echo + echo "${COLOR_BLUE}Running parallel builds for those Kubernetes versions: ${CURRENT_KUBERNETES_VERSIONS_AS_STRING}${COLOR_RESET}" echo } diff --git a/scripts/ci/libraries/_testing.sh b/scripts/ci/libraries/_testing.sh index 28d1fc6..638daf5 100644 --- a/scripts/ci/libraries/_testing.sh +++ b/scripts/ci/libraries/_testing.sh @@ -52,7 +52,7 @@ function testing::get_docker_compose_local() { function testing::get_maximum_parallel_test_jobs() { docker_engine_resources::get_available_cpus_in_docker - if [[ ${RUNS_ON} != *"self-hosted"* ]]; then + if [[ -n ${RUNS_ON=} && ${RUNS_ON} != *"self-hosted"* ]]; then echo echo "${COLOR_YELLOW}This is a Github Public runner - for now we are forcing max parallel Quarantined tests jobs to 1 for those${COLOR_RESET}" echo diff --git a/scripts/ci/libraries/_verbosity.sh b/scripts/ci/libraries/_verbosity.sh index dc3ca5a..68b356d 100644 --- a/scripts/ci/libraries/_verbosity.sh +++ b/scripts/ci/libraries/_verbosity.sh @@ -40,7 +40,7 @@ function verbosity::restore_exit_on_error_status() { # printed before execution. In case of DRY_RUN_DOCKER flag set to "true" # show the command to execute instead of executing them function docker_v { - if [[ ${DRY_RUN_DOCKER} != "false" ]]; then + if [[ ${DRY_RUN_DOCKER=} != "false" ]]; then echo echo "${COLOR_CYAN}docker" "${@}" "${COLOR_RESET}" echo diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index cf81b33..d639714 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -61,6 +61,7 @@ function output_all_basic_variables() { initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" initialization::ga_output python-versions-list-as-string "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[*]}" + initialization::ga_output kubernetes-versions-list-as-string "${CURRENT_KUBERNETES_VERSIONS[*]}" else initialization::ga_output python-versions \ "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" @@ -69,6 +70,7 @@ function output_all_basic_variables() { initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" initialization::ga_output python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" + initialization::ga_output kubernetes-versions-list-as-string "${DEFAULT_KUBERNETES_VERSION}" fi initialization::ga_output default-python-version "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" diff --git a/scripts/ci/testing/ci_run_quarantined_tests.sh b/scripts/ci/testing/ci_run_quarantined_tests.sh index 0c1108e..57e4aca 100755 --- a/scripts/ci/testing/ci_run_quarantined_tests.sh +++ b/scripts/ci/testing/ci_run_quarantined_tests.sh @@ -15,6 +15,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +set -euo pipefail # Enable automated tests execution RUN_TESTS="true" @@ -29,6 +30,8 @@ export SEMAPHORE_NAME # shellcheck source=scripts/ci/libraries/_script_init.sh . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh" +initialization::set_output_color_variables + BACKEND_TEST_TYPES=(mysql postgres sqlite) # Starts test types in parallel diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh index 5cb8d99..16aabbb 100755 --- a/scripts/in_container/entrypoint_ci.sh +++ b/scripts/in_container/entrypoint_ci.sh @@ -210,7 +210,7 @@ if [[ "${RUN_TESTS}" != "true" ]]; then fi set -u -export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}.xml" +export RESULT_LOG_FILE="/files/test_result-${TEST_TYPE}-${BACKEND}.xml" EXTRA_PYTEST_ARGS=( "--verbosity=0" @@ -218,7 +218,7 @@ EXTRA_PYTEST_ARGS=( "--durations=100" "--cov=airflow/" "--cov-config=.coveragerc" - "--cov-report=xml:/files/coverage.xml" + "--cov-report=xml:/files/coverage-${TEST_TYPE}-${BACKEND}.xml" "--color=yes" "--maxfail=50" "--pythonwarnings=ignore::DeprecationWarning"
