This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch optimize-multiplatform-builds in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 191b06e58835d26d9572fc5124cf13a4a9e77a4e Author: Jarek Potiuk <[email protected]> AuthorDate: Mon Mar 14 19:37:02 2022 +0100 fixup! Optimize Multiplatform cache builds --- .github/workflows/ci.yml | 1145 +---------------------------- scripts/ci/images/ci_stop_arm_instance.sh | 5 + 2 files changed, 7 insertions(+), 1143 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eec83b..d5315e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,1141 +223,6 @@ jobs: github.event_name == 'push' && steps.selective-checks.outputs.default-branch == 'main' - run-new-breeze-tests: - timeout-minutes: 10 - name: Breeze2 tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - steps: - - uses: actions/checkout@v2 - with: - persist-credentials: false - - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - run: python -m pytest ./dev/breeze/ -n auto --color=yes - - tests-ui: - timeout-minutes: 10 - name: React UI tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.run-ui-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup node" - uses: actions/setup-node@v2 - with: - node-version: 14 - - name: "Cache eslint" - uses: actions/cache@v2 - with: - path: 'airflow/ui/node_modules' - key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} - - run: yarn --cwd airflow/ui/ install --frozen-lockfile --non-interactive - - run: yarn --cwd airflow/ui/ run test - - tests-www: - timeout-minutes: 10 - name: React WWW tests - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.run-www-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup node" - uses: actions/setup-node@v2 - with: - node-version: 14 - - name: "Cache eslint" - uses: actions/cache@v2 - with: - path: 'airflow/www/node_modules' - key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} - - run: yarn --cwd airflow/www/ install --frozen-lockfile --non-interactive - - run: yarn --cwd airflow/www/ run test - - - test-openapi-client-generation: - timeout-minutes: 10 - name: "Test OpenAPI client generation" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.needs-api-codegen == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Generate client codegen diff" - run: ./scripts/ci/openapi/client_codegen_diff.sh - - test-examples-of-prod-image-building: - timeout-minutes: 60 - name: "Test examples of production image building" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.image-build == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Cache virtualenv environment" - uses: actions/cache@v2 - with: - path: '.build/.docker_venv' - key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - - name: "Test examples of PROD image building" - working-directory: docs/docker-stack/docker-examples - run: > - python -m pytest ../../../docker_tests/test_examples_of_prod_image_building.py - -n auto --color=yes - - test-docker-compose-quick-start: - timeout-minutes: 60 - name: "Test docker-compose quick start" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, prod-images] - if: needs.build-info.outputs.image-build == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Cache virtualenv environment" - uses: actions/cache@v2 - with: - path: '.build/.docker_venv' - key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - - name: "Test docker-compose quick start" - run: ./scripts/ci/images/ci_run_docker_compose_quick_start_test.sh - - ci-images: - timeout-minutes: 120 - name: "Wait for CI images" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - if: needs.build-info.outputs.image-build == 'true' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Cache virtualenv environment" - uses: actions/cache@v2 - with: - path: '.build/.docker_venv' - key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - - name: > - Wait for CI images - ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} - id: wait-for-images - # We wait for the images to be available either from the build-images workflow - # We are utilising single job to wait for all images because this job merely waits - # for the images to be available and run tests with the images. - run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh - env: - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - VERIFY_IMAGE: "true" - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - static-checks: - timeout-minutes: 30 - name: "Static checks" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - MOUNT_SELECTED_LOCAL_SOURCES: "true" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: needs.build-info.outputs.basic-checks-only == 'false' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Get Python version" - run: "echo \"::set-output name=host-python-version::$(python -c - 'import platform; print(platform.python_version())')\"" - id: host-python-version - - name: "Cache pre-commit local-installation" - uses: actions/cache@v2 - with: - path: ~/.local - key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('setup.py', 'setup.cfg') }}" - restore-keys: "\ -pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" - - name: "Cache pre-commit envs" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - - - name: "Cache eslint" - uses: actions/cache@v2 - with: - path: 'airflow/ui/node_modules' - key: ${{ runner.os }}-ui-node-modules-${{ hashFiles('airflow/ui/**/yarn.lock') }} - - name: "Static checks" - run: ./scripts/ci/static_checks/run_static_checks.sh - env: - VERBOSE: false - SKIP: "identity" - COLUMNS: 250 - - # Those checks are run if no image needs to be built for checks. This is for simple changes that - # Do not touch any of the python code or any of the important files that might require building - # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host - static-checks-basic-checks-only: - timeout-minutes: 30 - name: "Static checks: basic checks only" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - SKIP: "build,mypy,flake8,identity" - MOUNT_SELECTED_LOCAL_SOURCES: "true" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: needs.build-info.outputs.basic-checks-only == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - - name: > - Fetch incoming commit ${{ github.sha }} with its parent - uses: actions/checkout@v2 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: "Get Python version" - run: "echo \"::set-output name=host-python-version::$(python -c - 'import platform; print(platform.python_version())')\"" - id: host-python-version - - name: "Cache pre-commit local-installation" - uses: actions/cache@v2 - with: - path: ~/.local - key: "pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('setup.py', 'setup.cfg') }}" - restore-keys: "\ -pre-commit-local-installation-${{steps.host-python-version.outputs.host-python-version}}-" - - name: "Cache pre-commit envs" - uses: actions/cache@v2 - with: - path: ~/.cache/pre-commit - key: "pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}}-\ -${{ hashFiles('.pre-commit-config.yaml') }}" - restore-keys: pre-commit-basic-${{steps.host-python-version.outputs.host-python-version}} - - name: "Static checks: basic checks only" - run: ./scripts/ci/static_checks/run_basic_static_checks.sh "${{ github.sha }}" - env: - VERBOSE: false - docs: - timeout-minutes: 45 - name: "Build docs" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - if: needs.build-info.outputs.docs-build == 'true' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - uses: actions/setup-python@v2 - with: - python-version: ${{needs.build-info.outputs.defaultPythonVersion}} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Fetch inventory versions" - run: ./scripts/ci/docs/ci_docs_prepare.sh - - uses: actions/cache@v2 - id: cache-doc-inventories - with: - path: ./docs/_inventory_cache/ - key: docs-inventory-v1-${{ hashFiles('constraints.txt') }} - restore-keys: | - docs-inventory-${{ hashFiles('constraints.txt') }} - docs-inventory- - - name: "Build docs" - run: ./scripts/ci/docs/ci_docs.sh - - name: Configure AWS credentials - uses: ./.github/actions/configure-aws-credentials - if: > - github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && - github.event_name == 'push' - with: - aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-central-1 - - name: "Upload documentation to AWS S3" - if: > - github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' && - github.event_name == 'push' - run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs - - prepare-test-provider-packages-wheel: - timeout-minutes: 40 - name: "Build and test provider packages wheel" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - AIRFLOW_EXTRAS: "all" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - VERSION_SUFFIX_FOR_PYPI: ".dev0" - NON_INTERACTIVE: "true" - GENERATE_PROVIDERS_ISSUE: "true" - if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - if: needs.build-info.outputs.default-branch == 'main' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Prepare provider documentation" - run: ./scripts/ci/provider_packages/ci_prepare_provider_documentation.sh - - name: "Prepare provider packages: wheel" - run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh - env: - PACKAGE_FORMAT: "wheel" - - name: "Prepare airflow package: wheel" - run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh - env: - PACKAGE_FORMAT: "wheel" - - name: "Install and test provider packages and airflow via wheel files" - run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh - env: - USE_AIRFLOW_VERSION: "wheel" - PACKAGE_FORMAT: "wheel" - - name: "Install and test provider packages and airflow on Airflow 2.1 files" - run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh - env: - USE_AIRFLOW_VERSION: "2.1.0" - SKIP_TWINE_CHECK: "true" - PACKAGE_FORMAT: "wheel" - - prepare-test-provider-packages-sdist: - timeout-minutes: 40 - name: "Build and test provider packages sdist" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - AIRFLOW_EXTRAS: "all" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - VERSION_SUFFIX_FOR_PYPI: ".dev0" - NON_INTERACTIVE: "true" - GENERATE_PROVIDERS_ISSUE: "true" - if: needs.build-info.outputs.image-build == 'true' && needs.build-info.outputs.default-branch == 'main' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - if: needs.build-info.outputs.default-branch == 'main' - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Prepare provider packages: sdist" - run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh - env: - PACKAGE_FORMAT: "sdist" - - name: "Prepare airflow package: sdist" - run: ./scripts/ci/build_airflow/ci_build_airflow_packages.sh - env: - PACKAGE_FORMAT: "sdist" - - name: "Upload provider distribution artifacts" - uses: actions/upload-artifact@v2 - with: - name: airflow-provider-packages - path: "./dist/apache-airflow-providers-*.tar.gz" - retention-days: 1 - - name: "Install and test provider packages and airflow via sdist files" - run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh - env: - USE_AIRFLOW_VERSION: "sdist" - PACKAGE_FORMAT: "sdist" - - tests-helm: - timeout-minutes: 80 - name: "Python unit tests for helm chart" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - MOUNT_SELECTED_LOCAL_SOURCES: "true" - TEST_TYPES: "Helm" - BACKEND: "" - DB_RESET: "false" - PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}} - if: > - needs.build-info.outputs.needs-helm-tests == 'true' && - (github.repository == 'apache/airflow' || github.event_name != 'schedule') && - needs.build-info.outputs.default-branch == 'main' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Tests: Helm" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-helm - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-helm - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' - with: - name: > - coverage-helm - path: "./files/coverage*.xml" - retention-days: 7 - - tests-postgres: - timeout-minutes: 130 - name: > - Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}: - ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - postgres-version: ${{ fromJson(needs.build-info.outputs.postgresVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.postgresExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: postgres - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - POSTGRES_VERSION: ${{ matrix.postgres-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' - with: - name: > - coverage-postgres-${{matrix.python-version}}-${{matrix.postgres-version}} - path: "./files/coverage*.xml" - retention-days: 7 - - tests-mysql: - timeout-minutes: 130 - name: > - MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - mysql-version: ${{ fromJson(needs.build-info.outputs.mysqlVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.mysqlExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: mysql - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - MYSQL_VERSION: ${{ matrix.mysql-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' - with: - name: coverage-mysql-${{matrix.python-version}}-${{matrix.mysql-version}} - path: "./files/coverage*.xml" - retention-days: 7 - - tests-mssql: - timeout-minutes: 130 - name: > - MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - mssql-version: ${{ fromJson(needs.build-info.outputs.mssqlVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.mssqlExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: mssql - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - MSSQL_VERSION: ${{ matrix.mssql-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}}-${{matrix.mssql-version}} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' - with: - name: coverage-mssql-${{matrix.python-version}}-${{matrix.mssql-version}} - path: "./files/coverage*.xml" - retention-days: 7 - - tests-sqlite: - timeout-minutes: 130 - name: > - Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.testTypes}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} - exclude: ${{ fromJson(needs.build-info.outputs.sqliteExclude) }} - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - TEST_TYPES: "${{needs.build-info.outputs.testTypes}}" - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Test downgrade" - run: ./scripts/ci/testing/run_downgrade_test.sh - - name: "Tests: ${{needs.build-info.outputs.testTypes}}" - run: ./scripts/ci/testing/ci_run_airflow_testing.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-${{matrix.python-version}} - path: './files/airflow_logs*' - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-sqlite-${{matrix.python-version}} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' - with: - name: coverage-sqlite-${{matrix.python-version}} - path: ./files/coverage*.xml - retention-days: 7 - - tests-quarantined: - timeout-minutes: 60 - name: "Quarantined tests" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - continue-on-error: true - needs: [build-info, ci-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} - MYSQL_VERSION: ${{needs.build-info.outputs.defaultMySQLVersion}} - POSTGRES_VERSION: ${{needs.build-info.outputs.defaultPostgresVersion}} - TEST_TYPES: "Quarantined" - NUM_RUNS: 10 - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - if: needs.build-info.outputs.run-tests == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - name: "Set issue id for main" - if: github.ref == 'refs/heads/main' - run: | - echo "ISSUE_ID=10118" >> $GITHUB_ENV - - name: "Set issue id for v1-10-stable" - if: github.ref == 'refs/heads/v1-10-stable' - run: | - echo "ISSUE_ID=10127" >> $GITHUB_ENV - - name: "Set issue id for v1-10-test" - if: github.ref == 'refs/heads/v1-10-test' - run: | - echo "ISSUE_ID=10128" >> $GITHUB_ENV - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Pull CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}" - run: ./scripts/ci/images/ci_pull_ci_image_on_ci.sh - - name: "Tests: Quarantined" - run: ./scripts/ci/testing/ci_run_quarantined_tests.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload Quarantine test results" - uses: actions/upload-artifact@v2 - if: always() - with: - name: quarantined-tests - path: "files/test_result-*.xml" - retention-days: 7 - - name: "Upload airflow logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: airflow-logs-quarantined-${{ matrix.backend }} - path: "./files/airflow_logs*" - retention-days: 7 - - name: "Upload container logs" - uses: actions/upload-artifact@v2 - if: failure() - with: - name: container-logs-quarantined-${{ matrix.backend }} - path: "./files/container_logs*" - retention-days: 7 - - name: "Upload artifact for coverage" - uses: actions/upload-artifact@v2 - if: needs.build-info.outputs.runCoverage == 'true' - with: - name: coverage-quarantined-${{ matrix.backend }} - path: "./files/coverage*.xml" - retention-days: 7 - - upload-coverage: - timeout-minutes: 15 - name: "Upload coverage" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - continue-on-error: true - needs: - - build-info - - tests-postgres - - tests-sqlite - - tests-mysql - - tests-mssql - - tests-quarantined - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - # Only upload coverage on merges to main - if: needs.build-info.outputs.runCoverage == 'true' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Download all artifacts from the current build" - uses: actions/download-artifact@v2 - with: - path: ./coverage-files - - name: "Removes unnecessary artifacts" - run: ls ./coverage-files | grep -v coverage | xargs rm -rf - - name: "Upload all coverage reports to codecov" - uses: ./.github/actions/codecov-action - with: - directory: "./coverage-files" - - prod-images: - timeout-minutes: 120 - name: "Wait for PROD images" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, ci-images] - if: needs.build-info.outputs.image-build == 'true' - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{ needs.build-info.outputs.defaultPythonVersion }} - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Cache virtualenv environment" - uses: actions/cache@v2 - with: - path: '.build/.docker_venv' - key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - - name: > - Wait for PROD images - ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} - # We wait for the images to be available either from "build-images.yml' run as pull_request_target. - # We are utilising single job to wait for all images because this job merely waits - # For the images to be available and test them. - # - id: wait-for-images - run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh - env: - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - VERIFY_IMAGE: "true" - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - tests-kubernetes: - timeout-minutes: 50 - name: Helm Chart; ${{matrix.executor}} - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, prod-images] - strategy: - matrix: - executor: [KubernetesExecutor, CeleryExecutor, LocalExecutor] - fail-fast: false - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: postgres - RUN_TESTS: "true" - RUNTIME: "kubernetes" - KUBERNETES_MODE: "image" - EXECUTOR: ${{matrix.executor}} - KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" - HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - CURRENT_KUBERNETES_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.kubernetesVersionsListAsString}} - if: > - ( needs.build-info.outputs.run-kubernetes-tests == 'true' || - needs.build-info.outputs.needs-helm-tests == 'true' ) && - needs.build-info.outputs.default-branch == 'main' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Get all PROD images" - run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh - env: - VERIFY_IMAGE: "false" - - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v2 - with: - path: ".build/kubernetes-bin" - key: "kubernetes-binaries --${{ needs.build-info.outputs.defaultKindVersion }}\ --${{ needs.build-info.outputs.defaultHelmVersion }}" - restore-keys: "kubernetes-binaries" - - name: "Kubernetes Tests" - run: ./scripts/ci/kubernetes/ci_setup_clusters_and_run_kubernetes_tests_in_parallel.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload KinD logs" - uses: actions/upload-artifact@v2 - if: failure() || cancelled() - with: - name: > - kind-logs-${{matrix.executor}} - path: /tmp/kind_logs_* - retention-days: 7 - - tests-helm-executor-upgrade: - timeout-minutes: 80 - name: Helm Chart Executor Upgrade - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: [build-info, prod-images] - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - BACKEND: postgres - RUN_TESTS: "true" - RUNTIME: "kubernetes" - KUBERNETES_MODE: "image" - EXECUTOR: "KubernetesExecutor" - KIND_VERSION: "${{ needs.build-info.outputs.defaultKindVersion }}" - HELM_VERSION: "${{ needs.build-info.outputs.defaultHelmVersion }}" - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.pythonVersionsListAsString}} - CURRENT_KUBERNETES_VERSIONS_AS_STRING: > - ${{needs.build-info.outputs.kubernetesVersionsListAsString}} - if: > - needs.build-info.outputs.run-kubernetes-tests == 'true' && - needs.build-info.outputs.default-branch == 'main' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: "Get all PROD images" - run: ./scripts/ci/images/ci_wait_for_and_verify_all_prod_images.sh - env: - VERIFY_IMAGE: "false" - - name: "Cache virtualenv for kubernetes testing" - uses: actions/cache@v2 - with: - path: ".build/.kubernetes_venv" - key: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}\ - -${{needs.build-info.outputs.kubernetesVersionsListAsString}} - -${{needs.build-info.outputs.pythonVersionsListAsString}} - -${{ hashFiles('setup.py','setup.cfg') }}" - restore-keys: "kubernetes-${{ needs.build-info.outputs.defaultPythonVersion }}-\ - -${{needs.build-info.outputs.kubernetesVersionsListAsString}} - -${{needs.build-info.outputs.pythonVersionsListAsString}}" - - name: "Cache bin folder with tools for kubernetes testing" - uses: actions/cache@v2 - with: - path: ".build/kubernetes-bin" - key: "kubernetes-binaries - -${{ needs.build-info.outputs.defaultKindVersion }}\ - -${{ needs.build-info.outputs.defaultHelmVersion }}" - restore-keys: "kubernetes-binaries" - - name: "Kubernetes Helm Chart Executor Upgrade Tests" - run: ./scripts/ci/kubernetes/ci_upgrade_cluster_with_different_executors_in_parallel.sh - env: - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Upload KinD logs" - uses: actions/upload-artifact@v2 - if: failure() || cancelled() - with: - name: > - kind-logs-KubernetesExecutor - path: /tmp/kind_logs_* - retention-days: 7 - - constraints: - permissions: - contents: write - timeout-minutes: 40 - name: "Constraints" - runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} - needs: - - build-info - - ci-images - - prod-images - - static-checks - - tests-sqlite - - tests-mysql - - tests-mssql - - tests-postgres - env: - RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - CURRENT_PYTHON_MAJOR_MINOR_VERSIONS_AS_STRING: ${{needs.build-info.outputs.pythonVersionsListAsString}} - if: needs.build-info.outputs.upgradeToNewerDependencies != 'false' - steps: - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v2 - with: - persist-credentials: false - submodules: recursive - - name: "Setup python" - uses: actions/setup-python@v2 - with: - python-version: ${{ needs.build-info.outputs.defaultPythonVersion }} - cache: 'pip' - cache-dependency-path: ./dev/breeze/setup* - - run: python -m pip install --editable ./dev/breeze/ - - name: "Free space" - run: airflow-freespace - - name: > - Wait for CI images - ${{ needs.build-info.outputs.pythonVersions }}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }} - run: ./scripts/ci/images/ci_wait_for_and_verify_all_ci_images.sh - env: - VERIFY_IMAGE: "false" - - name: "Generate constraints with PyPI providers" - run: ./scripts/ci/constraints/ci_generate_all_constraints.sh - env: - GENERATE_CONSTRAINTS_MODE: "pypi-providers" - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Generate constraints with source providers" - run: ./scripts/ci/constraints/ci_generate_all_constraints.sh - env: - GENERATE_CONSTRAINTS_MODE: "source-providers" - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Generate constraints without providers" - run: ./scripts/ci/constraints/ci_generate_all_constraints.sh - env: - GENERATE_CONSTRAINTS_MODE: "no-providers" - PR_LABELS: "${{ needs.build-info.outputs.pullRequestLabels }}" - - name: "Set constraints branch name" - id: constraints-branch - run: ./scripts/ci/constraints/ci_branch_constraints.sh - # only actually checkout and push it when we are in apache/airflow repository - - name: Checkout ${{ steps.constraints-branch.outputs.branch }} - uses: actions/checkout@v2 - if: > - github.repository == 'apache/airflow' && - (github.event_name == 'push' || github.event_name == 'schedule') - with: - path: "repo" - ref: ${{ steps.constraints-branch.outputs.branch }} - persist-credentials: false - - name: "Commit changed constraint files for ${{needs.build-info.outputs.pythonVersions}}" - run: ./scripts/ci/constraints/ci_commit_constraints.sh - if: > - github.repository == 'apache/airflow' && - (github.event_name == 'push' || github.event_name == 'schedule') - - name: "Push changes" - uses: ./.github/actions/github-push-action - if: > - github.repository == 'apache/airflow' && - (github.event_name == 'push' || github.event_name == 'schedule') - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ steps.constraints-branch.outputs.branch }} - directory: "repo" # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build # is executed as result of direct push to "main" or one of the "vX-Y-test" branches @@ -1371,12 +236,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: - build-info - - constraints - - docs - # Only run it for direct pushes and scheduled builds - if: > - (github.event_name == 'push' || github.event_name == 'schedule' || - github.event_name == 'pull_request') && github.repository == 'apache/airflow' strategy: matrix: python-version: ${{ fromJson(needs.build-info.outputs.pythonVersions) }} @@ -1403,14 +262,14 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" - run: python -m pip install --editable ./dev/breeze/ - name: "Free space" run: airflow-freespace + - name: "Start ARM instance" + run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: "Build CI image cache and push ${{env.PYTHON_MAJOR_MINOR_VERSION}}" run: ./scripts/ci/images/ci_build_ci_image_on_ci.sh env: PREPARE_BUILDX_CACHE: "true" GITHUB_REGISTRY_PULL_IMAGE_TAG: "latest" GITHUB_REGISTRY_PUSH_IMAGE_TAG: "latest" - - name: "Start ARM instance" - run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: "Build CI image cache and push ${{env.PYTHON_MAJOR_MINOR_VERSION}}" run: ./scripts/ci/images/ci_build_prod_image_on_ci.sh env: diff --git a/scripts/ci/images/ci_stop_arm_instance.sh b/scripts/ci/images/ci_stop_arm_instance.sh index 0a0a8ce..a14ff08 100755 --- a/scripts/ci/images/ci_stop_arm_instance.sh +++ b/scripts/ci/images/ci_stop_arm_instance.sh @@ -23,6 +23,11 @@ WORKING_DIR="/tmp/armdocker" INSTANCE_INFO="${WORKING_DIR}/instance_info.json" function stop_arm_instance() { + if [[ ! -f "${INSTANCE_INFO}" ]]; then + echo "${COLOR_YELLOW}Skip killing ARM instance as it has not been started.${COLOR_RESET}" + return + fi + set -x INSTANCE_ID=$(jq < "${INSTANCE_INFO}" ".Instances[0].InstanceId" -r) aws ec2 stop-instances --instance-ids "${INSTANCE_ID}" }
