This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch extract-prod-check-workflows in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 4a29eb7fc92b2ca72184358af5128946001158b6 Author: Jarek Potiuk <[email protected]> AuthorDate: Sun Mar 3 11:37:35 2024 +0100 Refactor and extract production image checks to separate workflows Our CI workflow yaml had become a huge flying-spaghetti-monster piece of monolitic workflow, with lots of a duplication. This is the first step (of quite a few) where we separate parts of the workflow to micro-workflows. GitHub Actions has the option to call workflows from other workflows and (additionally to composite actions) such composite workflows can implement a nice workflow code reuse. This - first- attempt is to check if we can do it easily and it should be followed by a series of similar extraction steps. --- .github/workflows/build-images.yml | 15 +- .github/workflows/ci.yml | 425 ++++----------------- .github/workflows/prod-image-build.yml | 103 +++++ .github/workflows/prod-image-extra-checks.yml | 93 +++++ .../pre_commit_checkout_no_credentials.py | 2 + 5 files changed, 288 insertions(+), 350 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index da76fcbeff..6179f34c1d 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -42,11 +42,6 @@ env: IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" USE_SUDO: "true" INCLUDE_SUCCESS_OUTPUTS: "true" - # Version of Python used for reproducibility of the packages built - # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing - # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible - # TODO: remove me when we switch to Python 3.9 as minimal version - REPRODUCIBLE_PYTHON_VERSION: "3.9" concurrency: group: build-${{ github.event.pull_request.number || github.ref }} @@ -60,6 +55,12 @@ jobs: env: TARGET_BRANCH: ${{ github.event.pull_request.base.ref }} outputs: + # Version of Python used for reproducibility of the packages built + # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing + # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible + # TODO: remove me when we switch to Python 3.9 as minimal version + breeze-python-version: "3.9" + image-tag: ${{ github.event.pull_request.head.sha || github.sha }} python-versions: ${{ steps.selective-checks.outputs.python-versions }} upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} all-python-versions-list-as-string: >- @@ -306,7 +307,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} uses: ./.github/actions/build-prod-images with: @@ -388,7 +389,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze - name: > - Build ARM CI images ${{ env.IMAGE_TAG }} + Build ARM CI images ${{ needs.build-info.outputs.image-tag }} ${{needs.build-info.outputs.all-python-versions-list-as-string}} run: > breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cb19201dd..b9078345b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,11 +52,6 @@ env: INCLUDE_NOT_READY_PROVIDERS: "true" AIRFLOW_ENABLE_AIP_44: "true" MOUNT_SOURCES: "skip" - # Version of Python used for reproducibility of the packages built - # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing - # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible - # TODO: remove me when we switch to Python 3.9 as minimal version - REPRODUCIBLE_PYTHON_VERSION: "3.9" concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} @@ -74,6 +69,12 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} outputs: + # Version of Python used for reproducibility of the packages built + # Python 3.8 tarfile produces different tarballs than Python 3.9+ tarfile that's why we are forcing + # Python 3.9 for all release preparation commands to make sure that the tarballs are reproducible + # TODO: remove me when we switch to Python 3.9 as minimal version + breeze-python-version: "3.9" + image-tag: ${{ github.event.pull_request.head.sha || github.sha }} cache-directive: ${{ steps.selective-checks.outputs.cache-directive }} affected-providers-list-as-string: >- ${{ steps.selective-checks.outputs.affected-providers-list-as-string }} @@ -302,7 +303,7 @@ jobs: PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} BUILD_TIMEOUT_MINUTES: 70 - - name: Verify CI images ${{ matrix.python-version }}:${{ env.IMAGE_TAG }} + - name: Verify CI images ${{ matrix.python-version }}:${{ needs.build-info.outputs.image-tag }} run: breeze ci-image verify --python ${{ matrix.python-version }} env: PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} @@ -471,7 +472,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} id: wait-for-images run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest env: @@ -504,10 +505,10 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} + - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} run: breeze ci-image pull --run-in-parallel --tag-as-latest - - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} run: breeze ci-image verify --run-in-parallel env: PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} @@ -866,7 +867,7 @@ jobs: Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider documentation" @@ -929,7 +930,7 @@ jobs: Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider packages: sdist" @@ -1000,7 +1001,7 @@ jobs: Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: "Prepare provider packages: wheel" @@ -1115,7 +1116,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} - name: Setup git for tagging run: | git config --global user.email "[email protected]" @@ -1849,342 +1850,78 @@ jobs: echo Total number of unique warnings $(cat ./artifacts/test-warnings*/* | sort | uniq | wc -l) build-prod-images: - strategy: - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} - timeout-minutes: 80 - name: ${{needs.build-info.outputs.build-job-description}} PROD image (main) ${{matrix.python-version}} - runs-on: ["ubuntu-22.04"] + name: Build PROD images (in-workflow) needs: [build-info, build-ci-images] - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} - persist-credentials: false - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: Build PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + uses: ./.github/workflows/prod-image-build.yml + with: + build-type: "Regular" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }} + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} - build-prod-images-bullseye: - strategy: - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} - timeout-minutes: 80 - name: Build Bullseye PROD image (main) ${{matrix.python-version}} - runs-on: ["ubuntu-22.04"] + prod-image-extra-checks-main: + name: PROD image checks (main) + # Here we just need to wait for CI images: we build provider packages and can use source constraints needs: [build-info, build-ci-images] - if: needs.build-info.outputs.canary-run == 'true' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - uses: actions/checkout@v3 - with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} - persist-credentials: false - submodules: recursive - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: Build Bullseye PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - DEBIAN_VERSION: "bullseye" - # Do not override the "bookworm" image - just push a new bullseye image - # TODO: improve caching for that build - IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" + uses: ./.github/workflows/prod-image-extra-checks.yml + with: + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + if: needs.build-info.outputs.default-branch == 'main' - build-prod-images-mysql-client: - strategy: - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} - timeout-minutes: 80 - name: Build MySQL Client PROD image (main) ${{ matrix.python-version}} - runs-on: ["ubuntu-22.04"] - needs: [build-info, build-ci-images] - if: needs.build-info.outputs.canary-run == 'true' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - uses: actions/checkout@v3 - with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} - persist-credentials: false - submodules: recursive - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - - name: Build MySQL Client PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch == 'main' - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - INSTALL_MYSQL_CLIENT_TYPE: "mysql" - # Do not override the "mariadb" (original) image - just push a new mysql image - # TODO: improve caching for that build - IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" - - - build-prod-images-release-branch: - strategy: - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} - timeout-minutes: 80 - name: > - ${{needs.build-info.outputs.build-job-description}} PROD image (v2_*_test) ${{matrix.python-version}} - runs-on: ["ubuntu-22.04"] + prod-image-extra-checks-release-branch: + name: PROD image checks (release) + # Here we need to wait for generate-constraints to complete because we have to use PyPI constraints needs: [build-info, generate-constraints] - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - uses: actions/checkout@v4 - with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} - persist-credentials: false - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - name: Build Release PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - - build-prod-images-bullseye-release-branch: - strategy: - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} - timeout-minutes: 80 - name: Build Bullseye PROD image (v2_*_test) ${{matrix.python-version}} - runs-on: ["ubuntu-22.04"] - needs: [build-info, generate-constraints] - if: needs.build-info.outputs.canary-run == 'true' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - uses: actions/checkout@v3 - with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} - persist-credentials: false - submodules: recursive - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - name: Build Bullseye Release PROD Image ${{ matrix.python-version }}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - DEBIAN_VERSION: "bullseye" - # Do not override the "bookworm" image - just push a new bullseye image - # TODO: improve caching for that build - IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}" + uses: ./.github/workflows/prod-image-extra-checks.yml + with: + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + image-tag: ${{ needs.build-info.outputs.image-tag }} + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + if: needs.build-info.outputs.default-branch != 'main' - build-prod-images-mysql-release-branch: - strategy: - matrix: - python-version: ${{fromJson(needs.build-info.outputs.python-versions)}} - timeout-minutes: 80 - name: Build MySQL PROD image (v2_*_test) ${{matrix.python-version}} - runs-on: ["ubuntu-22.04"] - needs: [build-info, generate-constraints] - if: needs.build-info.outputs.canary-run == 'true' - env: - DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} - DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - BACKEND: sqlite - VERSION_SUFFIX_FOR_PYPI: "dev0" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - uses: actions/checkout@v3 - with: - ref: ${{ needs.build-info.outputs.targetCommitSha }} - persist-credentials: false - submodules: recursive - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - - name: Build Mysql PROD Image ${{matrix.python-version}}:${{env.IMAGE_TAG}} - uses: ./.github/actions/build-prod-images - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.default-branch != 'main' - with: - build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} - chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} - python-version: ${{ matrix.python-version }} - env: - UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} - DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} - INSTALL_MYSQL_CLIENT_TYPE: "mysql" - # Do not override the "mariadb" image - just push a new mysql image - # TODO: improve caching for that build - IMAGE_TAG: "mysql-${{ github.event.pull_request.head.sha || github.sha }}" wait-for-prod-images: timeout-minutes: 80 name: "Wait for PROD images" - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: [build-info, wait-for-ci-images, build-prod-images, build-prod-images-release-branch] - if: needs.build-info.outputs.prod-image-build == 'true' + runs-on: ["ubuntu-22.04"] + needs: [build-info, wait-for-ci-images, build-prod-images] + # This strange condition below is equivalent to: "all success or skipped" + # The "build-;rod-images" step might be skipped, in case production building happens in the + # "build-images" workflow, and in this case we sill want to wait for PROD images and run depending tests + if: > + always() && !failure() && !cancelled() && + needs.build-info.outputs.prod-image-build == 'true' env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" BACKEND: sqlite PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" - # Force more parallelism for pull even on public images + # Force more parallelism for pull on public images PARALLELISM: 6 steps: - name: Cleanup repo @@ -2198,7 +1935,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze if: needs.build-info.outputs.in-workflow-build == 'false' - - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} # We wait for the images to be available either from "build-images.yml' run as pull_request_target # or from build-prod-images (or build-prod-images-release-branch) above. # We are utilising single job to wait for all images because this job merely waits @@ -2225,7 +1962,9 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze - - name: Pull PROD image "${{needs.build-info.outputs.default-python-version}}":${{ env.IMAGE_TAG }} + - name: > + Pull PROD image + "${{needs.build-info.outputs.default-python-version}}":${{ needs.build-info.outputs.image-tag }} run: breeze prod-image pull --tag-as-latest env: PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" @@ -2264,13 +2003,13 @@ jobs: persist-credentials: false - name: "Install Breeze" uses: ./.github/actions/breeze - - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} run: breeze prod-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} # Force more parallelism for pull even on public images PARALLELISM: 6 - - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + - name: Verify PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} run: breeze prod-image verify --run-in-parallel env: PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} @@ -2306,7 +2045,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze - - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }} + - name: Pull PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} run: breeze prod-image pull --run-in-parallel --tag-as-latest env: PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} @@ -2438,7 +2177,7 @@ jobs: Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} uses: ./.github/actions/prepare_breeze_and_image with: - python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + python-version: ${{ needs.build-info.outputs.breeze-python-version }} env: # Always use default Python version of CI image for preparing packages PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" @@ -2545,7 +2284,7 @@ jobs: - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - name: > - Build CI ARM images ${{ env.IMAGE_TAG }} + Build CI ARM images ${{ needs.build-info.outputs.image-tag }} ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} run: > breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml new file mode 100644 index 0000000000..05176dda61 --- /dev/null +++ b/.github/workflows/prod-image-build.yml @@ -0,0 +1,103 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Build PROD images (in-workflow) +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + build-type: + required: true + type: string + debian-version: + type: string + default: "bookworm" + install-mysql-client-type: + type: string + default: "mariadb" + image-tag: + required: true + type: string + in-workflow-build: + required: true + type: string + python-versions: + required: true + type: string + branch: + required: true + type: string + build-provider-packages: + required: true + type: string + upgrade-to-newer-dependencies: + required: true + type: string + breeze-python-version: + required: true + type: string + chicken-egg-providers: + required: true + type: string + constraints-branch: + required: true + type: string + docker-cache: + required: true + type: string + debug-resources: + required: true + type: string +jobs: + build-prod-images: + strategy: + fail-fast: true + matrix: + python-version: ${{fromJson(inputs.python-versions)}} + timeout-minutes: 80 + name: Build PROD ${{inputs.build-type}} image ${{matrix.python-version}} + runs-on: ["ubuntu-22.04"] + if: ${{ inputs.in-workflow-build == 'true' }} + env: + DEFAULT_BRANCH: ${{ inputs.branch }} + DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} + BACKEND: sqlite + VERSION_SUFFIX_FOR_PYPI: "dev0" + DEBUG_RESOURCES: ${{ inputs.debug-resources}} + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - uses: actions/checkout@v4 + with: + ref: ${{ needs.build-info.outputs.targetCommitSha }} + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + python-version: ${{ inputs.breeze-python-version }} + - name: Build PROD Image ${{ matrix.python-version }}:${{ inputs.image-tag }} + uses: ./.github/actions/build-prod-images + with: + build-provider-packages: ${{ inputs.build-provider-packages }} + chicken-egg-providers: ${{ inputs.chicken-egg-providers }} + python-version: ${{ matrix.python-version }} + env: + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ inputs.cache-directive }} + DEBIAN_VERSION: ${{ inputs.debian-version }} + INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} + INCLUDE_NOT_READY_PROVIDERS: "true" diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml new file mode 100644 index 0000000000..2d0be2fdcb --- /dev/null +++ b/.github/workflows/prod-image-extra-checks.yml @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: PROD images extra checks +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + python-versions: + required: true + type: string + branch: + required: true + type: string + image-tag: + required: true + type: string + build-provider-packages: + required: true + type: string + upgrade-to-newer-dependencies: + required: true + type: string + breeze-python-version: + required: true + type: string + chicken-egg-providers: + required: true + type: string + constraints-branch: + required: true + type: string + docker-cache: + required: true + type: string + debug-resources: + required: true + type: string + canary-run: + required: true + type: string +jobs: + bullseye-image: + uses: ./.github/workflows/prod-image-build.yml + with: + build-type: "Bullseye" + image-tag: bullseye-${{ inputs.image-tag }} + debian-version: "bullseye" + python-versions: ${{ inputs.python-versions }} + branch: ${{ inputs.branch }} + # Always build images during the extra checks + in-workflow-build: 'true' + build-provider-packages: ${{ inputs.build-provider-packages }} + upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} + breeze-python-version: ${{ inputs.breeze-python-version }} + chicken-egg-providers: ${{ inputs.chicken-egg-providers }} + constraints-branch: ${{ inputs.constraints-branch }} + docker-cache: ${{ inputs.docker-cache }} + debug-resources: ${{ inputs.debug-resources }} + if: ${{ inputs.canary-run == 'true' }} + + myssql-client-image: + uses: ./.github/workflows/prod-image-build.yml + with: + build-type: "MySQL Client" + image-tag: mysql-${{ inputs.image-tag }} + install-mysql-client-type: "mysql" + python-versions: ${{ inputs.python-versions }} + branch: ${{ inputs.branch }} + # Always build images during the extra checks + in-workflow-build: 'true' + build-provider-packages: ${{ inputs.build-provider-packages }} + upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} + breeze-python-version: ${{ inputs.breeze-python-version }} + chicken-egg-providers: ${{ inputs.chicken-egg-providers }} + constraints-branch: ${{ inputs.constraints-branch }} + docker-cache: ${{ inputs.docker-cache }} + debug-resources: ${{ inputs.debug-resources }} + if: ${{ inputs.canary-run == 'true' }} diff --git a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py index 15c0650e18..02e8f0a20f 100755 --- a/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py +++ b/scripts/ci/pre_commit/pre_commit_checkout_no_credentials.py @@ -39,6 +39,8 @@ def check_file(the_file: Path) -> int: res = yaml.safe_load(the_file.read_text()) console.print(f"Checking file [yellow]{the_file}[/]") for job in res["jobs"].values(): + if job.get("steps") is None: + continue for step in job["steps"]: uses = step.get("uses") pretty_step = yaml.safe_dump(step, indent=2)
