This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch extract-finalization-of-tests-to-separate-workflow in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 43f59c4052979b66654b8c5011eb9009449cd02c Author: Jarek Potiuk <[email protected]> AuthorDate: Sat Mar 16 14:28:18 2024 +0100 Extract finalization of tests to separate workflow Finalization of the CI workflow is happening when all the regular tests succeeded and consists of updating constrainst, refreshing cache and building CI ARM images. All those jobs can be run in a separate workflow - again nicely folded together. --- .github/workflows/ci.yml | 118 ++-------------------- .github/workflows/finalize-tests.yml | 184 +++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 109 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 331782f1db..793d05ad50 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1118,13 +1118,11 @@ jobs: ( needs.build-info.outputs.run-kubernetes-tests == 'true' || needs.build-info.outputs.needs-helm-tests == 'true') - update-constraints: + finalize-tests: permissions: contents: write packages: write - timeout-minutes: 80 - name: "Update constraints" - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} + secrets: inherit needs: - build-info - build-docs @@ -1141,115 +1139,17 @@ jobs: - tests-integration-postgres - tests-integration-mysql - generate-constraints - env: - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} - if: needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - # Needed to perform push action - persist-credentials: false - - name: Cleanup docker - uses: ./.github/actions/cleanup-docker - - name: "Set constraints branch name" - id: constraints-branch - run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} - - name: Checkout ${{ steps.constraints-branch.outputs.branch }} - uses: actions/checkout@v4 - with: - path: "constraints" - ref: ${{ steps.constraints-branch.outputs.branch }} - persist-credentials: true - fetch-depth: 0 - - name: "Download constraints from the constraints generated by build CI image" - uses: actions/download-artifact@v4 - with: - name: constraints - path: ./files - - name: "Diff in constraints for ${{needs.build-info.outputs.python-versions}}" - run: ./scripts/ci/constraints/ci_diff_constraints.sh - # only commit and push constraints in canary runs (main) - - name: "Commit changed constraint files for ${{needs.build-info.outputs.python-versions}}" - run: ./scripts/ci/constraints/ci_commit_constraints.sh - if: needs.build-info.outputs.canary-run == 'true' - - name: "Push changes" - if: needs.build-info.outputs.canary-run == 'true' - working-directory: "constraints" - run: - git push - # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build - # is executed as result of direct push to "main" or one of the "vX-Y-test" branches - # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry - # It will automatically check if a new python image was released and will pull the latest one if needed - push-buildx-cache-to-github-registry: - name: Push Regular Image Cache - needs: [build-info, update-constraints] - uses: ./.github/workflows/push-image-cache.yml - permissions: - contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the - # "in-workflow-build" condition - packages: write - secrets: inherit + uses: ./.github/workflows/finalize-tests.yml with: - cache-type: "Regular" - include-prod-images: "true" - push-latest-images: "true" - use-uv: "true" + runs-on: ${{ needs.build-info.outputs.runs-on }} image-tag: ${{ needs.build-info.outputs.image-tag }} python-versions: ${{ needs.build-info.outputs.python-versions }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} branch: ${{ needs.build-info.outputs.default-branch }} - breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} - docker-cache: ${{ needs.build-info.outputs.cache-directive }} - if: needs.build-info.outputs.canary-run == 'true' - - # This is only a check if ARM images are successfully building when committer runs PR from - # Apache repository. This is needed in case you want to fix failing cache job in "canary" run - # There is no point in running this one in "canary" run, because the above step is doing the - # same build anyway. - build-ci-arm-images: - name: Build CI ARM images (in-workflow) - needs: - - build-info - - build-docs - - spellcheck-docs - - static-checks - - mypy - - tests-sqlite - - tests-mysql - - tests-postgres - - tests-non-db - - tests-integration-postgres - - tests-integration-mysql - uses: ./.github/workflows/ci-image-build.yml - permissions: - contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the - # "in-workflow-build" condition - packages: write - secrets: inherit - with: - platform: "arm64" - runs-on: ${{needs.build-info.outputs.runs-on}} - image-tag: ${{ needs.build-info.outputs.image-tag }} - python-versions: ${{ needs.build-info.outputs.python-versions }} - branch: ${{ needs.build-info.outputs.default-branch }} - use-uv: "true" - upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + default-python-version: ${{ needs.build-info.outputs.default-python-version }} breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} - constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} docker-cache: ${{ needs.build-info.outputs.cache-directive }} - if: > - needs.build-info.outputs.in-workflow-build == 'true' && - needs.build-info.outputs.canary-run != 'true' + canary-run: ${{ needs.build-info.outputs.canary-run }} diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml new file mode 100644 index 0000000000..64ebfeb708 --- /dev/null +++ b/.github/workflows/finalize-tests.yml @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Finalize tests +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runs-on: + description: "The array of labels (in json form) determining type of the runner to use for the build." + required: false + default: '["ubuntu-22.04"]' + type: string + image-tag: + description: "Tag to set for the image" + required: true + type: string + python-versions: + description: "JSON-formatted array of Python versions to test" + required: true + type: string + python-versions-list-as-string: + description: "Stringified array of all Python versions to test - separated by spaces." + required: true + type: string + branch: + description: "The default branch to use for the build" + required: true + type: string + constraints-branch: + description: "The branch to use for constraints" + required: true + type: string + default-python-version: + description: Which version of python should be used get CI image + required: true + type: string + breeze-python-version: + description: > + Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds) + required: true + type: string + in-workflow-build: + description: "Whether the build is executed as part of the workflow (true/false)" + required: true + type: string + upgrade-to-newer-dependencies: + description: "Whether to upgrade to newer dependencies (true/false)" + required: true + type: string + docker-cache: + description: "The type of docker cache to use (none/regular/early)" + required: true + type: string + canary-run: + description: "Whether this is a canary run (true/false)" + required: true + type: string +jobs: + update-constraints: + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 80 + name: "Update constraints" + permissions: + contents: write + packages: read + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + DEBUG_RESOURCES: ${{ inputs.debug-resources}} + PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }} + IMAGE_TAG: ${{ inputs.image-tag }} + if: inputs.upgrade-to-newer-dependencies != 'false' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + # Needed to perform push action + persist-credentials: false + - name: Cleanup docker + uses: ./.github/actions/cleanup-docker + - name: "Set constraints branch name" + id: constraints-branch + run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT} + - name: Checkout ${{ steps.constraints-branch.outputs.branch }} + uses: actions/checkout@v4 + with: + path: "constraints" + ref: ${{ steps.constraints-branch.outputs.branch }} + persist-credentials: true + fetch-depth: 0 + - name: "Download constraints from the constraints generated by build CI image" + uses: actions/download-artifact@v4 + with: + name: constraints + path: ./files + - name: "Diff in constraints for Python: ${{ inputs.python-versions-list-as-string }}" + run: ./scripts/ci/constraints/ci_diff_constraints.sh + # only commit and push constraints in canary runs (main) + - name: "Commit changed constraint files for Python: ${{ inputs.python-versions-list-as-string }}" + run: ./scripts/ci/constraints/ci_commit_constraints.sh + if: inputs.canary-run == 'true' + - name: "Push changes" + if: inputs.canary-run == 'true' + working-directory: "constraints" + run: + git push + + # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build + # is executed as result of direct push to "main" or one of the "vX-Y-test" branches + # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry + # It will automatically check if a new python image was released and will pull the latest one if needed + push-buildx-cache-to-github-registry: + name: Push Regular Image Cache + needs: [update-constraints] + uses: ./.github/workflows/push-image-cache.yml + permissions: + contents: read + packages: write + secrets: inherit + with: + cache-type: "Regular" + include-prod-images: "true" + push-latest-images: "true" + use-uv: "true" + image-tag: ${{ inputs.image-tag }} + python-versions: ${{ inputs.python-versions }} + branch: ${{ inputs.branch }} + breeze-python-version: ${{ inputs.breeze-python-version }} + constraints-branch: ${{ inputs.constraints-branch }} + docker-cache: ${{ inputs.docker-cache }} + if: inputs.canary-run == 'true' + + # This is only a check if ARM images are successfully building when committer runs PR from + # Apache repository. This is needed in case you want to fix failing cache job in "canary" run + # There is no point in running this one in "canary" run, because the above step is doing the + # same build anyway. + build-ci-arm-images: + name: Build CI ARM images (in-workflow) + needs: + - build-info + - build-docs + - spellcheck-docs + - static-checks + - mypy + - tests-sqlite + - tests-mysql + - tests-postgres + - tests-non-db + - tests-integration-postgres + - tests-integration-mysql + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + packages: write + secrets: inherit + with: + platform: "arm64" + runs-on: ${{ inputs.runs-on }} + image-tag: ${{ inputs.image-tag }} + python-versions: ${{ inputs.python-versions }} + branch: ${{ inputs.branch }} + use-uv: "true" + upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies }} + breeze-python-version: ${{ inputs.breeze-python-version }} + constraints-branch: ${{ inputs.constraints-branch }} + docker-cache: ${{ inputs.docker-cache }} + if: inputs.in-workflow-build == 'true' && inputs.canary-run != 'true'
