This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch separate-cache-workflow in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 2124318a0b249cc4376bc8be07f0248f05b7a952 Author: Jarek Potiuk <[email protected]> AuthorDate: Tue Mar 12 11:42:10 2024 +0100 Separate cache build to a separate workflow called from the main ci.yml This is another step of decomposing our workflows and introducing more DRY approach for GitHub Actions workflows we have. Builds on changes implemented in #37865 and #38057 --- .github/workflows/ci.yml | 202 ++++++++------------------------- .github/workflows/push-image-cache.yml | 149 ++++++++++++++++++++++++ 2 files changed, 196 insertions(+), 155 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b0cebe6df..6a30c4d066 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,70 +177,30 @@ jobs: # We only push CI cache as PROD cache usually does not gain as much from fresh cache because # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time push-early-buildx-cache-to-github-registry: - timeout-minutes: 110 - name: "Push Early Image Cache" - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: - - build-info - strategy: - fail-fast: false - matrix: - platform: ["linux/amd64", "linux/arm64"] - env: - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - UPGRADE_TO_NEWER_DEPENDENCIES: false - VERSION_SUFFIX_FOR_PYPI: "dev0" - continue-on-error: true - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' - - name: "Install Breeze" - uses: ./.github/actions/breeze - if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' - - name: "Start ARM instance" - run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: > - matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' - && needs.build-info.outputs.default-branch == 'main' - - name: "Push CI cache ${{ matrix.platform }}" - run: | - for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do - breeze ci-image build --builder airflow_cache --prepare-buildx-cache \ - --python ${PYTHON} --platform ${{ matrix.platform }} - done - env: - COMMIT_SHA: ${{ github.sha }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' - - name: "Push CI latest image ${{ matrix.platform }}" - run: | - for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do - breeze ci-image build --tag-as-latest --push --run-in-parallel \ - --python ${PYTHON} --platform ${{ matrix.platform }} - done - env: - COMMIT_SHA: ${{ github.sha }} - # We only push "amd" image as it is really only needed for any kind of automated builds in CI - # and currently there is not an easy way to make multi-platform image from two separate builds - if: > - matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' - && needs.build-info.outputs.default-branch == 'main' - - name: "Stop ARM instance" - run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: > - always() && matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' - && needs.build-info.outputs.default-branch == 'main' - - name: "Clean docker cache for ${{ matrix.platform }}" - run: docker system prune --all --force - if: > - matrix.platform == 'linux/amd64' && needs.build-info.outputs.canary-run == 'true' - && needs.build-info.outputs.default-branch == 'main' + name: Push Early Image Cache + needs: [build-info, build-ci-images] + uses: ./.github/workflows/push-image-cache.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the + # "in-workflow-build" condition + packages: write + secrets: inherit + with: + cache-type: "Early" + include-prod-images: "false" + push-latest-images: "false" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + use-uv: "true" + breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} + if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' + # Check that after earlier cache push, breeze command will build quickly check-that-image-builds-quickly: timeout-minutes: 5 @@ -1926,9 +1886,7 @@ jobs: # This strange condition below is equivalent to: "all success or skipped" # The "build-prod-images" step might be skipped, in case production building happens in the # "build-images" workflow, and in this case we still want to wait for PROD images and run dependent tests - if: > - always() && !failure() && !cancelled() && - needs.build-info.outputs.prod-image-build == 'true' + if: always() && !failure() && !cancelled() && needs.build-info.outputs.prod-image-build == 'true' env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" BACKEND: sqlite @@ -2158,100 +2116,34 @@ jobs: working-directory: "constraints" run: git push - # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build # is executed as result of direct push to "main" or one of the "vX-Y-test" branches # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry # It will automatically check if a new python image was released and will pull the latest one if needed push-buildx-cache-to-github-registry: - timeout-minutes: 110 - name: "Push Image Cache" - runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} - needs: - - build-info - - update-constraints + name: Push Regular Image Cache + needs: [build-info, update-constraints] + uses: ./.github/workflows/push-image-cache.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the + # "in-workflow-build" condition + packages: write + secrets: inherit + with: + cache-type: "Regular" + include-prod-images: "true" + push-latest-images: "true" + use-uv: "true" + image-tag: ${{ needs.build-info.outputs.image-tag }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + breeze-python-version: ${{ needs.build-info.outputs.breeze-python-version }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.cache-directive }} if: needs.build-info.outputs.canary-run == 'true' - strategy: - fail-fast: false - matrix: - platform: ["linux/amd64", "linux/arm64"] - env: - RUNS_ON: "${{needs.build-info.outputs.runs-on}}" - VERSION_SUFFIX_FOR_PYPI: "dev0" - steps: - - name: Cleanup repo - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: > - Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} - uses: ./.github/actions/prepare_breeze_and_image - with: - python-version: ${{ needs.build-info.outputs.breeze-python-version }} - env: - # Always use default Python version of CI image for preparing packages - PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Prepare airflow package for PROD build" - run: breeze release-management prepare-airflow-package --package-format wheel - env: - VERSION_SUFFIX_FOR_PYPI: "dev0" - - name: "Prepare providers packages for PROD build" - run: > - breeze release-management prepare-provider-packages - --package-list-file ./prod_image_installed_providers.txt - --package-format wheel - env: - VERSION_SUFFIX_FOR_PYPI: "dev0" - if: needs.build-info.outputs.default-branch == 'main' - - name: "Start ARM instance" - run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: matrix.platform == 'linux/arm64' - - name: "Push CI cache ${{ matrix.platform }}" - run: | - for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do - breeze ci-image build --builder airflow_cache --prepare-buildx-cache \ - --python ${PYTHON} --platform ${{ matrix.platform }} - done - env: - COMMIT_SHA: ${{ github.sha }} - - name: "Push CI latest image ${{ matrix.platform }}" - run: | - for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do - breeze ci-image build --tag-as-latest --push \ - --python ${PYTHON} --platform ${{ matrix.platform }} - done - env: - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - COMMIT_SHA: ${{ github.sha }} - if: matrix.platform == 'linux/amd64' - - name: "Copy dist packages to docker-context files" - run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" - run: | - for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do - breeze prod-image build --builder airflow_cache --install-packages-from-context \ - --airflow-constraints-mode constraints-source-providers --prepare-buildx-cache \ - --platform ${{ matrix.platform }} --python ${PYTHON} - done - env: - COMMIT_SHA: ${{ github.sha }} - - name: "Push PROD latest image ${{ matrix.platform }}" - run: | - for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do - breeze prod-image build --tag-as-latest --install-packages-from-context \ - --push --run-in-parallel --python ${PYTHON} --platform ${{ matrix.platform }} - done - env: - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} - COMMIT_SHA: ${{ github.sha }} - if: matrix.platform == 'linux/amd64' - - name: "Stop ARM instance" - run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() && matrix.platform == 'linux/arm64' # This is only a check if ARM images are successfully building when committer runs PR from # Apache repository. This is needed in case you want to fix failing cache job in "canary" run diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml new file mode 100644 index 0000000000..979d7c1271 --- /dev/null +++ b/.github/workflows/push-image-cache.yml @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Push image cache +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + cache-type: + description: "Type of cache to push (Early / Regular)." + required: true + type: string + include-prod-images: + description: "Whether to include prod images in the cache (true/false)." + required: true + type: string + push-latest-images: + description: "Whether to also push latest images (true/false)." + required: true + type: string + debian-version: + description: "Base Debian distribution to use for the build (bookworm/bullseye)" + type: string + default: "bookworm" + install-mysql-client-type: + description: "MySQL client type to use during build (mariadb/mysql)" + type: string + default: "mariadb" + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string + image-tag: + description: "Tag to set for the image" + required: true + type: string + python-versions: + description: "JSON-formatted array of Python versions to build images from" + required: true + type: string + branch: + description: "Branch used to run the CI jobs in (main/v2_*_test)." + required: true + type: string + breeze-python-version: + description: < + Which version of python should be used to install Breeze (3.9 is minimum for reproducible builds) + required: true + type: string + constraints-branch: + description: "Branch used to construct constraints URL from." + required: true + type: string + docker-cache: + description: "Docker cache specification to build the image (registry, local, disabled)." + required: true + type: string +jobs: + push-ci-image-cache: + runs-on: ${{fromJSON(inputs.runs-on)}} + strategy: + fail-fast: false + matrix: + platform: ["amd64", "arm64"] + python: ${{fromJson(inputs.python-versions)}} + env: + RUNS_ON: "\ + ${{ matrix.platform == 'amd64' && '[\"ubuntu-22.04\"]' || '[\"self-hosted\", \"Linux\", \"X64\"]' }}" + DEBIAN_VERSION: ${{ inputs.debian-version }} + INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} + USE_UV: ${{ inputs.use-uv }} + IMAGE_TAG: ${{ inputs.image-tag }} + DEFAULT_BRANCH: ${{ inputs.branch }} + DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} + UPGRADE_TO_NEWER_DEPENDENCIES: "false" + DOCKER_CACHE: ${{ inputs.docker-cache }} + VERSION_SUFFIX_FOR_PYPI: "dev0" + COMMIT_SHA: ${{ github.sha }} + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + python-version: ${{ inputs.breeze-python-version }} + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + if: inputs.include-prod-images == 'true' + - name: "Prepare airflow package for PROD build" + run: breeze release-management prepare-airflow-package --package-format wheel + if: inputs.include-prod-images == 'true' + - name: "Prepare providers packages for PROD build" + run: > + breeze release-management prepare-provider-packages + --package-list-file ./prod_image_installed_providers.txt + --package-format wheel + if: inputs.include-prod-images == 'true' + - name: "Start ARM instance" + run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh + if: matrix.platform == 'arm64' + - name: "Push CI ${{ inputs.cache-type }} cache ${{ matrix.platform }}: ${{ matrix.python }}" + run: > + breeze ci-image build --builder airflow_cache --prepare-buildx-cache + --python ${{ matrix.python }} --platform "linux/${{ matrix.platform }}" + env: + - name: "Push CI latest image ${{ matrix.platform }}:${{ matrix.python }}" + run: > + breeze ci-image build --tag-as-latest --push --run-in-parallel --python "${{matrix.python}}" + --platform "linux/${{ matrix.platform }}" + # We only push "amd" image as it is really only needed for any kind of automated builds in CI + # and currently there is not an easy way to make multi-platform image from two separate builds + if: matrix.platform == 'amd64' && inputs.push-latest-images == 'true' + - name: "Copy dist packages to docker-context files" + run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files + if: inputs.include-prod-images == 'true' + - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" + run: > + breeze prod-image build --builder airflow_cache --install-packages-from-context + --airflow-constraints-mode constraints-source-providers --prepare-buildx-cache + --platform "linux/${{ matrix.platform }}" --python ${{ matrix.python }} + - name: "Push PROD latest image ${{ matrix.platform }}" + run: > + breeze prod-image build --tag-as-latest --install-packages-from-context + --push --run-in-parallel --python ${{ matrix.python}} --platform "linux/${{ matrix.platform }}" + if: matrix.platform == 'linux/amd64' && inputs.push-latest-images == 'true' + - name: "Stop ARM instance" + run: ./scripts/ci/images/ci_stop_arm_instance.sh + if: always() && matrix.platform == 'linux/arm64' + - name: "Clean docker cache for ${{ matrix.platform }}" + run: docker system prune --all --force + if: matrix.platform == 'amd64'
