This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch extract-early-image-checks-workflow
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cf411afc5016150fc71573354b547de562aa4a26
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 16 19:37:13 2024 +0100
Extract early image checks as separate workflow
Another (likely last) extraction of workflows from the main
ci.yml workflow.
It also moves the ARM CI build into the same early image workflow
as it could be run easly in the process.
---
.github/workflows/all-prod-image-tests.yml | 2 +-
.github/workflows/ci.yml | 161 +++-----------------
.github/workflows/early-image-checks.yml | 235 +++++++++++++++++++++++++++++
.github/workflows/finalize-tests.yml | 27 +---
4 files changed, 254 insertions(+), 171 deletions(-)
diff --git a/.github/workflows/all-prod-image-tests.yml
b/.github/workflows/all-prod-image-tests.yml
index a2f0036442..b47acd25b4 100644
--- a/.github/workflows/all-prod-image-tests.yml
+++ b/.github/workflows/all-prod-image-tests.yml
@@ -46,7 +46,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
docker-cache:
- description: "Whether to use Docker cache (true/false)"
+ description: "Docker cache specification to build the image (registry,
local, disabled)."
required: true
type: string
canary-run:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c92087d43d..0d0a4b2c4f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -172,70 +172,6 @@ jobs:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
- # Push early BuildX cache to GitHub Registry in Apache repository, This
cache does not wait for all the
- # tests to complete - it is run very early in the build process for "main"
merges in order to refresh
- # cache using the current constraints. This will speed up cache refresh in
cases when pyproject.toml
- # changes or in case of Dockerfile changes. Failure in this step is not a
problem (at most it will
- # delay cache refresh. It does not attempt to upgrade to newer dependencies.
- # We only push CI cache as PROD cache usually does not gain as much from
fresh cache because
- # it uses prepared airflow and provider packages that invalidate the cache
anyway most of the time
- push-early-buildx-cache-to-github-registry:
- name: Push Early Image Cache
- needs: [build-info, build-ci-images]
- uses: ./.github/workflows/push-image-cache.yml
- permissions:
- contents: read
- # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
- # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
- # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
- # "in-workflow-build" condition
- packages: write
- secrets: inherit
- with:
- cache-type: "Early"
- include-prod-images: "false"
- push-latest-images: "false"
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- branch: ${{ needs.build-info.outputs.default-branch }}
- use-uv: "true"
- breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- if: needs.build-info.outputs.canary-run == 'true' &&
needs.build-info.outputs.default-branch == 'main'
-
- # Check that after earlier cache push, breeze command will build quickly
- check-that-image-builds-quickly:
- timeout-minutes: 5
- name: "Check that image builds quickly"
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
- needs:
- - build-info
- - push-early-buildx-cache-to-github-registry
- env:
- RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- UPGRADE_TO_NEWER_DEPENDENCIES: false
- PLATFORM: "linux/amd64"
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- PYTHON_VERSION: ${{needs.build-info.outputs.default-python-version}}
- if: >
- needs.build-info.outputs.canary-run == 'true'
- && needs.build-info.outputs.default-branch == 'main'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- - name: "Check that image builds quickly"
- run: breeze shell --max-time 120
-
basic-tests:
name: "Basic tests"
needs: [build-info]
@@ -315,87 +251,24 @@ jobs:
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
if: needs.build-info.outputs.in-workflow-build == 'false'
- generate-constraints:
- permissions:
- contents: read
- timeout-minutes: 70
- name: >
- Generate constraints with CI image verifying
- ${{needs.build-info.outputs.all-python-versions-list-as-string}}
- runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+ early-image-checks:
+ name: "Early image checks"
needs: [build-info, wait-for-ci-images]
- env:
- RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
- PYTHON_VERSIONS:
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- VERSION_SUFFIX_FOR_PYPI: "dev0"
- INCLUDE_SUCCESS_OUTPUTS: "true"
- if: needs.build-info.outputs.ci-image-build == 'true'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- with:
- python-version: ${{ needs.build-info.outputs.breeze-python-version }}
- - name: Login to ghcr.io
- run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{
needs.build-info.outputs.image-tag }}
- run: breeze ci-image pull --run-in-parallel --tag-as-latest
- - name: Verify CI images ${{ env.PYTHON_VERSIONS }}:${{
needs.build-info.outputs.image-tag }}
- run: breeze ci-image verify --run-in-parallel
- env:
- PYTHON_VERSIONS:
${{needs.build-info.outputs.all-python-versions-list-as-string}}
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- - name: "Source constraints"
- shell: bash
- run: >
- breeze release-management generate-constraints --run-in-parallel
- --airflow-constraints-mode constraints-source-providers
- - name: "No providers constraints"
- shell: bash
- timeout-minutes: 25
- run: >
- breeze release-management generate-constraints --run-in-parallel
- --airflow-constraints-mode constraints-no-providers
- - name: "Prepare chicken-eggs provider packages"
- # In case of provider packages which use latest dev0 version of
providers, we should prepare them
- # from the source code, not from the PyPI because they have
apache-airflow>=X.Y.Z dependency
- # And when we prepare them from sources they will have
apache-airflow>=X.Y.Z.dev0
- shell: bash
- run: >
- breeze release-management prepare-provider-packages
--include-not-ready-providers
- --package-format wheel --version-suffix-for-pypi dev0
- ${{ needs.build-info.outputs.chicken-egg-providers }}
- if: needs.build-info.outputs.chicken-egg-providers != ''
- - name: "PyPI constraints"
- shell: bash
- timeout-minutes: 25
- run: >
- breeze release-management generate-constraints --run-in-parallel
- --airflow-constraints-mode constraints
- --chicken-egg-providers "${{
needs.build-info.outputs.chicken-egg-providers }}"
- - name: "Dependency upgrade summary"
- shell: bash
- run: |
- for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
- echo "Summarizing Python $PYTHON_VERSION"
- cat "files/constraints-${PYTHON_VERSION}"/*.md >>
$GITHUB_STEP_SUMMARY || true
- done
- - name: "Upload constraint artifacts"
- uses: actions/upload-artifact@v4
- with:
- name: constraints
- path: ./files/constraints-*/constraints-*.txt
- retention-days: 7
- if-no-files-found: error
+ uses: ./.github/workflows/early-image-checks.yml
+ with:
+ runs-on: ${{needs.build-info.outputs.runs-on}}
+ image-tag: ${{ needs.build-info.outputs.image-tag }}
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
+ default-python-version: ${{
needs.build-info.outputs.default-python-version }}
+ breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
+ chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
+ ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
+ docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
static-checks:
timeout-minutes: 45
diff --git a/.github/workflows/early-image-checks.yml
b/.github/workflows/early-image-checks.yml
new file mode 100644
index 0000000000..db70ed90cb
--- /dev/null
+++ b/.github/workflows/early-image-checks.yml
@@ -0,0 +1,235 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Early image checks
+on: # yamllint disable-line rule:truthy
+ workflow_call:
+ inputs:
+ runs-on:
+ description: "The array of labels (in json form) determining type of
the runner to use for the build."
+ required: false
+ default: '["ubuntu-22.04"]'
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
+ required: true
+ type: string
+ python-versions:
+ description: "The list of python versions (stringified JSON array) to
run the tests on."
+ required: true
+ type: string
+ python-versions-list-as-string:
+ description: "The list of python versions as string separated by
spaces"
+ required: true
+ type: string
+ branch:
+ description: "Branch used to run the CI jobs in (main/v2_*_test)."
+ required: true
+ type: string
+ constraints-branch:
+ description: "Branch used to get constraints from"
+ required: true
+ type: string
+ default-python-version:
+ description: Which version of python should be used get CI image
+ required: true
+ type: string
+ breeze-python-version:
+ description: >
+ Which version of python should be used to install Breeze (3.9 is
minimum for reproducible builds)
+ required: true
+ type: string
+ chicken-egg-providers:
+ description: "List of providers that should be prepared from sources"
+ required: false
+ type: string
+ ci-image-build:
+ description: "Whether to build CI images (true/false)"
+ required: true
+ type: string
+ docker-cache:
+ description: "Docker cache specification to build the image (registry,
local, disabled)."
+ required: true
+ type: string
+ canary-run:
+ description: "Whether this is a canary run (true/false)"
+ required: true
+ type: string
+ debug-resources:
+ description: "Whether to debug resources (true/false)"
+ required: true
+ type: string
+jobs:
+ # Push early BuildX cache to GitHub Registry in Apache repository, This
cache does not wait for all the
+ # tests to complete - it is run very early in the build process for "main"
merges in order to refresh
+ # cache using the current constraints. This will speed up cache refresh in
cases when pyproject.toml
+ # changes or in case of Dockerfile changes. Failure in this step is not a
problem (at most it will
+ # delay cache refresh. It does not attempt to upgrade to newer dependencies.
+ # We only push CI cache as PROD cache usually does not gain as much from
fresh cache because
+ # it uses prepared airflow and provider packages that invalidate the cache
anyway most of the time
+ push-early-buildx-cache-to-github-registry:
+ name: Push Early Image Cache
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
+ # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
+ # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
+ # "in-workflow-build" condition
+ packages: write
+ secrets: inherit
+ with:
+ cache-type: "Early"
+ include-prod-images: "false"
+ push-latest-images: "false"
+ image-tag: ${{ inputs.image-tag }}
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ use-uv: "true"
+ breeze-python-version: ${{ inputs.breeze-python-version }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true' && inputs.branch == 'main'
+
+ # Check that after earlier cache push, breeze command will build quickly
+ check-that-image-builds-quickly:
+ timeout-minutes: 5
+ name: Check that image builds quickly
+ runs-on: ${{ fromJSON(inputs.runs-on) }}
+ env:
+ UPGRADE_TO_NEWER_DEPENDENCIES: false
+ PLATFORM: "linux/amd64"
+ PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
+ PYTHON_VERSION: ${{ inputs.default-python-version }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
+ if: inputs.canary-run == 'true' && inputs.branch == 'main'
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ - name: "Check that image builds quickly"
+ run: breeze shell --max-time 120
+
+ generate-constraints:
+ permissions:
+ contents: read
+ timeout-minutes: 70
+ name: Generate constraints ${{ inputs.python-versions-list-as-string }}
+ runs-on: ${{ fromJSON(inputs.runs-on) }}
+ env:
+ PYTHON_VERSIONS: ${{ inputs.python-versions-list-as-string }}
+ DEBUG_RESOURCES: ${{ inputs.debug-resources }}
+ VERSION_SUFFIX_FOR_PYPI: "dev0"
+ INCLUDE_SUCCESS_OUTPUTS: "true"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ if: inputs.ci-image-build == 'true'
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Cleanup docker
+ uses: ./.github/actions/cleanup-docker
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ with:
+ python-version: ${{ inputs.breeze-python-version }}
+ - name: Login to ghcr.io
+ run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
+ - name: Pull CI images ${{ inputs.python-versions-list-as-string }}:${{
inputs.image-tag }}
+ run: breeze ci-image pull --run-in-parallel --tag-as-latest
+ - name: Verify CI images ${{ inputs.python-versions-list-as-string
}}:${{ inputs.image-tag }}
+ run: breeze ci-image verify --run-in-parallel
+ - name: "Source constraints"
+ shell: bash
+ run: >
+ breeze release-management generate-constraints --run-in-parallel
+ --airflow-constraints-mode constraints-source-providers
+ - name: "No providers constraints"
+ shell: bash
+ timeout-minutes: 25
+ run: >
+ breeze release-management generate-constraints --run-in-parallel
+ --airflow-constraints-mode constraints-no-providers
+ - name: "Prepare chicken-eggs provider packages"
+ # In case of provider packages which use latest dev0 version of
providers, we should prepare them
+ # from the source code, not from the PyPI because they have
apache-airflow>=X.Y.Z dependency
+ # And when we prepare them from sources they will have
apache-airflow>=X.Y.Z.dev0
+ shell: bash
+ run: >
+ breeze release-management prepare-provider-packages
--include-not-ready-providers
+ --package-format wheel --version-suffix-for-pypi dev0
+ ${{ inputs.chicken-egg-providers }}
+ if: inputs.chicken-egg-providers != ''
+ - name: "PyPI constraints"
+ shell: bash
+ timeout-minutes: 25
+ run: >
+ breeze release-management generate-constraints --run-in-parallel
+ --airflow-constraints-mode constraints
+ --chicken-egg-providers "${{ inputs.chicken-egg-providers }}"
+ - name: "Dependency upgrade summary"
+ shell: bash
+ run: |
+ for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
+ echo "Summarizing Python $PYTHON_VERSION"
+ cat "files/constraints-${PYTHON_VERSION}"/*.md >>
$GITHUB_STEP_SUMMARY || true
+ done
+ - name: "Upload constraint artifacts"
+ uses: actions/upload-artifact@v4
+ with:
+ name: constraints
+ path: ./files/constraints-*/constraints-*.txt
+ retention-days: 7
+ if-no-files-found: error
+
+ # This is only a check if ARM images are successfully building when
committer runs PR from
+ # Apache repository. This is needed in case you want to fix failing cache
job in "canary" run
+ # There is no point in running this one in "canary" run, because the above
step is doing the
+ # same build anyway.
+ build-ci-arm-images:
+ name: Build CI ARM images (in-workflow)
+ uses: ./.github/workflows/ci-image-build.yml
+ permissions:
+ contents: read
+ packages: write
+ secrets: inherit
+ with:
+ platform: "arm64"
+ push-image: "false"
+ runs-on: ${{ inputs.runs-on }}
+ image-tag: ${{ inputs.image-tag }}
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ use-uv: "true"
+ upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
+ breeze-python-version: ${{ inputs.breeze-python-version }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true'
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index f76398e35f..0f5ed906d3 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -63,7 +63,7 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
docker-cache:
- description: "The type of docker cache to use (none/regular/early)"
+ description: "Docker cache specification to build the image (registry,
local, disabled)."
required: true
type: string
canary-run:
@@ -147,31 +147,6 @@ jobs:
docker-cache: ${{ inputs.docker-cache }}
if: inputs.canary-run == 'true'
- # This is only a check if ARM images are successfully building when
committer runs PR from
- # Apache repository. This is needed in case you want to fix failing cache
job in "canary" run
- # There is no point in running this one in "canary" run, because the above
step is doing the
- # same build anyway.
- build-ci-arm-images:
- name: Build CI ARM images (in-workflow)
- uses: ./.github/workflows/ci-image-build.yml
- permissions:
- contents: read
- packages: write
- secrets: inherit
- with:
- platform: "arm64"
- push-image: "false"
- runs-on: ${{ inputs.runs-on }}
- image-tag: ${{ inputs.image-tag }}
- python-versions: ${{ inputs.python-versions }}
- branch: ${{ inputs.branch }}
- use-uv: "true"
- upgrade-to-newer-dependencies: ${{ inputs.upgrade-to-newer-dependencies
}}
- breeze-python-version: ${{ inputs.breeze-python-version }}
- constraints-branch: ${{ inputs.constraints-branch }}
- docker-cache: ${{ inputs.docker-cache }}
- if: inputs.in-workflow-build == 'true' && inputs.canary-run != 'true'
-
summarize-warnings:
timeout-minutes: 15
name: "Summarize warnings"
