This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a03324382b Improve finalization of tests (#38499)
a03324382b is described below
commit a03324382b2f891fbcecf5ff637213a08e9b877b
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Mar 27 01:57:07 2024 +0100
Improve finalization of tests (#38499)
Last step of the test preparation is pushing the latest images for
AMD to github registry. This should happen using the cache of ours
so we should use the `airflow_cache` builders to make it really fast.
For PROD image we also want to run them in parallel, because this
way we will reuse provider packages and airflow packages
built earlier in the same job.
Also while we are touching builders, we remove last place where
builders were passed by env variables.
Co-authored-by: Idan Tepper <65827206+idantep...@users.noreply.github.com>
---
.github/workflows/ci-image-build.yml | 6 ++----
.github/workflows/finalize-tests.yml | 9 +++++----
.github/workflows/prod-image-build.yml | 5 ++---
.github/workflows/push-image-cache.yml | 12 ++++++------
4 files changed, 15 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 177cd39fbb..e384d7cf83 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -192,15 +192,13 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || ''
}}"
Build ${{ inputs.push-image == 'true' && ' & push ' || '' }}
${{ inputs.platform }}:${{ matrix.python-version }}:${{
inputs.image-tag }}
run: >
- breeze ci-image build --tag-as-latest --image-tag "${{
inputs.image-tag }}"
- --python "${{ matrix.python-version }}"
- --platform "${{ inputs.platform }}"
+ breeze ci-image build --builder airflow_cache --tag-as-latest
--image-tag "${{ inputs.image-tag }}"
+ --python "${{ matrix.python-version }}" --platform "${{
inputs.platform }}"
env:
DOCKER_CACHE: ${{ inputs.docker-cache }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- BUILDER: "airflow_cache"
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
if: inputs.do-build == 'true'
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index a10b1826d1..52c451545b 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -121,7 +121,7 @@ jobs:
git push
push-buildx-cache-to-github-registry-amd:
- name: Push Regular Image Cache
+ name: Push Regular AMD Image Cache
needs: [update-constraints]
uses: ./.github/workflows/push-image-cache.yml
permissions:
@@ -129,7 +129,8 @@ jobs:
packages: write
secrets: inherit
with:
- cache-type: "Regular"
+ runs-on: '["ubuntu-22.04"]'
+ cache-type: "Regular AMD"
include-prod-images: "true"
push-latest-images: "true"
platform: "linux/amd64"
@@ -142,7 +143,7 @@ jobs:
if: inputs.canary-run == 'true'
push-buildx-cache-to-github-registry-arm:
- name: Push Regular Image Cache
+ name: Push Regular ARM Image Cache
needs: [update-constraints]
uses: ./.github/workflows/push-image-cache.yml
permissions:
@@ -151,7 +152,7 @@ jobs:
secrets: inherit
with:
runs-on: ${{ inputs.runs-on }}
- cache-type: "Regular"
+ cache-type: "Regular ARM"
include-prod-images: "true"
push-latest-images: "true"
platform: "linux/arm64"
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 27cd62d42c..9556633c40 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -252,7 +252,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
PUSH: ${{ inputs.push-image }}
DOCKER_CACHE: ${{ inputs.docker-cache }}
DEBIAN_VERSION: ${{ inputs.debian-version }}
- BUILDER: "airflow_cache"
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
@@ -260,8 +259,8 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
- name: "Build PROD images with PyPi providers ${{ matrix.python-version
}}:${{ inputs.image-tag }}"
shell: bash
run: >
- breeze prod-image build --tag-as-latest --image-tag "${{
inputs.image-tag }}"
- --commit-sha "${{ github.sha }}"
+ breeze prod-image build --builder airflow_cache --tag-as-latest
+ --image-tag "${{ inputs.image-tag }}" --commit-sha "${{ github.sha
}}"
--install-packages-from-context --airflow-constraints-mode
constraints
--use-constraints-for-context-packages --python "${{
matrix.python-version }}"
env:
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index c89ef49302..186b2f0987 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -74,8 +74,8 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
jobs:
- push-image-cache:
- name: "Push CI ${{ inputs.cache-type }} image cache"
+ push-ci-image-cache:
+ name: "Push CI ${{ inputs.cache-type }}:${{ matrix.python }} image cache "
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
@@ -120,12 +120,12 @@ jobs:
if: always() && inputs.platform == 'linux/arm64'
- name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)"
run: >
- breeze ci-image build --push
+ breeze ci-image build --builder airflow_cache --push
--python "${{ matrix.python }}" --platform "${{ inputs.platform }}"
if: inputs.push-latest-images == 'true' && inputs.platform ==
'linux/amd64'
push-prod-image-cache:
- name: "Push PROD ${{ inputs.cache-type }} image cache"
+ name: "Push PROD ${{ inputs.cache-type }}:${{ matrix.python }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
@@ -186,6 +186,6 @@ jobs:
# and we can do it after we stopped the ARM instance as it is not
needed anymore
- name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)"
run: >
- breeze prod-image build --install-packages-from-context
- --push --python ${{ matrix.python}} --platform "${{ inputs.platform
}}"
+ breeze prod-image build --builder airflow_cache
--install-packages-from-context
+ --push --platform "${{ inputs.platform }}"
if: inputs.push-latest-images == 'true' && inputs.platform ==
'linux/amd64'
