This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 911048b212 Fix image cache optimizations - speeding up the build
(#38442)
911048b212 is described below
commit 911048b212ae5dbd46742bec6f6f9017570d9e9d
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Mar 25 18:16:06 2024 +0100
Fix image cache optimizations - speeding up the build (#38442)
The recent refactors in workflows broke the way how cache had been used
in the CI builds. This PR brings back the optimizations by using the
cache and rebuilding it.
---
.github/workflows/additional-ci-image-checks.yml | 56 +++++++++---------
.github/workflows/build-images.yml | 3 +-
.github/workflows/ci-image-build.yml | 17 +++---
.github/workflows/ci.yml | 2 +
.github/workflows/finalize-tests.yml | 67 +++++++++++++--------
.github/workflows/prod-image-build.yml | 9 ++-
.github/workflows/prod-image-extra-checks.yml | 3 +
.github/workflows/push-image-cache.yml | 68 ++++++++++------------
.../airflow_breeze/utils/docker_command_utils.py | 6 +-
dev/breeze/src/airflow_breeze/utils/image.py | 14 +----
10 files changed, 128 insertions(+), 117 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 1bee163b0f..6de5862aef 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -89,42 +89,40 @@ jobs:
# delay cache refresh. It does not attempt to upgrade to newer dependencies.
# We only push CI cache as PROD cache usually does not gain as much from
fresh cache because
# it uses prepared airflow and provider packages that invalidate the cache
anyway most of the time
- # push-early-buildx-cache-to-github-registry:
- # name: Push Early Image Cache
- # uses: ./.github/workflows/push-image-cache.yml
- # permissions:
- # contents: read
- # # This write is only given here for `push` events from
"apache/airflow" repo. It is not given for PRs
- # # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
- # # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
- # # "in-workflow-build" condition
- # packages: write
- # secrets: inherit
- # with:
- # runs-on: ${{ inputs.runs-on }}
- # cache-type: "Early"
- # include-prod-images: "false"
- # push-latest-images: "false"
- # image-tag: ${{ inputs.image-tag }}
- # python-versions: ${{ inputs.python-versions }}
- # branch: ${{ inputs.branch }}
- # use-uv: "true"
- # include-success-outputs: ${{ inputs.include-success-outputs }}
- # constraints-branch: ${{ inputs.constraints-branch }}
- # docker-cache: ${{ inputs.docker-cache }}
- # if: inputs.canary-run == 'true' && inputs.branch == 'main'
+ push-early-buildx-cache-to-github-registry:
+ name: Push Early Image Cache
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
+ # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
+ # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
+ # "in-workflow-build" condition
+ packages: write
+ secrets: inherit
+ with:
+ # Runs on Public runners
+ cache-type: "Early"
+ include-prod-images: "false"
+ push-latest-images: "false"
+ platform: "linux/amd64"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ use-uv: "true"
+ include-success-outputs: ${{ inputs.include-success-outputs }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true' && inputs.branch == 'main'
# Check that after earlier cache push, breeze command will build quickly
check-that-image-builds-quickly:
- timeout-minutes: 5
+ timeout-minutes: 11
name: Check that image builds quickly
runs-on: ["ubuntu-22.04"]
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
- PLATFORM: "linux/amd64"
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
PYTHON_VERSION: ${{ inputs.default-python-version }}
- IMAGE_TAG: ${{ inputs.image-tag }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: inputs.canary-run == 'true' && inputs.branch == 'main'
steps:
@@ -142,7 +140,7 @@ jobs:
- name: "Login to ghcr.io"
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- name: "Check that image builds quickly"
- run: breeze shell --max-time 120
+ run: breeze shell --max-time 600 --platform "linux/amd64"
# This is only a check if ARM images are successfully building when
committer runs PR from
# Apache repository. This is needed in case you want to fix failing cache
job in "canary" run
@@ -156,11 +154,11 @@ jobs:
packages: write
secrets: inherit
with:
- platform: "arm64"
push-image: "false"
runs-on: ${{ inputs.runs-on }}
image-tag: ${{ inputs.image-tag }}
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/arm64"
branch: ${{ inputs.branch }}
constraints-branch: ${{ inputs.constraints-branch }}
use-uv: "true"
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index d2d574c806..6897e2c07c 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -180,13 +180,13 @@ jobs:
upload-constraints: "true"
use-uv: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
-
build-prod-images:
name: Build PROD images
permissions:
@@ -209,6 +209,7 @@ jobs:
push-image: "true"
use-uv: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 9eff0ced3a..177cd39fbb 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -48,9 +48,8 @@ on: # yamllint disable-line rule:truthy
default: "false"
type: string
platform:
- description: >
- Name of the platform for the build - 'amd64/arm64'
- default: "amd64"
+ description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+ required: true
type: string
push-image:
description: "Whether to push image to the registry (true/false)"
@@ -107,8 +106,8 @@ jobs:
timeout-minutes: 110
name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-CI ${{inputs.platform}} image\
-${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\
+CI ${{ inputs.platform }} image\
+${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
runs-on: ${{ fromJSON(inputs.runs-on) }}
env:
@@ -185,7 +184,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies
!= 'false'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.do-build == 'true' && inputs.platform == 'arm64'
+ if: inputs.do-build == 'true' && inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
if: inputs.do-build == 'true'
@@ -195,19 +194,19 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || ''
}}"
run: >
breeze ci-image build --tag-as-latest --image-tag "${{
inputs.image-tag }}"
--python "${{ matrix.python-version }}"
- --platform "linux/${{ inputs.platform }}"
+ --platform "${{ inputs.platform }}"
env:
DOCKER_CACHE: ${{ inputs.docker-cache }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- BUILDER: ${{ inputs.platform == 'amd64' && 'default' ||
'airflow_cache' }}
+ BUILDER: "airflow_cache"
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
if: inputs.do-build == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.do-build == 'true' && inputs.platform == 'arm64'
+ if: always() && inputs.do-build == 'true' && inputs.platform ==
'linux/arm64'
- name: "Source constraints: ${{ matrix.python-version }}"
shell: bash
run: >
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 37b6dd3c7d..5a869a160f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -196,6 +196,7 @@ jobs:
with:
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
upload-constraints: "true"
branch: ${{ needs.build-info.outputs.default-branch }}
@@ -496,6 +497,7 @@ jobs:
build-type: "Regular"
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
+ platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
push-image: "true"
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index 48971ff789..a10b1826d1 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -120,31 +120,48 @@ jobs:
run:
git push
- # Push BuildX cache to GitHub Registry in Apache repository, if all tests
are successful and build
- # is executed as result of direct push to "main" or one of the "vX-Y-test"
branches
- # It rebuilds all images using just-pushed constraints using buildx and
pushes them to registry
- # It will automatically check if a new python image was released and will
pull the latest one if needed
- # push-buildx-cache-to-github-registry:
- # name: Push Regular Image Cache
- # needs: [update-constraints]
- # uses: ./.github/workflows/push-image-cache.yml
- # permissions:
- # contents: read
- # packages: write
- # secrets: inherit
- # with:
- # runs-on: ${{ inputs.runs-on }}
- # cache-type: "Regular"
- # include-prod-images: "true"
- # push-latest-images: "true"
- # use-uv: "true"
- # image-tag: ${{ inputs.image-tag }}
- # python-versions: ${{ inputs.python-versions }}
- # branch: ${{ inputs.branch }}
- # constraints-branch: ${{ inputs.constraints-branch }}
- # include-success-outputs: ${{ inputs.include-success-outputs }}
- # docker-cache: ${{ inputs.docker-cache }}
- # if: inputs.canary-run == 'true'
+ push-buildx-cache-to-github-registry-amd:
+ name: Push Regular Image Cache
+ needs: [update-constraints]
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ packages: write
+ secrets: inherit
+ with:
+ cache-type: "Regular"
+ include-prod-images: "true"
+ push-latest-images: "true"
+ platform: "linux/amd64"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ use-uv: "true"
+ include-success-outputs: ${{ inputs.include-success-outputs }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true'
+
+ push-buildx-cache-to-github-registry-arm:
+ name: Push Regular Image Cache
+ needs: [update-constraints]
+ uses: ./.github/workflows/push-image-cache.yml
+ permissions:
+ contents: read
+ packages: write
+ secrets: inherit
+ with:
+ runs-on: ${{ inputs.runs-on }}
+ cache-type: "Regular"
+ include-prod-images: "true"
+ push-latest-images: "true"
+ platform: "linux/arm64"
+ python-versions: ${{ inputs.python-versions }}
+ branch: ${{ inputs.branch }}
+ constraints-branch: ${{ inputs.constraints-branch }}
+ use-uv: "true"
+ include-success-outputs: ${{ inputs.include-success-outputs }}
+ docker-cache: ${{ inputs.docker-cache }}
+ if: inputs.canary-run == 'true'
summarize-warnings:
timeout-minutes: 15
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 6005e44d56..27cd62d42c 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -77,6 +77,10 @@ on: # yamllint disable-line rule:truthy
description: "JSON-formatted array of Python versions to build images
from"
required: true
type: string
+ platform:
+ description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+ required: true
+ type: string
branch:
description: "Branch used to run the CI jobs in (main/v2_*_test)."
required: true
@@ -111,8 +115,8 @@ jobs:
timeout-minutes: 80
name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-PROD ${{inputs.build-type}} image\
-${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\
+PROD ${{ inputs.build-type }} image\
+${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
runs-on: ${{ fromJSON(inputs.runs-on) }}
env:
@@ -248,6 +252,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
PUSH: ${{ inputs.push-image }}
DOCKER_CACHE: ${{ inputs.docker-cache }}
DEBIAN_VERSION: ${{ inputs.debian-version }}
+ BUILDER: "airflow_cache"
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
diff --git a/.github/workflows/prod-image-extra-checks.yml
b/.github/workflows/prod-image-extra-checks.yml
index 69dee4e4a9..faa904e438 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -59,6 +59,7 @@ jobs:
image-tag: bullseye-${{ inputs.image-tag }}
debian-version: "bullseye"
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
@@ -76,6 +77,7 @@ jobs:
image-tag: mysql-${{ inputs.image-tag }}
install-mysql-client-type: "mysql"
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
@@ -93,6 +95,7 @@ jobs:
image-tag: mysql-${{ inputs.image-tag }}
install-mysql-client-type: "mysql"
python-versions: ${{ inputs.python-versions }}
+ platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index dbc8893a8f..c89ef49302 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -23,14 +23,14 @@ on: # yamllint disable-line rule:truthy
runs-on:
description: "The array of labels (in json form) determining type of
the runner to use for the build."
required: false
- default: '["self-hosted", "Linux", "X64"]'
+ default: "[\"ubuntu-22.04\"]"
type: string
cache-type:
description: "Type of cache to push (Early / Regular)."
required: true
type: string
include-prod-images:
- description: "Whether to include prod images in the cache
(true/false)."
+ description: "Whether to build PROD image cache additionally to CI
image cache (true/false)."
required: true
type: string
push-latest-images:
@@ -45,12 +45,8 @@ on: # yamllint disable-line rule:truthy
description: "MySQL client type to use during build (mariadb/mysql)"
type: string
default: "mariadb"
- use-uv:
- description: "Whether to use uv to build the image (true/false)"
- required: true
- type: string
- image-tag:
- description: "Tag to set for the image"
+ platform:
+ description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
required: true
type: string
python-versions:
@@ -65,6 +61,10 @@ on: # yamllint disable-line rule:truthy
description: "Branch used to construct constraints URL from."
required: true
type: string
+ use-uv:
+ description: "Whether to use uv to build the image (true/false)"
+ required: true
+ type: string
include-success-outputs:
description: "Whether to include success outputs (true/false)."
required: true
@@ -74,25 +74,22 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
jobs:
- push-ci-image-cache:
- name: "Push CI image cache"
+ push-image-cache:
+ name: "Push CI ${{ inputs.cache-type }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
- python: ${{fromJson(inputs.python-versions)}}
+ python: ${{ fromJson(inputs.python-versions) }}
env:
- RUNS_ON: '["self-hosted", "Linux", "X64"]'
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
USE_UV: ${{ inputs.use-uv }}
- IMAGE_TAG: ${{ inputs.image-tag }}
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
UPGRADE_TO_NEWER_DEPENDENCIES: "false"
DOCKER_CACHE: ${{ inputs.docker-cache }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
- PLATFORM: "linux/amd64,linux/arm64"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.sha }}
VERBOSE: "true"
@@ -109,45 +106,40 @@ jobs:
uses: ./.github/actions/cleanup-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
- - name: "Cleanup dist and context file"
- run: rm -fv ./dist/* ./docker-context-files/*
- if: inputs.include-prod-images == 'true'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+ if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }}
(AMD/ARM)"
+ - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }}
${{ inputs.platform }}"
run: >
- breeze ci-image build --builder airflow_cache
- --prepare-buildx-cache --run-in-parallel --platform
"linux/amd64,linux/arm64"
- --python ${{ matrix.python }}
+ breeze ci-image build --builder airflow_cache --prepare-buildx-cache
+ --platform "${{ inputs.platform }}" --python ${{ matrix.python }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always()
- - name: "Push CI latest images: ${{ matrix.python }} (AMD ONLY)"
+ if: always() && inputs.platform == 'linux/arm64'
+ - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)"
run: >
- breeze ci-image build --tag-as-latest --push --python
"${{matrix.python}}" --platform "linux/amd64"
- if: inputs.push-latest-images == 'true'
+ breeze ci-image build --push
+ --python "${{ matrix.python }}" --platform "${{ inputs.platform }}"
+ if: inputs.push-latest-images == 'true' && inputs.platform ==
'linux/amd64'
push-prod-image-cache:
- name: "Push PROD image cache"
+ name: "Push PROD ${{ inputs.cache-type }} image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
- python: ${{fromJson(inputs.python-versions)}}
+ python: ${{ fromJson(inputs.python-versions) }}
env:
- RUNS_ON: '["self-hosted", "Linux", "X64"]'
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
USE_UV: ${{ inputs.use-uv }}
- IMAGE_TAG: ${{ inputs.image-tag }}
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
UPGRADE_TO_NEWER_DEPENDENCIES: "false"
DOCKER_CACHE: ${{ inputs.docker-cache }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
- PLATFORM: "linux/amd64,linux/arm64"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.sha }}
VERBOSE: "true"
@@ -177,23 +169,23 @@ jobs:
run: cp -v --no-preserve=mode,ownership ./dist/*.whl
./docker-context-files
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
+ if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- - name: "Push PROD ${{ inputs.cache-type }} cache: ${{
matrix.python-version }} (AMD/ARM)"
+ - name: "Push PROD ${{ inputs.cache-type }} cache: ${{
matrix.python-version }} ${{ inputs.platform }}"
run: >
breeze prod-image build --builder airflow_cache
- --prepare-buildx-cache --run-in-parallel --platform
"linux/amd64,linux/arm64"
+ --prepare-buildx-cache --platform "${{ inputs.platform }}"
--install-packages-from-context --airflow-constraints-mode
constraints-source-providers
--python ${{ matrix.python }}
- if: inputs.include-prod-images == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always()
+ if: always() && inputs.platform == 'linux/arm64'
# We only push "AMD" images as it is really only needed for any kind
of automated builds in CI
# and currently there is not an easy way to make multi-platform image
from two separate builds
# and we can do it after we stopped the ARM instance as it is not
needed anymore
- - name: "Push PROD latest image: ${{ matrix.python }} (AMD ONLY)"
+ - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)"
run: >
- breeze prod-image build --tag-as-latest
--install-packages-from-context
- --push --python ${{ matrix.python}} --platform "linux/amd64"
- if: inputs.push-latest-images == 'true'
+ breeze prod-image build --install-packages-from-context
+ --push --python ${{ matrix.python}} --platform "${{ inputs.platform
}}"
+ if: inputs.push-latest-images == 'true' && inputs.platform ==
'linux/amd64'
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 3ad92a19e5..cb75be5f19 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -381,12 +381,15 @@ def prepare_base_build_command(image_params:
CommonBuildParams) -> list[str]:
]
)
if not image_params.docker_host:
+ builder = get_and_use_docker_context(image_params.builder)
build_command_param.extend(
[
"--builder",
- get_and_use_docker_context(image_params.builder),
+ builder,
]
)
+ if builder != "default":
+ build_command_param.append("--load")
else:
build_command_param.append("build")
return build_command_param
@@ -658,6 +661,7 @@ def autodetect_docker_context():
def get_and_use_docker_context(context: str):
if context == "autodetect":
context = autodetect_docker_context()
+ run_command(["docker", "context", "create", context], check=False)
output = run_command(["docker", "context", "use", context], check=False)
if output.returncode != 0:
get_console().print(
diff --git a/dev/breeze/src/airflow_breeze/utils/image.py
b/dev/breeze/src/airflow_breeze/utils/image.py
index 5172afc15e..3adc920a4c 100644
--- a/dev/breeze/src/airflow_breeze/utils/image.py
+++ b/dev/breeze/src/airflow_breeze/utils/image.py
@@ -197,18 +197,8 @@ def tag_image_as_latest(image_params: CommonBuildParams,
output: Output | None)
check=False,
)
if command.returncode != 0:
- return command
- if image_params.push:
- command = run_command(
- [
- "docker",
- "push",
- image_params.airflow_image_name + ":latest",
- ],
- output=output,
- capture_output=True,
- check=False,
- )
+ get_console(output=output).print(command.stdout)
+ get_console(output=output).print(command.stderr)
return command
