This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e41b3c58e2 Revert "Fix image cache optimizations - speeding up the
build (#38442)" (#38468)
e41b3c58e2 is described below
commit e41b3c58e2d2f5968bd97b8d3b05a16e926be980
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Mar 25 19:14:24 2024 +0100
Revert "Fix image cache optimizations - speeding up the build (#38442)"
(#38468)
This reverts commit 911048b212ae5dbd46742bec6f6f9017570d9e9d.
---
.github/workflows/additional-ci-image-checks.yml | 56 +++++++++---------
.github/workflows/build-images.yml | 3 +-
.github/workflows/ci-image-build.yml | 17 +++---
.github/workflows/ci.yml | 2 -
.github/workflows/finalize-tests.yml | 67 ++++++++-------------
.github/workflows/prod-image-build.yml | 9 +--
.github/workflows/prod-image-extra-checks.yml | 3 -
.github/workflows/push-image-cache.yml | 68 ++++++++++++----------
.../airflow_breeze/utils/docker_command_utils.py | 6 +-
dev/breeze/src/airflow_breeze/utils/image.py | 14 ++++-
10 files changed, 117 insertions(+), 128 deletions(-)
diff --git a/.github/workflows/additional-ci-image-checks.yml
b/.github/workflows/additional-ci-image-checks.yml
index 6de5862aef..1bee163b0f 100644
--- a/.github/workflows/additional-ci-image-checks.yml
+++ b/.github/workflows/additional-ci-image-checks.yml
@@ -89,40 +89,42 @@ jobs:
# delay cache refresh. It does not attempt to upgrade to newer dependencies.
# We only push CI cache as PROD cache usually does not gain as much from
fresh cache because
# it uses prepared airflow and provider packages that invalidate the cache
anyway most of the time
- push-early-buildx-cache-to-github-registry:
- name: Push Early Image Cache
- uses: ./.github/workflows/push-image-cache.yml
- permissions:
- contents: read
- # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
- # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
- # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
- # "in-workflow-build" condition
- packages: write
- secrets: inherit
- with:
- # Runs on Public runners
- cache-type: "Early"
- include-prod-images: "false"
- push-latest-images: "false"
- platform: "linux/amd64"
- python-versions: ${{ inputs.python-versions }}
- branch: ${{ inputs.branch }}
- constraints-branch: ${{ inputs.constraints-branch }}
- use-uv: "true"
- include-success-outputs: ${{ inputs.include-success-outputs }}
- docker-cache: ${{ inputs.docker-cache }}
- if: inputs.canary-run == 'true' && inputs.branch == 'main'
+ # push-early-buildx-cache-to-github-registry:
+ # name: Push Early Image Cache
+ # uses: ./.github/workflows/push-image-cache.yml
+ # permissions:
+ # contents: read
+ # # This write is only given here for `push` events from
"apache/airflow" repo. It is not given for PRs
+ # # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
+ # # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
+ # # "in-workflow-build" condition
+ # packages: write
+ # secrets: inherit
+ # with:
+ # runs-on: ${{ inputs.runs-on }}
+ # cache-type: "Early"
+ # include-prod-images: "false"
+ # push-latest-images: "false"
+ # image-tag: ${{ inputs.image-tag }}
+ # python-versions: ${{ inputs.python-versions }}
+ # branch: ${{ inputs.branch }}
+ # use-uv: "true"
+ # include-success-outputs: ${{ inputs.include-success-outputs }}
+ # constraints-branch: ${{ inputs.constraints-branch }}
+ # docker-cache: ${{ inputs.docker-cache }}
+ # if: inputs.canary-run == 'true' && inputs.branch == 'main'
# Check that after earlier cache push, breeze command will build quickly
check-that-image-builds-quickly:
- timeout-minutes: 11
+ timeout-minutes: 5
name: Check that image builds quickly
runs-on: ["ubuntu-22.04"]
env:
UPGRADE_TO_NEWER_DEPENDENCIES: false
+ PLATFORM: "linux/amd64"
PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }}
PYTHON_VERSION: ${{ inputs.default-python-version }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
if: inputs.canary-run == 'true' && inputs.branch == 'main'
steps:
@@ -140,7 +142,7 @@ jobs:
- name: "Login to ghcr.io"
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- name: "Check that image builds quickly"
- run: breeze shell --max-time 600 --platform "linux/amd64"
+ run: breeze shell --max-time 120
# This is only a check if ARM images are successfully building when
committer runs PR from
# Apache repository. This is needed in case you want to fix failing cache
job in "canary" run
@@ -154,11 +156,11 @@ jobs:
packages: write
secrets: inherit
with:
+ platform: "arm64"
push-image: "false"
runs-on: ${{ inputs.runs-on }}
image-tag: ${{ inputs.image-tag }}
python-versions: ${{ inputs.python-versions }}
- platform: "linux/arm64"
branch: ${{ inputs.branch }}
constraints-branch: ${{ inputs.constraints-branch }}
use-uv: "true"
diff --git a/.github/workflows/build-images.yml
b/.github/workflows/build-images.yml
index 6897e2c07c..d2d574c806 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -180,13 +180,13 @@ jobs:
upload-constraints: "true"
use-uv: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
- platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
+
build-prod-images:
name: Build PROD images
permissions:
@@ -209,7 +209,6 @@ jobs:
push-image: "true"
use-uv: "true"
image-tag: ${{ needs.build-info.outputs.image-tag }}
- platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
constraints-branch: ${{ needs.build-info.outputs.constraints-branch }}
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 177cd39fbb..9eff0ced3a 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -48,8 +48,9 @@ on: # yamllint disable-line rule:truthy
default: "false"
type: string
platform:
- description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
- required: true
+ description: >
+ Name of the platform for the build - 'amd64/arm64'
+ default: "amd64"
type: string
push-image:
description: "Whether to push image to the registry (true/false)"
@@ -106,8 +107,8 @@ jobs:
timeout-minutes: 110
name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-CI ${{ inputs.platform }} image\
-${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
+CI ${{inputs.platform}} image\
+${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
runs-on: ${{ fromJSON(inputs.runs-on) }}
env:
@@ -184,7 +185,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies
!= 'false'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.do-build == 'true' && inputs.platform == 'linux/arm64'
+ if: inputs.do-build == 'true' && inputs.platform == 'arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
if: inputs.do-build == 'true'
@@ -194,19 +195,19 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || ''
}}"
run: >
breeze ci-image build --tag-as-latest --image-tag "${{
inputs.image-tag }}"
--python "${{ matrix.python-version }}"
- --platform "${{ inputs.platform }}"
+ --platform "linux/${{ inputs.platform }}"
env:
DOCKER_CACHE: ${{ inputs.docker-cache }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- BUILDER: "airflow_cache"
+ BUILDER: ${{ inputs.platform == 'amd64' && 'default' ||
'airflow_cache' }}
PUSH: ${{ inputs.push-image }}
VERBOSE: "true"
if: inputs.do-build == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.do-build == 'true' && inputs.platform ==
'linux/arm64'
+ if: always() && inputs.do-build == 'true' && inputs.platform == 'arm64'
- name: "Source constraints: ${{ matrix.python-version }}"
shell: bash
run: >
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5a869a160f..37b6dd3c7d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -196,7 +196,6 @@ jobs:
with:
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
- platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
upload-constraints: "true"
branch: ${{ needs.build-info.outputs.default-branch }}
@@ -497,7 +496,6 @@ jobs:
build-type: "Regular"
do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
- platform: "linux/amd64"
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
push-image: "true"
diff --git a/.github/workflows/finalize-tests.yml
b/.github/workflows/finalize-tests.yml
index a10b1826d1..48971ff789 100644
--- a/.github/workflows/finalize-tests.yml
+++ b/.github/workflows/finalize-tests.yml
@@ -120,48 +120,31 @@ jobs:
run:
git push
- push-buildx-cache-to-github-registry-amd:
- name: Push Regular Image Cache
- needs: [update-constraints]
- uses: ./.github/workflows/push-image-cache.yml
- permissions:
- contents: read
- packages: write
- secrets: inherit
- with:
- cache-type: "Regular"
- include-prod-images: "true"
- push-latest-images: "true"
- platform: "linux/amd64"
- python-versions: ${{ inputs.python-versions }}
- branch: ${{ inputs.branch }}
- constraints-branch: ${{ inputs.constraints-branch }}
- use-uv: "true"
- include-success-outputs: ${{ inputs.include-success-outputs }}
- docker-cache: ${{ inputs.docker-cache }}
- if: inputs.canary-run == 'true'
-
- push-buildx-cache-to-github-registry-arm:
- name: Push Regular Image Cache
- needs: [update-constraints]
- uses: ./.github/workflows/push-image-cache.yml
- permissions:
- contents: read
- packages: write
- secrets: inherit
- with:
- runs-on: ${{ inputs.runs-on }}
- cache-type: "Regular"
- include-prod-images: "true"
- push-latest-images: "true"
- platform: "linux/arm64"
- python-versions: ${{ inputs.python-versions }}
- branch: ${{ inputs.branch }}
- constraints-branch: ${{ inputs.constraints-branch }}
- use-uv: "true"
- include-success-outputs: ${{ inputs.include-success-outputs }}
- docker-cache: ${{ inputs.docker-cache }}
- if: inputs.canary-run == 'true'
+ # Push BuildX cache to GitHub Registry in Apache repository, if all tests
are successful and build
+ # is executed as result of direct push to "main" or one of the "vX-Y-test"
branches
+ # It rebuilds all images using just-pushed constraints using buildx and
pushes them to registry
+ # It will automatically check if a new python image was released and will
pull the latest one if needed
+ # push-buildx-cache-to-github-registry:
+ # name: Push Regular Image Cache
+ # needs: [update-constraints]
+ # uses: ./.github/workflows/push-image-cache.yml
+ # permissions:
+ # contents: read
+ # packages: write
+ # secrets: inherit
+ # with:
+ # runs-on: ${{ inputs.runs-on }}
+ # cache-type: "Regular"
+ # include-prod-images: "true"
+ # push-latest-images: "true"
+ # use-uv: "true"
+ # image-tag: ${{ inputs.image-tag }}
+ # python-versions: ${{ inputs.python-versions }}
+ # branch: ${{ inputs.branch }}
+ # constraints-branch: ${{ inputs.constraints-branch }}
+ # include-success-outputs: ${{ inputs.include-success-outputs }}
+ # docker-cache: ${{ inputs.docker-cache }}
+ # if: inputs.canary-run == 'true'
summarize-warnings:
timeout-minutes: 15
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 27cd62d42c..6005e44d56 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -77,10 +77,6 @@ on: # yamllint disable-line rule:truthy
description: "JSON-formatted array of Python versions to build images
from"
required: true
type: string
- platform:
- description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
- required: true
- type: string
branch:
description: "Branch used to run the CI jobs in (main/v2_*_test)."
required: true
@@ -115,8 +111,8 @@ jobs:
timeout-minutes: 80
name: "\
${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \
-PROD ${{ inputs.build-type }} image\
-${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\
+PROD ${{inputs.build-type}} image\
+${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\
${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
runs-on: ${{ fromJSON(inputs.runs-on) }}
env:
@@ -252,7 +248,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}"
PUSH: ${{ inputs.push-image }}
DOCKER_CACHE: ${{ inputs.docker-cache }}
DEBIAN_VERSION: ${{ inputs.debian-version }}
- BUILDER: "airflow_cache"
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{
inputs.upgrade-to-newer-dependencies }}
INCLUDE_NOT_READY_PROVIDERS: "true"
diff --git a/.github/workflows/prod-image-extra-checks.yml
b/.github/workflows/prod-image-extra-checks.yml
index faa904e438..69dee4e4a9 100644
--- a/.github/workflows/prod-image-extra-checks.yml
+++ b/.github/workflows/prod-image-extra-checks.yml
@@ -59,7 +59,6 @@ jobs:
image-tag: bullseye-${{ inputs.image-tag }}
debian-version: "bullseye"
python-versions: ${{ inputs.python-versions }}
- platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
@@ -77,7 +76,6 @@ jobs:
image-tag: mysql-${{ inputs.image-tag }}
install-mysql-client-type: "mysql"
python-versions: ${{ inputs.python-versions }}
- platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
@@ -95,7 +93,6 @@ jobs:
image-tag: mysql-${{ inputs.image-tag }}
install-mysql-client-type: "mysql"
python-versions: ${{ inputs.python-versions }}
- platform: "linux/amd64"
branch: ${{ inputs.branch }}
# Always build images during the extra checks and never push them
push-image: "false"
diff --git a/.github/workflows/push-image-cache.yml
b/.github/workflows/push-image-cache.yml
index c89ef49302..dbc8893a8f 100644
--- a/.github/workflows/push-image-cache.yml
+++ b/.github/workflows/push-image-cache.yml
@@ -23,14 +23,14 @@ on: # yamllint disable-line rule:truthy
runs-on:
description: "The array of labels (in json form) determining type of
the runner to use for the build."
required: false
- default: "[\"ubuntu-22.04\"]"
+ default: '["self-hosted", "Linux", "X64"]'
type: string
cache-type:
description: "Type of cache to push (Early / Regular)."
required: true
type: string
include-prod-images:
- description: "Whether to build PROD image cache additionally to CI
image cache (true/false)."
+ description: "Whether to include prod images in the cache
(true/false)."
required: true
type: string
push-latest-images:
@@ -45,8 +45,12 @@ on: # yamllint disable-line rule:truthy
description: "MySQL client type to use during build (mariadb/mysql)"
type: string
default: "mariadb"
- platform:
- description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+ use-uv:
+ description: "Whether to use uv to build the image (true/false)"
+ required: true
+ type: string
+ image-tag:
+ description: "Tag to set for the image"
required: true
type: string
python-versions:
@@ -61,10 +65,6 @@ on: # yamllint disable-line rule:truthy
description: "Branch used to construct constraints URL from."
required: true
type: string
- use-uv:
- description: "Whether to use uv to build the image (true/false)"
- required: true
- type: string
include-success-outputs:
description: "Whether to include success outputs (true/false)."
required: true
@@ -74,22 +74,25 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
jobs:
- push-image-cache:
- name: "Push CI ${{ inputs.cache-type }} image cache"
+ push-ci-image-cache:
+ name: "Push CI image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
- python: ${{ fromJson(inputs.python-versions) }}
+ python: ${{fromJson(inputs.python-versions)}}
env:
+ RUNS_ON: '["self-hosted", "Linux", "X64"]'
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
USE_UV: ${{ inputs.use-uv }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
UPGRADE_TO_NEWER_DEPENDENCIES: "false"
DOCKER_CACHE: ${{ inputs.docker-cache }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
+ PLATFORM: "linux/amd64,linux/arm64"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.sha }}
VERBOSE: "true"
@@ -106,40 +109,45 @@ jobs:
uses: ./.github/actions/cleanup-docker
- name: "Install Breeze"
uses: ./.github/actions/breeze
+ - name: "Cleanup dist and context file"
+ run: rm -fv ./dist/* ./docker-context-files/*
+ if: inputs.include-prod-images == 'true'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }}
${{ inputs.platform }}"
+ - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }}
(AMD/ARM)"
run: >
- breeze ci-image build --builder airflow_cache --prepare-buildx-cache
- --platform "${{ inputs.platform }}" --python ${{ matrix.python }}
+ breeze ci-image build --builder airflow_cache
+ --prepare-buildx-cache --run-in-parallel --platform
"linux/amd64,linux/arm64"
+ --python ${{ matrix.python }}
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.platform == 'linux/arm64'
- - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)"
+ if: always()
+ - name: "Push CI latest images: ${{ matrix.python }} (AMD ONLY)"
run: >
- breeze ci-image build --push
- --python "${{ matrix.python }}" --platform "${{ inputs.platform }}"
- if: inputs.push-latest-images == 'true' && inputs.platform ==
'linux/amd64'
+ breeze ci-image build --tag-as-latest --push --python
"${{matrix.python}}" --platform "linux/amd64"
+ if: inputs.push-latest-images == 'true'
push-prod-image-cache:
- name: "Push PROD ${{ inputs.cache-type }} image cache"
+ name: "Push PROD image cache"
runs-on: ${{ fromJSON(inputs.runs-on) }}
strategy:
fail-fast: false
matrix:
- python: ${{ fromJson(inputs.python-versions) }}
+ python: ${{fromJson(inputs.python-versions)}}
env:
+ RUNS_ON: '["self-hosted", "Linux", "X64"]'
DEBIAN_VERSION: ${{ inputs.debian-version }}
INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }}
USE_UV: ${{ inputs.use-uv }}
+ IMAGE_TAG: ${{ inputs.image-tag }}
DEFAULT_BRANCH: ${{ inputs.branch }}
DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
UPGRADE_TO_NEWER_DEPENDENCIES: "false"
DOCKER_CACHE: ${{ inputs.docker-cache }}
VERSION_SUFFIX_FOR_PYPI: "dev0"
+ PLATFORM: "linux/amd64,linux/arm64"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COMMIT_SHA: ${{ github.sha }}
VERBOSE: "true"
@@ -169,23 +177,23 @@ jobs:
run: cp -v --no-preserve=mode,ownership ./dist/*.whl
./docker-context-files
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.platform == 'linux/arm64'
- name: Login to ghcr.io
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- - name: "Push PROD ${{ inputs.cache-type }} cache: ${{
matrix.python-version }} ${{ inputs.platform }}"
+ - name: "Push PROD ${{ inputs.cache-type }} cache: ${{
matrix.python-version }} (AMD/ARM)"
run: >
breeze prod-image build --builder airflow_cache
- --prepare-buildx-cache --platform "${{ inputs.platform }}"
+ --prepare-buildx-cache --run-in-parallel --platform
"linux/amd64,linux/arm64"
--install-packages-from-context --airflow-constraints-mode
constraints-source-providers
--python ${{ matrix.python }}
+ if: inputs.include-prod-images == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.platform == 'linux/arm64'
+ if: always()
# We only push "AMD" images as it is really only needed for any kind
of automated builds in CI
# and currently there is not an easy way to make multi-platform image
from two separate builds
# and we can do it after we stopped the ARM instance as it is not
needed anymore
- - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)"
+ - name: "Push PROD latest image: ${{ matrix.python }} (AMD ONLY)"
run: >
- breeze prod-image build --install-packages-from-context
- --push --python ${{ matrix.python}} --platform "${{ inputs.platform
}}"
- if: inputs.push-latest-images == 'true' && inputs.platform ==
'linux/amd64'
+ breeze prod-image build --tag-as-latest
--install-packages-from-context
+ --push --python ${{ matrix.python}} --platform "linux/amd64"
+ if: inputs.push-latest-images == 'true'
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index cb75be5f19..3ad92a19e5 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -381,15 +381,12 @@ def prepare_base_build_command(image_params:
CommonBuildParams) -> list[str]:
]
)
if not image_params.docker_host:
- builder = get_and_use_docker_context(image_params.builder)
build_command_param.extend(
[
"--builder",
- builder,
+ get_and_use_docker_context(image_params.builder),
]
)
- if builder != "default":
- build_command_param.append("--load")
else:
build_command_param.append("build")
return build_command_param
@@ -661,7 +658,6 @@ def autodetect_docker_context():
def get_and_use_docker_context(context: str):
if context == "autodetect":
context = autodetect_docker_context()
- run_command(["docker", "context", "create", context], check=False)
output = run_command(["docker", "context", "use", context], check=False)
if output.returncode != 0:
get_console().print(
diff --git a/dev/breeze/src/airflow_breeze/utils/image.py
b/dev/breeze/src/airflow_breeze/utils/image.py
index 3adc920a4c..5172afc15e 100644
--- a/dev/breeze/src/airflow_breeze/utils/image.py
+++ b/dev/breeze/src/airflow_breeze/utils/image.py
@@ -197,8 +197,18 @@ def tag_image_as_latest(image_params: CommonBuildParams,
output: Output | None)
check=False,
)
if command.returncode != 0:
- get_console(output=output).print(command.stdout)
- get_console(output=output).print(command.stderr)
+ return command
+ if image_params.push:
+ command = run_command(
+ [
+ "docker",
+ "push",
+ image_params.airflow_image_name + ":latest",
+ ],
+ output=output,
+ capture_output=True,
+ check=False,
+ )
return command
