This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new e41b3c58e2 Revert "Fix image cache optimizations - speeding up the build (#38442)" (#38468) e41b3c58e2 is described below commit e41b3c58e2d2f5968bd97b8d3b05a16e926be980 Author: Jarek Potiuk <ja...@potiuk.com> AuthorDate: Mon Mar 25 19:14:24 2024 +0100 Revert "Fix image cache optimizations - speeding up the build (#38442)" (#38468) This reverts commit 911048b212ae5dbd46742bec6f6f9017570d9e9d. --- .github/workflows/additional-ci-image-checks.yml | 56 +++++++++--------- .github/workflows/build-images.yml | 3 +- .github/workflows/ci-image-build.yml | 17 +++--- .github/workflows/ci.yml | 2 - .github/workflows/finalize-tests.yml | 67 ++++++++------------- .github/workflows/prod-image-build.yml | 9 +-- .github/workflows/prod-image-extra-checks.yml | 3 - .github/workflows/push-image-cache.yml | 68 ++++++++++++---------- .../airflow_breeze/utils/docker_command_utils.py | 6 +- dev/breeze/src/airflow_breeze/utils/image.py | 14 ++++- 10 files changed, 117 insertions(+), 128 deletions(-) diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 6de5862aef..1bee163b0f 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -89,40 +89,42 @@ jobs: # delay cache refresh. It does not attempt to upgrade to newer dependencies. # We only push CI cache as PROD cache usually does not gain as much from fresh cache because # it uses prepared airflow and provider packages that invalidate the cache anyway most of the time - push-early-buildx-cache-to-github-registry: - name: Push Early Image Cache - uses: ./.github/workflows/push-image-cache.yml - permissions: - contents: read - # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs - # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. - # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the - # "in-workflow-build" condition - packages: write - secrets: inherit - with: - # Runs on Public runners - cache-type: "Early" - include-prod-images: "false" - push-latest-images: "false" - platform: "linux/amd64" - python-versions: ${{ inputs.python-versions }} - branch: ${{ inputs.branch }} - constraints-branch: ${{ inputs.constraints-branch }} - use-uv: "true" - include-success-outputs: ${{ inputs.include-success-outputs }} - docker-cache: ${{ inputs.docker-cache }} - if: inputs.canary-run == 'true' && inputs.branch == 'main' + # push-early-buildx-cache-to-github-registry: + # name: Push Early Image Cache + # uses: ./.github/workflows/push-image-cache.yml + # permissions: + # contents: read + # # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + # # For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the + # # "in-workflow-build" condition + # packages: write + # secrets: inherit + # with: + # runs-on: ${{ inputs.runs-on }} + # cache-type: "Early" + # include-prod-images: "false" + # push-latest-images: "false" + # image-tag: ${{ inputs.image-tag }} + # python-versions: ${{ inputs.python-versions }} + # branch: ${{ inputs.branch }} + # use-uv: "true" + # include-success-outputs: ${{ inputs.include-success-outputs }} + # constraints-branch: ${{ inputs.constraints-branch }} + # docker-cache: ${{ inputs.docker-cache }} + # if: inputs.canary-run == 'true' && inputs.branch == 'main' # Check that after earlier cache push, breeze command will build quickly check-that-image-builds-quickly: - timeout-minutes: 11 + timeout-minutes: 5 name: Check that image builds quickly runs-on: ["ubuntu-22.04"] env: UPGRADE_TO_NEWER_DEPENDENCIES: false + PLATFORM: "linux/amd64" PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.default-python-version }} PYTHON_VERSION: ${{ inputs.default-python-version }} + IMAGE_TAG: ${{ inputs.image-tag }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} if: inputs.canary-run == 'true' && inputs.branch == 'main' steps: @@ -140,7 +142,7 @@ jobs: - name: "Login to ghcr.io" run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - name: "Check that image builds quickly" - run: breeze shell --max-time 600 --platform "linux/amd64" + run: breeze shell --max-time 120 # This is only a check if ARM images are successfully building when committer runs PR from # Apache repository. This is needed in case you want to fix failing cache job in "canary" run @@ -154,11 +156,11 @@ jobs: packages: write secrets: inherit with: + platform: "arm64" push-image: "false" runs-on: ${{ inputs.runs-on }} image-tag: ${{ inputs.image-tag }} python-versions: ${{ inputs.python-versions }} - platform: "linux/arm64" branch: ${{ inputs.branch }} constraints-branch: ${{ inputs.constraints-branch }} use-uv: "true" diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 6897e2c07c..d2d574c806 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -180,13 +180,13 @@ jobs: upload-constraints: "true" use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} - platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} docker-cache: ${{ needs.build-info.outputs.cache-directive }} + build-prod-images: name: Build PROD images permissions: @@ -209,7 +209,6 @@ jobs: push-image: "true" use-uv: "true" image-tag: ${{ needs.build-info.outputs.image-tag }} - platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} constraints-branch: ${{ needs.build-info.outputs.constraints-branch }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 177cd39fbb..9eff0ced3a 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -48,8 +48,9 @@ on: # yamllint disable-line rule:truthy default: "false" type: string platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" - required: true + description: > + Name of the platform for the build - 'amd64/arm64' + default: "amd64" type: string push-image: description: "Whether to push image to the registry (true/false)" @@ -106,8 +107,8 @@ jobs: timeout-minutes: 110 name: "\ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ -CI ${{ inputs.platform }} image\ -${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\ +CI ${{inputs.platform}} image\ +${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" runs-on: ${{ fromJSON(inputs.runs-on) }} env: @@ -184,7 +185,7 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies != 'false' - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: inputs.do-build == 'true' && inputs.platform == 'linux/arm64' + if: inputs.do-build == 'true' && inputs.platform == 'arm64' - name: Login to ghcr.io run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin if: inputs.do-build == 'true' @@ -194,19 +195,19 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" run: > breeze ci-image build --tag-as-latest --image-tag "${{ inputs.image-tag }}" --python "${{ matrix.python-version }}" - --platform "${{ inputs.platform }}" + --platform "linux/${{ inputs.platform }}" env: DOCKER_CACHE: ${{ inputs.docker-cache }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BUILDER: "airflow_cache" + BUILDER: ${{ inputs.platform == 'amd64' && 'default' || 'airflow_cache' }} PUSH: ${{ inputs.push-image }} VERBOSE: "true" if: inputs.do-build == 'true' - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() && inputs.do-build == 'true' && inputs.platform == 'linux/arm64' + if: always() && inputs.do-build == 'true' && inputs.platform == 'arm64' - name: "Source constraints: ${{ matrix.python-version }}" shell: bash run: > diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a869a160f..37b6dd3c7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,7 +196,6 @@ jobs: with: do-build: ${{ needs.build-info.outputs.in-workflow-build }} image-tag: ${{ needs.build-info.outputs.image-tag }} - platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} upload-constraints: "true" branch: ${{ needs.build-info.outputs.default-branch }} @@ -497,7 +496,6 @@ jobs: build-type: "Regular" do-build: ${{ needs.build-info.outputs.in-workflow-build }} image-tag: ${{ needs.build-info.outputs.image-tag }} - platform: "linux/amd64" python-versions: ${{ needs.build-info.outputs.python-versions }} branch: ${{ needs.build-info.outputs.default-branch }} push-image: "true" diff --git a/.github/workflows/finalize-tests.yml b/.github/workflows/finalize-tests.yml index a10b1826d1..48971ff789 100644 --- a/.github/workflows/finalize-tests.yml +++ b/.github/workflows/finalize-tests.yml @@ -120,48 +120,31 @@ jobs: run: git push - push-buildx-cache-to-github-registry-amd: - name: Push Regular Image Cache - needs: [update-constraints] - uses: ./.github/workflows/push-image-cache.yml - permissions: - contents: read - packages: write - secrets: inherit - with: - cache-type: "Regular" - include-prod-images: "true" - push-latest-images: "true" - platform: "linux/amd64" - python-versions: ${{ inputs.python-versions }} - branch: ${{ inputs.branch }} - constraints-branch: ${{ inputs.constraints-branch }} - use-uv: "true" - include-success-outputs: ${{ inputs.include-success-outputs }} - docker-cache: ${{ inputs.docker-cache }} - if: inputs.canary-run == 'true' - - push-buildx-cache-to-github-registry-arm: - name: Push Regular Image Cache - needs: [update-constraints] - uses: ./.github/workflows/push-image-cache.yml - permissions: - contents: read - packages: write - secrets: inherit - with: - runs-on: ${{ inputs.runs-on }} - cache-type: "Regular" - include-prod-images: "true" - push-latest-images: "true" - platform: "linux/arm64" - python-versions: ${{ inputs.python-versions }} - branch: ${{ inputs.branch }} - constraints-branch: ${{ inputs.constraints-branch }} - use-uv: "true" - include-success-outputs: ${{ inputs.include-success-outputs }} - docker-cache: ${{ inputs.docker-cache }} - if: inputs.canary-run == 'true' + # Push BuildX cache to GitHub Registry in Apache repository, if all tests are successful and build + # is executed as result of direct push to "main" or one of the "vX-Y-test" branches + # It rebuilds all images using just-pushed constraints using buildx and pushes them to registry + # It will automatically check if a new python image was released and will pull the latest one if needed + # push-buildx-cache-to-github-registry: + # name: Push Regular Image Cache + # needs: [update-constraints] + # uses: ./.github/workflows/push-image-cache.yml + # permissions: + # contents: read + # packages: write + # secrets: inherit + # with: + # runs-on: ${{ inputs.runs-on }} + # cache-type: "Regular" + # include-prod-images: "true" + # push-latest-images: "true" + # use-uv: "true" + # image-tag: ${{ inputs.image-tag }} + # python-versions: ${{ inputs.python-versions }} + # branch: ${{ inputs.branch }} + # constraints-branch: ${{ inputs.constraints-branch }} + # include-success-outputs: ${{ inputs.include-success-outputs }} + # docker-cache: ${{ inputs.docker-cache }} + # if: inputs.canary-run == 'true' summarize-warnings: timeout-minutes: 15 diff --git a/.github/workflows/prod-image-build.yml b/.github/workflows/prod-image-build.yml index 27cd62d42c..6005e44d56 100644 --- a/.github/workflows/prod-image-build.yml +++ b/.github/workflows/prod-image-build.yml @@ -77,10 +77,6 @@ on: # yamllint disable-line rule:truthy description: "JSON-formatted array of Python versions to build images from" required: true type: string - platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" - required: true - type: string branch: description: "Branch used to run the CI jobs in (main/v2_*_test)." required: true @@ -115,8 +111,8 @@ jobs: timeout-minutes: 80 name: "\ ${{ inputs.do-build == 'true' && 'Build' || 'Skip building' }} \ -PROD ${{ inputs.build-type }} image\ -${{ matrix.python-version }}${{ inputs.do-build == 'true' && ':' || '' }}\ +PROD ${{inputs.build-type}} image\ +${{matrix.python-version}}${{ inputs.do-build == 'true' && ':' || '' }}\ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" runs-on: ${{ fromJSON(inputs.runs-on) }} env: @@ -252,7 +248,6 @@ ${{ inputs.do-build == 'true' && inputs.image-tag || '' }}" PUSH: ${{ inputs.push-image }} DOCKER_CACHE: ${{ inputs.docker-cache }} DEBIAN_VERSION: ${{ inputs.debian-version }} - BUILDER: "airflow_cache" INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ inputs.upgrade-to-newer-dependencies }} INCLUDE_NOT_READY_PROVIDERS: "true" diff --git a/.github/workflows/prod-image-extra-checks.yml b/.github/workflows/prod-image-extra-checks.yml index faa904e438..69dee4e4a9 100644 --- a/.github/workflows/prod-image-extra-checks.yml +++ b/.github/workflows/prod-image-extra-checks.yml @@ -59,7 +59,6 @@ jobs: image-tag: bullseye-${{ inputs.image-tag }} debian-version: "bullseye" python-versions: ${{ inputs.python-versions }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" @@ -77,7 +76,6 @@ jobs: image-tag: mysql-${{ inputs.image-tag }} install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" @@ -95,7 +93,6 @@ jobs: image-tag: mysql-${{ inputs.image-tag }} install-mysql-client-type: "mysql" python-versions: ${{ inputs.python-versions }} - platform: "linux/amd64" branch: ${{ inputs.branch }} # Always build images during the extra checks and never push them push-image: "false" diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index c89ef49302..dbc8893a8f 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -23,14 +23,14 @@ on: # yamllint disable-line rule:truthy runs-on: description: "The array of labels (in json form) determining type of the runner to use for the build." required: false - default: "[\"ubuntu-22.04\"]" + default: '["self-hosted", "Linux", "X64"]' type: string cache-type: description: "Type of cache to push (Early / Regular)." required: true type: string include-prod-images: - description: "Whether to build PROD image cache additionally to CI image cache (true/false)." + description: "Whether to include prod images in the cache (true/false)." required: true type: string push-latest-images: @@ -45,8 +45,12 @@ on: # yamllint disable-line rule:truthy description: "MySQL client type to use during build (mariadb/mysql)" type: string default: "mariadb" - platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string + image-tag: + description: "Tag to set for the image" required: true type: string python-versions: @@ -61,10 +65,6 @@ on: # yamllint disable-line rule:truthy description: "Branch used to construct constraints URL from." required: true type: string - use-uv: - description: "Whether to use uv to build the image (true/false)" - required: true - type: string include-success-outputs: description: "Whether to include success outputs (true/false)." required: true @@ -74,22 +74,25 @@ on: # yamllint disable-line rule:truthy required: true type: string jobs: - push-image-cache: - name: "Push CI ${{ inputs.cache-type }} image cache" + push-ci-image-cache: + name: "Push CI image cache" runs-on: ${{ fromJSON(inputs.runs-on) }} strategy: fail-fast: false matrix: - python: ${{ fromJson(inputs.python-versions) }} + python: ${{fromJson(inputs.python-versions)}} env: + RUNS_ON: '["self-hosted", "Linux", "X64"]' DEBIAN_VERSION: ${{ inputs.debian-version }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} USE_UV: ${{ inputs.use-uv }} + IMAGE_TAG: ${{ inputs.image-tag }} DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} UPGRADE_TO_NEWER_DEPENDENCIES: "false" DOCKER_CACHE: ${{ inputs.docker-cache }} VERSION_SUFFIX_FOR_PYPI: "dev0" + PLATFORM: "linux/amd64,linux/arm64" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMMIT_SHA: ${{ github.sha }} VERBOSE: "true" @@ -106,40 +109,45 @@ jobs: uses: ./.github/actions/cleanup-docker - name: "Install Breeze" uses: ./.github/actions/breeze + - name: "Cleanup dist and context file" + run: rm -fv ./dist/* ./docker-context-files/* + if: inputs.include-prod-images == 'true' - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: inputs.platform == 'linux/arm64' - name: Login to ghcr.io run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }} ${{ inputs.platform }}" + - name: "Push CI ${{ inputs.cache-type }} cache: ${{ matrix.python }} (AMD/ARM)" run: > - breeze ci-image build --builder airflow_cache --prepare-buildx-cache - --platform "${{ inputs.platform }}" --python ${{ matrix.python }} + breeze ci-image build --builder airflow_cache + --prepare-buildx-cache --run-in-parallel --platform "linux/amd64,linux/arm64" + --python ${{ matrix.python }} - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() && inputs.platform == 'linux/arm64' - - name: "Push CI latest images: ${{ matrix.python }} (linux/amd64 only)" + if: always() + - name: "Push CI latest images: ${{ matrix.python }} (AMD ONLY)" run: > - breeze ci-image build --push - --python "${{ matrix.python }}" --platform "${{ inputs.platform }}" - if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64' + breeze ci-image build --tag-as-latest --push --python "${{matrix.python}}" --platform "linux/amd64" + if: inputs.push-latest-images == 'true' push-prod-image-cache: - name: "Push PROD ${{ inputs.cache-type }} image cache" + name: "Push PROD image cache" runs-on: ${{ fromJSON(inputs.runs-on) }} strategy: fail-fast: false matrix: - python: ${{ fromJson(inputs.python-versions) }} + python: ${{fromJson(inputs.python-versions)}} env: + RUNS_ON: '["self-hosted", "Linux", "X64"]' DEBIAN_VERSION: ${{ inputs.debian-version }} INSTALL_MYSQL_CLIENT_TYPE: ${{ inputs.install-mysql-client-type }} USE_UV: ${{ inputs.use-uv }} + IMAGE_TAG: ${{ inputs.image-tag }} DEFAULT_BRANCH: ${{ inputs.branch }} DEFAULT_CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }} UPGRADE_TO_NEWER_DEPENDENCIES: "false" DOCKER_CACHE: ${{ inputs.docker-cache }} VERSION_SUFFIX_FOR_PYPI: "dev0" + PLATFORM: "linux/amd64,linux/arm64" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} COMMIT_SHA: ${{ github.sha }} VERBOSE: "true" @@ -169,23 +177,23 @@ jobs: run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh - if: inputs.platform == 'linux/arm64' - name: Login to ghcr.io run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin - - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} ${{ inputs.platform }}" + - name: "Push PROD ${{ inputs.cache-type }} cache: ${{ matrix.python-version }} (AMD/ARM)" run: > breeze prod-image build --builder airflow_cache - --prepare-buildx-cache --platform "${{ inputs.platform }}" + --prepare-buildx-cache --run-in-parallel --platform "linux/amd64,linux/arm64" --install-packages-from-context --airflow-constraints-mode constraints-source-providers --python ${{ matrix.python }} + if: inputs.include-prod-images == 'true' - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh - if: always() && inputs.platform == 'linux/arm64' + if: always() # We only push "AMD" images as it is really only needed for any kind of automated builds in CI # and currently there is not an easy way to make multi-platform image from two separate builds # and we can do it after we stopped the ARM instance as it is not needed anymore - - name: "Push PROD latest image: ${{ matrix.python }} (linux/amd64 ONLY)" + - name: "Push PROD latest image: ${{ matrix.python }} (AMD ONLY)" run: > - breeze prod-image build --install-packages-from-context - --push --python ${{ matrix.python}} --platform "${{ inputs.platform }}" - if: inputs.push-latest-images == 'true' && inputs.platform == 'linux/amd64' + breeze prod-image build --tag-as-latest --install-packages-from-context + --push --python ${{ matrix.python}} --platform "linux/amd64" + if: inputs.push-latest-images == 'true' diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index cb75be5f19..3ad92a19e5 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -381,15 +381,12 @@ def prepare_base_build_command(image_params: CommonBuildParams) -> list[str]: ] ) if not image_params.docker_host: - builder = get_and_use_docker_context(image_params.builder) build_command_param.extend( [ "--builder", - builder, + get_and_use_docker_context(image_params.builder), ] ) - if builder != "default": - build_command_param.append("--load") else: build_command_param.append("build") return build_command_param @@ -661,7 +658,6 @@ def autodetect_docker_context(): def get_and_use_docker_context(context: str): if context == "autodetect": context = autodetect_docker_context() - run_command(["docker", "context", "create", context], check=False) output = run_command(["docker", "context", "use", context], check=False) if output.returncode != 0: get_console().print( diff --git a/dev/breeze/src/airflow_breeze/utils/image.py b/dev/breeze/src/airflow_breeze/utils/image.py index 3adc920a4c..5172afc15e 100644 --- a/dev/breeze/src/airflow_breeze/utils/image.py +++ b/dev/breeze/src/airflow_breeze/utils/image.py @@ -197,8 +197,18 @@ def tag_image_as_latest(image_params: CommonBuildParams, output: Output | None) check=False, ) if command.returncode != 0: - get_console(output=output).print(command.stdout) - get_console(output=output).print(command.stderr) + return command + if image_params.push: + command = run_command( + [ + "docker", + "push", + image_params.airflow_image_name + ":latest", + ], + output=output, + capture_output=True, + check=False, + ) return command