This is an automated email from the ASF dual-hosted git repository. potiuk pushed a commit to branch separate-release-image-workflows-per-python in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 66248c51ef54ace8d4b376619372e02ff0620487 Author: Jarek Potiuk <[email protected]> AuthorDate: Wed May 7 13:24:17 2025 +0200 Split release image into per-python independent matrix of workflows The release workflow now will run separately for each image - which means that if both AMD / ARM images of the same python version have finished, the merge step for that Python version will run immediately rather than waiting for all Python versions to complete. This means that some images might be available a bit faster and that even if a single image releaase will fail for some reason, the other images will appear before we re-run that failed image job. It also adds the possibility of overriding the python version images - we can now additionally filter which image versions should be run. --- .github/workflows/release_dockerhub_image.yml | 238 ++++----------------- ...mage.yml => release_single_dockerhub_image.yml} | 129 ++++------- 2 files changed, 83 insertions(+), 284 deletions(-) diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 9d8655ecaea..ab7834f08f3 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -25,13 +25,21 @@ on: # yamllint disable-line rule:truthy required: true amdOnly: type: boolean - description: 'Limit to amd64 only (faster testing)' + description: 'Limit to amd64 images' default: false + skipLatest: + type: boolean + description: 'Force disabling auto-latest-tag' + default: false + limitPythonVersions: + type: string + description: 'Force python versions (e.g. "3.9 3.10")' + default: '' permissions: contents: read packages: read concurrency: - group: ${{ github.event.inputs.airflowVersion }} + group: ${{ inputs.airflowVersion }} cancel-in-progress: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -42,19 +50,18 @@ jobs: name: "Build Info" runs-on: ["ubuntu-24.04"] outputs: - pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} + pythonVersions: ${{ steps.determine-python-versions.outputs.python-versions }} platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }} - skipLatest: ${{ steps.selective-checks.outputs.skipLatest }} amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} env: GITHUB_CONTEXT: ${{ toJson(github) }} VERBOSE: true - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - AMD_ONLY: ${{ github.event.inputs.amdOnly }} + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + AMD_ONLY: ${{ inputs.amdOnly }} + LIMIT_PYTHON_VERSIONS: ${{ inputs.limitPythonVersions }} + SKIP_LATEST: ${{ inputs.skipLatest }} if: contains(fromJSON('[ "ashb", "eladkal", @@ -73,6 +80,8 @@ jobs: echo "=========================" echo "Airflow version: '${AIRFLOW_VERSION}'" echo "AMD only: '${AMD_ONLY}'" + echo "Skip latest: '${SKIP_LATEST}'" + echo "Limit python versions: '${LIMIT_PYTHON_VERSIONS}'" - name: "Cleanup repo" shell: bash run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -106,195 +115,40 @@ jobs: else echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >> "${GITHUB_OUTPUT}" fi - - build-images: - timeout-minutes: 50 - # yamllint disable rule:line-length - name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }}" - runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(needs.build-info.outputs.amd-runners) || fromJSON(needs.build-info.outputs.arm-runners) }} - needs: [build-info] - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} - platform: ${{ fromJSON(needs.build-info.outputs.platformMatrix) }} - env: - AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - PLATFORM: ${{ matrix.platform }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "false" - - name: Free space - run: breeze ci free-space --answer yes - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Login to hub.docker.com" - run: > - echo ${{ secrets.DOCKERHUB_TOKEN }} | - docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - - name: "Get env vars for metadata" + - name: "Override python versions if specified" shell: bash - run: | - echo "ARTIFACT_NAME=metadata-${PYTHON_MAJOR_MINOR_VERSION}-${PLATFORM/\//_}" >> "${GITHUB_ENV}" - echo "MANIFEST_FILE_NAME=metadata-${AIRFLOW_VERSION}-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" - echo "MANIFEST_SLIM_FILE_NAME=metadata-${AIRFLOW_VERSION}-slim-${PLATFORM/\//_}-${PYTHON_MAJOR_MINOR_VERSION}.json" >> "${GITHUB_ENV}" - - name: Login to ghcr.io + id: determine-python-versions env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - - name: "Install buildx plugin" - # yamllint disable rule:line-length + ALL_PYTHON_VERSIONS: ${{ steps.selective-checks.outputs.all-python-versions }} run: | - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc + # override python versions if specified + if [[ "${LIMIT_PYTHON_VERSIONS}" != "" ]]; then + # join python versions with commas + IFS=' ' read -r -a pythonVersions <<< "${LIMIT_PYTHON_VERSIONS}" + pythonVersions=("${pythonVersions[@]// /\",\"}") + echo 'python-versions=["'"${pythonVersions}"'"]' >> "${GITHUB_OUTPUT}" + else + echo 'python-versions=${ALL_PYTHON_VERSIONS}' >> "${GITHUB_OUTPUT}" + fi - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt install docker-buildx-plugin - - name: "Create airflow_cache builder" - run: docker buildx create --name airflow_cache --driver docker-container - - name: > - Build regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} - --metadata-folder dist - - name: > - Verify regular image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze prod-image verify --pull --manifest-file dist/${MANIFEST_FILE_NAME} - - name: > - Release slim images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze release-management release-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} --slim-images - --metadata-folder dist - - name: > - Verify slim image: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }} - run: > - breeze prod-image verify --pull --slim-image --manifest-file dist/${MANIFEST_SLIM_FILE_NAME} - - name: "List upload-able artifacts" - shell: bash - run: find ./dist -name '*.json' - - name: "Upload metadata artifact ${{ env.ARTIFACT_NAME }}" - uses: actions/upload-artifact@v4 - with: - name: ${{ env.ARTIFACT_NAME }} - path: ./dist/metadata-* - retention-days: 7 - if-no-files-found: error - - name: "Docker logout" - run: docker logout - if: always() - merge-images: - timeout-minutes: 5 - name: "Merge: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - runs-on: ["ubuntu-22.04"] - needs: [build-info, build-images] + release-images: + name: "Release images" + needs: [build-info] strategy: fail-fast: false matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} - env: - AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} - COMMIT_SHA: ${{ github.sha }} - REPOSITORY: ${{ github.repository }} - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "false" - - name: Free space - run: breeze ci free-space --answer yes - - name: "Cleanup dist and context file" - run: rm -fv ./dist/* ./docker-context-files/* - - name: "Login to hub.docker.com" - run: > - echo ${{ secrets.DOCKERHUB_TOKEN }} | - docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} - - name: Login to ghcr.io - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ACTOR: ${{ github.actor }} - run: echo "${GITHUB_TOKEN}" | docker login ghcr.io -u ${ACTOR} --password-stdin - - name: "Download metadata artifacts" - uses: actions/download-artifact@v4 - with: - path: ./dist - pattern: metadata-${{ matrix.python-version }}-* - - name: "List downloaded artifacts" - shell: bash - run: find ./dist -name '*.json' - - name: "Install buildx plugin" - # yamllint disable rule:line-length - run: | - sudo apt-get update - sudo apt-get install ca-certificates curl - sudo install -m 0755 -d /etc/apt/keyrings - sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - sudo chmod a+r /etc/apt/keyrings/docker.asc - - # Add the repository to Apt sources: - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt-get update - sudo apt install docker-buildx-plugin - - name: "Install regctl" - # yamllint disable rule:line-length - run: | - mkdir -p ~/bin - curl -L https://github.com/regclient/regclient/releases/latest/download/regctl-linux-amd64 >${HOME}/bin/regctl - chmod 755 ${HOME}/bin/regctl - echo "${HOME}/bin" >>${GITHUB_PATH} - - name: "Merge regular images ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - run: > - breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist - - name: "Merge slim images ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" - run: > - breeze release-management merge-prod-images --dockerhub-repo "${REPOSITORY}" - --airflow-version "${AIRFLOW_VERSION}" ${SKIP_LATEST} - --python ${PYTHON_MAJOR_MINOR_VERSION} --metadata-folder dist --slim-images - - name: "Docker logout" - run: docker logout - if: always() + python: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} + uses: ./.github/workflows/release_single_dockerhub_image.yml + secrets: + DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + permissions: + contents: read + with: + pythonVersion: ${{ matrix.python }} + airflowVersion: ${{ needs.build-info.outputs.airflowVersion }} + platformMatrix: ${{ needs.build-info.outputs.platformMatrix }} + armRunners: ${{ needs.build-info.outputs.arm-runners }} + amdRunners: ${{ needs.build-info.outputs.amd-runners }} + skipLatest: ${{ inputs.skipLatest }} diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml similarity index 71% copy from .github/workflows/release_dockerhub_image.yml copy to .github/workflows/release_single_dockerhub_image.yml index 9d8655ecaea..6c44fe47abc 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -16,20 +16,41 @@ # under the License. # --- -name: "Release PROD images" +name: "Release single PROD image" on: # yamllint disable-line rule:truthy - workflow_dispatch: + workflow_call: inputs: airflowVersion: description: 'Airflow version (e.g. 3.0.1, 3.0.1rc1, 3.0.1b1)' + type: string + required: true + platformMatrix: + description: 'Platform matrix formatted as json (e.g. ["linux/amd64", "linux/arm64"])' + type: string + required: true + pythonVersion: + description: 'Python version (e.g. 3.8, 3.9, 3.10, 3.11)' + type: string + required: true + skipLatest: + description: "Skip tagging latest release (true/false)" + type: string + required: true + amdRunners: + description: "Amd64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" + type: string + required: true + armRunners: + description: "Arm64 runners (e.g. [\"ubuntu-22.04\", \"ubuntu-24.04\"])" + type: string + required: true + secrets: + DOCKERHUB_USER: + required: true + DOCKERHUB_TOKEN: required: true - amdOnly: - type: boolean - description: 'Limit to amd64 only (faster testing)' - default: false permissions: contents: read - packages: read concurrency: group: ${{ github.event.inputs.airflowVersion }} cancel-in-progress: true @@ -37,92 +58,20 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VERBOSE: true jobs: - build-info: - timeout-minutes: 10 - name: "Build Info" - runs-on: ["ubuntu-24.04"] - outputs: - pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} - allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} - defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} - platformMatrix: ${{ steps.determine-matrix.outputs.platformMatrix }} - airflowVersion: ${{ steps.check-airflow-version.outputs.airflowVersion }} - skipLatest: ${{ steps.selective-checks.outputs.skipLatest }} - amd-runners: ${{ steps.selective-checks.outputs.amd-runners }} - arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - VERBOSE: true - AIRFLOW_VERSION: ${{ github.event.inputs.airflowVersion }} - AMD_ONLY: ${{ github.event.inputs.amdOnly }} - if: contains(fromJSON('[ - "ashb", - "eladkal", - "ephraimbuddy", - "jedcunningham", - "kaxil", - "pierrejeambrun", - "potiuk", - "utkarsharma2" - ]'), github.event.sender.login) - steps: - - name: "Input parameters summary" - shell: bash - run: | - echo "Input parameters summary" - echo "=========================" - echo "Airflow version: '${AIRFLOW_VERSION}'" - echo "AMD only: '${AMD_ONLY}'" - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: "Cleanup docker" - run: ./scripts/ci/cleanup_docker.sh - - name: "Install uv" - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - name: "Check airflow version" - id: check-airflow-version - shell: bash - run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}" - - name: "Install Breeze" - uses: ./.github/actions/breeze - with: - use-uv: "true" - - name: Selective checks - id: selective-checks - env: - VERBOSE: "false" - run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} - - name: "Determine build matrix" - shell: bash - id: determine-matrix - run: | - if [[ "${AMD_ONLY}" = "true" ]]; then - echo 'platformMatrix=["linux/amd64"]' >> "${GITHUB_OUTPUT}" - else - echo 'platformMatrix=["linux/amd64", "linux/arm64"]' >> "${GITHUB_OUTPUT}" - fi - build-images: timeout-minutes: 50 # yamllint disable rule:line-length name: "Build: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}, ${{ matrix.platform }}" - runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(needs.build-info.outputs.amd-runners) || fromJSON(needs.build-info.outputs.arm-runners) }} - needs: [build-info] + runs-on: ${{ (matrix.platform == 'linux/amd64') && fromJSON(inputs.amdRunners) || fromJSON(inputs.armRunners) }} strategy: fail-fast: false matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} - platform: ${{ fromJSON(needs.build-info.outputs.platformMatrix) }} + platform: ${{ fromJSON(inputs.platformMatrix) }} env: - AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.pythonVersion }} PLATFORM: ${{ matrix.platform }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} + SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} COMMIT_SHA: ${{ github.sha }} REPOSITORY: ${{ github.repository }} steps: @@ -216,15 +165,11 @@ jobs: timeout-minutes: 5 name: "Merge: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}" runs-on: ["ubuntu-22.04"] - needs: [build-info, build-images] - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(needs.build-info.outputs.pythonVersions) }} + needs: [build-images] env: - AIRFLOW_VERSION: ${{ needs.build-info.outputs.airflowVersion }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - SKIP_LATEST: ${{ needs.build-info.outputs.skipLatest == 'true' && '--skip-latest' || '' }} + AIRFLOW_VERSION: ${{ inputs.airflowVersion }} + PYTHON_MAJOR_MINOR_VERSION: ${{ inputs.python-version }} + SKIP_LATEST: ${{ inputs.skipLatest == 'true' && '--skip-latest' || '' }} COMMIT_SHA: ${{ github.sha }} REPOSITORY: ${{ github.repository }} steps:
