This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0a985f70b4 Restore "iffy" solution for skipping tasks for in-workflow
build (#38077)
0a985f70b4 is described below
commit 0a985f70b4fbe01f56f6bd76922d9998d1a74d0e
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Mar 12 16:00:39 2024 +0100
Restore "iffy" solution for skipping tasks for in-workflow build (#38077)
GitHub actions has strange and complex way of getting conditional
jobs. When you want a job to depend on another job to be running
when it is either successful of fully skipped, you have to write
a rather peculiar condition:
```
always() && !failure() && !cancelled()
```
However, it turned out that this condition does not propoagate to
to task that depends on the downstream tasks - for example if
"wait-for-ci-images" had this condition and preceding "build-ci-images"
was skipped, the "wait-for-ci-images" was happily running, but
any of the other jobs that depended on "wait-for-ci-images" has
been skipped :(. Simply - skipping tasks immediately propagates
to all the downstream tasks and we cannot easily mitigate it.
This means that we have to come back to a solution that
was implemented before #38057 - where instead of skipping the whole
job, we skip all the steps in this kob. That makes it a little more
clunky but makes the skipping status not propagate to downstream
jobs.
---
.github/workflows/ci-image-build.yml | 25 +++++++++++++++++++------
.github/workflows/ci.yml | 26 ++++++++++----------------
.github/workflows/prod-image-build.yml | 33 +++++++++++++++++++++++++--------
3 files changed, 54 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/ci-image-build.yml
b/.github/workflows/ci-image-build.yml
index 6820468a04..cfb3154dab 100644
--- a/.github/workflows/ci-image-build.yml
+++ b/.github/workflows/ci-image-build.yml
@@ -25,6 +25,13 @@ on: # yamllint disable-line rule:truthy
required: false
default: '["ubuntu-22.04"]'
type: string
+ do-build:
+ description: >
+ Whether to actually do the build (true/false). If set to false, the
build is done
+ already in pull-request-target workflow, so we skip it here.
+ required: false
+ default: "true"
+ type: string
platform:
description: >
Name of the platform for the build - 'amd64/arm64'
@@ -82,7 +89,9 @@ jobs:
matrix:
python-version: ${{fromJson(inputs.python-versions)}}
timeout-minutes: 110
- name: Build CI ${{inputs.build-type}} image
${{matrix.python-version}}:${{inputs.image-tag}}
+ name: >
+ ${{ inputs.do-build == 'true' && 'Build' || 'Skip build' }}
+ CI ${{inputs.build-type}} image
${{matrix.python-version}}:${{inputs.image-tag}}
runs-on: ${{fromJson(inputs.runs-on)}}
env:
BACKEND: sqlite
@@ -92,23 +101,26 @@ jobs:
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ if: inputs.do-build == 'true'
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
+ if: inputs.do-build == 'true'
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
python-version: ${{ inputs.breeze-python-version }}
+ if: inputs.do-build == 'true'
- name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
shell: bash
run: |
pip install rich>=12.4.4 pyyaml
python
scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
- if: inputs.upgrade-to-newer-dependencies != 'false'
+ if: inputs.skip-build != 'true' &&
inputs.upgrade-to-newer-dependencies != 'false'
- name: "Start ARM instance"
run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh
- if: inputs.platform == 'arm64'
+ if: inputs.do-build == 'true' && inputs.platform == 'arm64'
- name: "Build & Push ${{ inputs.platform }}:${{ inputs.image-tag }} ${{
matrix.python-version }}"
shell: bash
run: >
@@ -122,15 +134,16 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
USE_UV: ${{ inputs.use-uv }}
BUILDER: ${{ inputs.platform == 'amd64' && 'default' ||
'airflow_cache' }}
+ if: inputs.do-build == 'true'
- name: "Stop ARM instance"
run: ./scripts/ci/images/ci_stop_arm_instance.sh
- if: always() && inputs.platform == 'arm64'
+ if: always() && inputs.do-build == 'true' && inputs.platform == 'arm64'
- name: "Source constraints: ${{ matrix.python-version }}"
shell: bash
run: >
breeze release-management generate-constraints --python "${{
matrix.python-version }}"
--airflow-constraints-mode constraints-source-providers --answer yes
- if: inputs.upload-constraints == 'true'
+ if: inputs.do-build == 'true' && inputs.upload-constraints == 'true'
- name: "Upload constraint artifacts"
uses: actions/upload-artifact@v4
with:
@@ -138,4 +151,4 @@ jobs:
path: ./files/constraints-*/constraints-source-providers-*.txt
retention-days: 7
if-no-files-found: error
- if: inputs.upload-constraints == 'true'
+ if: inputs.do-build == 'true' && inputs.upload-constraints == 'true'
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5b0cebe6df..3a47f5f29b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -271,7 +271,9 @@ jobs:
run: breeze shell --max-time 120
build-ci-images:
- name: Build CI images (in-workflow)
+ name: >
+ ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' ||
'Skip building' }}
+ CI images in-workflow
needs: [build-info]
uses: ./.github/workflows/ci-image-build.yml
permissions:
@@ -283,6 +285,7 @@ jobs:
packages: write
secrets: inherit
with:
+ do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
upload-constraints: "true"
@@ -292,7 +295,6 @@ jobs:
breeze-python-version: ${{
needs.build-info.outputs.breeze-python-version }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- if: needs.build-info.outputs.in-workflow-build == 'true'
run-breeze-tests:
timeout-minutes: 10
@@ -438,12 +440,7 @@ jobs:
name: "Wait for CI images"
runs-on: ["ubuntu-22.04"]
needs: [build-info, build-ci-images]
- # This strange condition below is equivalent to: "all success or skipped"
- # The "build-ci-images" step might be skipped, in case production building
happens in the
- # "build-images" workflow, and in this case we still want to wait for CI
images and run dependent jobs
- if: >
- always() && !failure() && !cancelled() &&
- needs.build-info.outputs.ci-image-build == 'true'
+ if: needs.build-info.outputs.ci-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
@@ -1858,7 +1855,9 @@ jobs:
echo Total number of unique warnings $(cat
./artifacts/test-warnings*/* | sort | uniq | wc -l)
build-prod-images:
- name: Build PROD images (in-workflow)
+ name: >
+ ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' ||
'Skip building' }}
+ PROD images in-workflow
needs: [build-info, build-ci-images]
uses: ./.github/workflows/prod-image-build.yml
permissions:
@@ -1871,6 +1870,7 @@ jobs:
secrets: inherit
with:
build-type: "Regular"
+ do-build: ${{ needs.build-info.outputs.in-workflow-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
python-versions: ${{ needs.build-info.outputs.python-versions }}
branch: ${{ needs.build-info.outputs.default-branch }}
@@ -1882,7 +1882,6 @@ jobs:
chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- if: needs.build-info.outputs.in-workflow-build == 'true'
prod-image-extra-checks-main:
name: PROD image extra checks (main)
@@ -1923,12 +1922,7 @@ jobs:
name: "Wait for PROD images"
runs-on: ["ubuntu-22.04"]
needs: [build-info, wait-for-ci-images, build-prod-images]
- # This strange condition below is equivalent to: "all success or skipped"
- # The "build-prod-images" step might be skipped, in case production
building happens in the
- # "build-images" workflow, and in this case we still want to wait for PROD
images and run dependent tests
- if: >
- always() && !failure() && !cancelled() &&
- needs.build-info.outputs.prod-image-build == 'true'
+ if: needs.build-info.outputs.prod-image-build == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
BACKEND: sqlite
diff --git a/.github/workflows/prod-image-build.yml
b/.github/workflows/prod-image-build.yml
index 3c5b396764..6a87daaa0a 100644
--- a/.github/workflows/prod-image-build.yml
+++ b/.github/workflows/prod-image-build.yml
@@ -26,6 +26,13 @@ on: # yamllint disable-line rule:truthy
variations.
required: true
type: string
+ do-build:
+ description: >
+ Whether to actually do the build (true/false). If set to false, the
build is done
+ already in pull-request-target workflow, so we skip it here.
+ required: false
+ default: "true"
+ type: string
debian-version:
description: "Base Debian distribution to use for the build
(bookworm/bullseye)"
type: string
@@ -86,7 +93,9 @@ jobs:
matrix:
python-version: ${{fromJson(inputs.python-versions)}}
timeout-minutes: 80
- name: Build PROD ${{inputs.build-type}} image
${{matrix.python-version}}:${{inputs.image-tag}}
+ name: >
+ ${{ inputs.do-build == 'true' && 'Build' || 'Skip build' }}
+ PROD ${{inputs.build-type}} image
${{matrix.python-version}}:${{inputs.image-tag}}
runs-on: ["ubuntu-22.04"]
env:
BACKEND: sqlite
@@ -96,30 +105,34 @@ jobs:
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ if: inputs.do-build == 'true'
- uses: actions/checkout@v4
with:
ref: ${{ needs.build-info.outputs.targetCommitSha }}
persist-credentials: false
+ if: inputs.do-build == 'true'
- name: "Install Breeze"
uses: ./.github/actions/breeze
with:
python-version: ${{ inputs.breeze-python-version }}
+ if: inputs.do-build == 'true'
- name: "Regenerate dependencies in case they was modified manually so
that we can build an image"
shell: bash
run: |
pip install rich>=12.4.4 pyyaml
python
scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
- if: inputs.upgrade-to-newer-dependencies != 'false'
+ if: inputs.do-build == 'true' && inputs.upgrade-to-newer-dependencies
!= 'false'
- name: "Cleanup dist and context file"
shell: bash
run: rm -fv ./dist/* ./docker-context-files/*
+ if: inputs.do-build == 'true'
- name: "Prepare providers packages"
shell: bash
run: >
breeze release-management prepare-provider-packages
--package-list-file ./prod_image_installed_providers.txt
--package-format wheel --version-suffix-for-pypi dev0
- if: ${{ inputs.build-provider-packages == 'true' }}
+ if: inputs.do-build == 'true' && inputs.build-provider-packages ==
'true'
- name: "Prepare chicken-eggs provider packages"
# In case of provider packages which use latest dev0 version of
providers, we should prepare them
# from the source code, not from the PyPI because they have
apache-airflow>=X.Y.Z dependency
@@ -128,27 +141,31 @@ jobs:
run: >
breeze release-management prepare-provider-packages
--package-format wheel --version-suffix-for-pypi dev0 ${{
inputs.chicken-egg-providers }}
- if: ${{ inputs.build-provider-packages != 'true' &&
inputs.chicken-egg-providers != '' }}
+ if: >
+ inputs.do-build == 'true' && inputs.build-provider-packages !=
'true' &&
+ inputs.chicken-egg-providers != ''
- name: "Prepare airflow package"
shell: bash
run: >
breeze release-management prepare-airflow-package
--package-format wheel --version-suffix-for-pypi dev0
+ if: inputs.do-build == 'true'
- name: "Copy dist packages to docker-context files"
shell: bash
run: cp -v --no-preserve=mode,ownership ./dist/*.whl
./docker-context-files
+ if: inputs.do-build == 'true'
- name: "Download constraints from the CI build"
uses: actions/download-artifact@v4
with:
name: source-constraints-${{ matrix.python-version }}
path: ./docker-context-files
- if: ${{ inputs.build-provider-packages == 'true' }}
+ if: inputs.do-build == 'true' && inputs.build-provider-packages ==
'true'
- name: "Download constraints from the Generate & Verify build"
uses: actions/download-artifact@v4
with:
name: constraints
path: ./docker-context-files
- if: ${{ inputs.build-provider-packages != 'true' }}
+ if: inputs.do-build == 'true' && inputs.build-provider-packages !=
'true'
- name: "Build PROD images w/ source providers ${{ matrix.python-version
}}:${{ inputs.image-tag }}"
shell: bash
run: >
@@ -165,7 +182,7 @@ jobs:
INCLUDE_NOT_READY_PROVIDERS: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
USE_UV: ${{ inputs.use-uv }}
- if: ${{ inputs.build-provider-packages == 'true' }}
+ if: inputs.do-build == 'true' && inputs.build-provider-packages ==
'true'
- name: "Build PROD images with PyPi providers ${{ matrix.python-version
}}:${{ inputs.image-tag }}"
shell: bash
run: >
@@ -182,4 +199,4 @@ jobs:
INCLUDE_NOT_READY_PROVIDERS: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
USE_UV: ${{ inputs.use-uv }}
- if: ${{ inputs.build-provider-packages != 'true' }}
+ if: inputs.do-build == 'true' && inputs.build-provider-packages !=
'true'
