This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-1-test by this push:
new 20e4ace8926 [v3-1-test] Synchronize and fix ARM CI workflows (#56856)
(#56858)
20e4ace8926 is described below
commit 20e4ace8926c99890308815a2dc5e664c353d133
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sun Oct 19 22:11:07 2025 +0200
[v3-1-test] Synchronize and fix ARM CI workflows (#56856) (#56858)
There were a few issues with ARM workflows:
* not all jobs were run in ARM tests - we do not want to run mysql
of course, but other tests should be fine to run on ARM
* some conditions were not updated (we have to somehow duplicate
amd and arm job definition because we run out of composite
workflows - so sometimes conditions are not synced)
* most importantly - we uploaded prek cache in build-info job, but
that job only run on AMD, not on ARM so the ARM cache was really
an AMD one (and it caused unterminated strings in doctoc installation
It's not possible to upload same artifact twice in the same job and
since we use prek in several jobs we should make sure that the cache
is only uploaded once per job. This was the reason why it was initially
uploaded in build-info job (and save-cache was set to false elsewhere).
With this PR, we have save-cache in 3 places:
* basic checks
* static CI-image bound checks
* in octopin (Python 3.11)
Basic checks and static checks are mutually exclusive (controlled by
basic-checks-only flag) - so we can safely upload cache in both.
In all other places we only install prek with cache, but we do not
save the cache as artifact.
(cherry picked from commit 59089cda66691985cce66200216adb0ebb8cc720)
Co-authored-by: Jarek Potiuk <[email protected]>
---
.github/actions/install-prek/action.yml | 4 +-
.github/workflows/basic-tests.yml | 2 +-
.github/workflows/ci-amd.yml | 8 -
.github/workflows/ci-arm.yml | 274 ++++++++++++++++++++++++++++++--
.github/workflows/ci-image-checks.yml | 2 +-
scripts/ci/prek/check_license.py | 2 -
6 files changed, 264 insertions(+), 28 deletions(-)
diff --git a/.github/actions/install-prek/action.yml
b/.github/actions/install-prek/action.yml
index 498522e3a2f..10f80d14959 100644
--- a/.github/actions/install-prek/action.yml
+++ b/.github/actions/install-prek/action.yml
@@ -64,7 +64,7 @@ runs:
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
# yamllint disable rule:line-length
- key: cache-prek-v8-${{ inputs.platform }}-python${{
inputs.python-version }}-uv${{ inputs.uv-version }}-${{
hashFiles('**/.pre-commit-config.yaml') }}
+ key: cache-prek-v9-${{ inputs.platform }}-python${{
inputs.python-version }}-uv${{ inputs.uv-version }}-${{
hashFiles('**/.pre-commit-config.yaml') }}
path: /tmp/
id: restore-prek-cache
- name: "Check if prek cache tarball exists"
@@ -112,7 +112,7 @@ runs:
uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
# yamllint disable rule:line-length
- key: cache-prek-v8-${{ inputs.platform }}-python${{
inputs.python-version }}-uv${{ inputs.uv-version }}-${{
hashFiles('**/.pre-commit-config.yaml') }}
+ key: cache-prek-v9-${{ inputs.platform }}-python${{
inputs.python-version }}-uv${{ inputs.uv-version }}-${{
hashFiles('**/.pre-commit-config.yaml') }}
path: /tmp/cache-prek.tar.gz
if-no-files-found: 'error'
retention-days: '2'
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index a523837a19a..1fafc6e5937 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -229,7 +229,7 @@ jobs:
with:
python-version: ${{ steps.breeze.outputs.host-python-version }}
platform: ${{ inputs.platform }}
- save-cache: false
+ save-cache: true
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
with:
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index be234644bc9..c5bff2395b0 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -165,13 +165,6 @@ jobs:
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
- - name: "Install and cache prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{ steps.breeze.outputs.host-python-version }}
- platform: "linux/amd64"
- save-cache: true
run-pin-versions-hook:
name: "Run pin-versions hook"
needs: [build-info]
@@ -881,7 +874,6 @@ jobs:
- tests-non-db-providers
- tests-postgres-core
- tests-postgres-providers
- # - tests-special
- tests-sqlite-core
- tests-sqlite-providers
- tests-task-sdk
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index 6fb8a1bd537..a5450489141 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -117,7 +117,8 @@ jobs:
shared-distributions-as-json: ${{
steps.selective-checks.outputs.shared-distributions-as-json }}
skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }}
skip-providers-tests: ${{
steps.selective-checks.outputs.skip-providers-tests }}
- source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
+ source-head-repo: ${{ steps.source-run-info.outputs.head-repo }}
+ source-head-ref: ${{ steps.source-run-info.outputs.head-ref }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
testable-core-integrations: ${{
steps.selective-checks.outputs.testable-core-integrations }}
testable-providers-integrations: ${{
steps.selective-checks.outputs.testable-providers-integrations }}
@@ -157,13 +158,7 @@ jobs:
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
- - name: "Install and cache prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{ steps.breeze.outputs.host-python-version }}
- platform: "linux/arm64"
- save-cache: true
+
basic-tests:
name: "Basic tests"
needs: [build-info]
@@ -248,6 +243,41 @@ jobs:
debug-resources: ${{ needs.build-info.outputs.debug-resources }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
+ ci-image-checks:
+ name: "CI image checks"
+ needs: [build-info, build-ci-images]
+ uses: ./.github/workflows/ci-image-checks.yml
+ permissions:
+ id-token: write
+ contents: read
+ with:
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ run-mypy: ${{ needs.build-info.outputs.run-mypy }}
+ mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
+ python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
+ branch: ${{ needs.build-info.outputs.default-branch }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
+ docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
+ latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only
}}
+ basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+ upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
+ ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+ docs-build: ${{ needs.build-info.outputs.docs-build }}
+ run-api-codegen: ${{ needs.build-info.outputs.run-api-codegen }}
+ default-postgres-version: ${{
needs.build-info.outputs.default-postgres-version }}
+ run-coverage: ${{ needs.build-info.outputs.run-coverage }}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ source-head-repo: ${{ needs.build-info.outputs.source-head-repo }}
+ source-head-ref: ${{ needs.build-info.outputs.source-head-ref }}
+ secrets:
+ DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+ DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+
providers:
name: "provider distributions tests"
uses: ./.github/workflows/test-providers.yml
@@ -257,7 +287,8 @@ jobs:
packages: read
if: >
needs.build-info.outputs.skip-providers-tests != 'true' &&
- needs.build-info.outputs.latest-versions-only != 'true'
+ needs.build-info.outputs.latest-versions-only != 'true' &&
+ needs.build-info.outputs.run-unit-tests == 'true'
with:
runners: ${{ needs.build-info.outputs.arm-runners }}
platform: "linux/arm64"
@@ -412,7 +443,6 @@ jobs:
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
-
tests-non-db-core:
name: "Non-DB tests: core"
uses: ./.github/workflows/run-unit-tests.yml
@@ -473,6 +503,99 @@ jobs:
default-branch: ${{ needs.build-info.outputs.default-branch }}
if: needs.build-info.outputs.run-unit-tests == 'true'
+ tests-special:
+ name: "Special tests"
+ uses: ./.github/workflows/special-tests.yml
+ needs: [build-info, build-ci-images]
+ permissions:
+ contents: read
+ packages: read
+ if: >
+ needs.build-info.outputs.run-unit-tests == 'true' &&
+ (needs.build-info.outputs.canary-run == 'true' ||
+ needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' ||
+ needs.build-info.outputs.full-tests-needed == 'true')
+ with:
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ core-test-types-list-as-strings-in-json: >
+ ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
+ providers-test-types-list-as-strings-in-json: >
+ ${{
needs.build-info.outputs.providers-test-types-list-as-strings-in-json }}
+ run-coverage: ${{ needs.build-info.outputs.run-coverage }}
+ default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ default-postgres-version: ${{
needs.build-info.outputs.default-postgres-version }}
+ excluded-providers-as-string: ${{
needs.build-info.outputs.excluded-providers-as-string }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
+ skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+
+ tests-with-lowest-direct-resolution-core:
+ name: "Low dep tests:core"
+ needs: [build-info, build-ci-images]
+ uses: ./.github/workflows/run-unit-tests.yml
+ permissions:
+ contents: read
+ packages: read
+ if: >
+ needs.build-info.outputs.run-unit-tests == 'true'
+ with:
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ test-name: "LowestDeps"
+ force-lowest-dependencies: "true"
+ test-scope: "All"
+ test-group: "core"
+ backend: "sqlite"
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ backend-versions: "['${{
needs.build-info.outputs.default-postgres-version }}']"
+ excluded-providers-as-string: ""
+ excludes: "[]"
+ test-types-as-strings-in-json: >
+ ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
+ run-coverage: ${{ needs.build-info.outputs.run-coverage }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+ monitor-delay-time-in-seconds: 120
+ skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+
+ tests-with-lowest-direct-resolution-providers:
+ name: "Low dep tests: providers"
+ needs: [build-info, build-ci-images]
+ uses: ./.github/workflows/run-unit-tests.yml
+ permissions:
+ contents: read
+ packages: read
+ if: needs.build-info.outputs.run-unit-tests == 'true'
+ with:
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ test-name: "LowestDeps"
+ force-lowest-dependencies: "true"
+ test-scope: "All"
+ test-group: "providers"
+ backend: "sqlite"
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ backend-versions: "['${{
needs.build-info.outputs.default-postgres-version }}']"
+ excluded-providers-as-string: ${{
needs.build-info.outputs.excluded-providers-as-string }}
+ excludes: "[]"
+ test-types-as-strings-in-json: >
+ ${{
needs.build-info.outputs.individual-providers-test-types-list-as-strings-in-json
}}
+ include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
+ run-coverage: ${{ needs.build-info.outputs.run-coverage }}
+ debug-resources: ${{ needs.build-info.outputs.debug-resources }}
+ monitor-delay-time-in-seconds: 120
+ skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+
build-prod-images:
name: Build PROD images
needs: [build-info, build-ci-images, generate-constraints]
@@ -499,6 +622,23 @@ jobs:
disable-airflow-repo-cache: ${{
needs.build-info.outputs.disable-airflow-repo-cache }}
prod-image-build: ${{ needs.build-info.outputs.prod-image-build }}
+ additional-prod-image-tests:
+ name: "Additional PROD image tests"
+ needs: [build-info, build-prod-images, generate-constraints]
+ uses: ./.github/workflows/additional-prod-image-tests.yml
+ with:
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ default-branch: ${{ needs.build-info.outputs.default-branch }}
+ constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
+ upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
+ docker-cache: ${{ needs.build-info.outputs.docker-cache }}
+ disable-airflow-repo-cache: ${{
needs.build-info.outputs.disable-airflow-repo-cache }}
+ default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ if: needs.build-info.outputs.prod-image-build == 'true'
+
tests-kubernetes:
name: "Kubernetes tests"
uses: ./.github/workflows/k8s-tests.yml
@@ -518,9 +658,30 @@ jobs:
( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
needs.build-info.outputs.run-helm-tests == 'true')
+ tests-task-sdk:
+ name: "Task SDK tests"
+ uses: ./.github/workflows/airflow-distributions-tests.yml
+ needs: [build-info, build-ci-images]
+ permissions:
+ contents: read
+ packages: read
+ with:
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ distribution-name: "task-sdk"
+ distribution-cmd-format: "prepare-task-sdk-distributions"
+ test-type: "task-sdk-tests"
+ use-local-venv: 'false'
+ test-timeout: 20
+ if: needs.build-info.outputs.run-task-sdk-tests == 'true'
+
tests-go-sdk:
name: "Go SDK tests"
- needs: [build-info, build-ci-images]
+ needs: [build-info]
runs-on: ${{ fromJSON(needs.build-info.outputs.arm-runners) }}
timeout-minutes: 15
permissions:
@@ -543,11 +704,38 @@ jobs:
with:
go-version: 1.24
cache-dependency-path: go-sdk/go.sum
+ # keep this in sync with go.mod in go-sdk/
+ - name: Setup Gotestsum
+ shell: bash
+ run: |
+ go install
gotest.tools/gotestsum@c4a0df2e75a225d979a444342dd3db752b53619f # v1.13.0
+ gotestsum --version
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: Run Go tests
working-directory: ./go-sdk
- run: go test -v ./...
+ run: gotestsum --format github-actions ./...
+
+ tests-airflow-ctl:
+ name: "Airflow CTL tests"
+ uses: ./.github/workflows/airflow-distributions-tests.yml
+ needs: [build-info]
+ permissions:
+ contents: read
+ packages: read
+ with:
+ runners: ${{ needs.build-info.outputs.arm-runners }}
+ platform: "linux/arm64"
+ default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
+ python-versions: ${{ needs.build-info.outputs.python-versions }}
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ canary-run: ${{ needs.build-info.outputs.canary-run }}
+ distribution-name: "airflow-ctl"
+ distribution-cmd-format: "prepare-airflow-ctl-distributions"
+ test-type: "airflow-ctl-tests"
+ use-local-venv: 'true'
+ test-timeout: 20
+ if: needs.build-info.outputs.run-airflow-ctl-tests == 'true'
finalize-tests:
name: Finalize tests
@@ -558,11 +746,12 @@ jobs:
if: always() && !failure() && !cancelled()
needs:
- additional-ci-image-checks
+ - additional-prod-image-tests
- basic-tests
- build-info
- - basic-tests
- - generate-constraints
- build-prod-images
+ - ci-image-checks
+ - generate-constraints
- providers
- tests-helm
- tests-kubernetes
@@ -572,6 +761,11 @@ jobs:
- tests-postgres-providers
- tests-sqlite-core
- tests-sqlite-providers
+ - tests-task-sdk
+ - tests-airflow-ctl
+ - tests-go-sdk
+ - tests-with-lowest-direct-resolution-core
+ - tests-with-lowest-direct-resolution-providers
uses: ./.github/workflows/finalize-tests.yml
with:
runners: ${{ needs.build-info.outputs.arm-runners }}
@@ -612,3 +806,55 @@ jobs:
type: "mrkdwn"
text: "🚨🕒 Failure Alert: Scheduled CI (ARM) 🕒🚨\n\n*Details:*
<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id
}}|View the failure log>"
# yamllint enable rule:line-length
+
+ summarize-warnings:
+ timeout-minutes: 15
+ name: "Summarize warnings"
+ needs:
+ - build-info
+ - tests-non-db-core
+ - tests-non-db-providers
+ - tests-postgres-core
+ - tests-postgres-providers
+ - tests-sqlite-core
+ - tests-sqlite-providers
+ - tests-task-sdk
+ - tests-airflow-ctl
+ - tests-special
+ - tests-with-lowest-direct-resolution-core
+ - tests-with-lowest-direct-resolution-providers
+ runs-on: ${{ fromJSON(needs.build-info.outputs.arm-runners) }}
+ if: needs.build-info.outputs.run-unit-tests == 'true'
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
+ with:
+ persist-credentials: false
+ - name: "Free up disk space"
+ shell: bash
+ run: ./scripts/tools/free_up_disk_space.sh
+ - name: "Download all test warning artifacts from the current build"
+ uses:
actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ path: ./artifacts
+ pattern: test-warnings-*
+ - name: "Setup python"
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 #
v5.6.0
+ with:
+ python-version: "${{ inputs.default-python-version }}"
+ - name: "Summarize all warnings"
+ run: |
+ ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \
+ --pattern "**/warnings-*.txt" \
+ --output ./files
+ - name: "Upload artifact for summarized warnings"
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
# v4.6.2
+ with:
+ name: test-summarized-arm-runner-warnings
+ path: ./files/warn-summary-*.txt
+ retention-days: 7
+ if-no-files-found: ignore
+ overwrite: true
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index e7ed9d0de3e..6bd4d16d985 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -153,7 +153,7 @@ jobs:
with:
python-version: ${{steps.breeze.outputs.host-python-version}}
platform: ${{ inputs.platform }}
- save-cache: false
+ save-cache: true
- name: "Static checks"
run: prek --all-files --show-diff-on-failure --color always
env:
diff --git a/scripts/ci/prek/check_license.py b/scripts/ci/prek/check_license.py
index ffac66c5954..c60d6bc9fa9 100755
--- a/scripts/ci/prek/check_license.py
+++ b/scripts/ci/prek/check_license.py
@@ -37,8 +37,6 @@ cmd = [
"--user",
f"{os.getuid()}:{os.getgid()}",
"--rm",
- "--platform",
- "linux/amd64",
"ghcr.io/apache/airflow-apache-rat:0.16.1-2024.03.23@sha256:83c4d2610ec4a439d1809a67fadbdc9a1df089ab130b32209351bdd4527a3f02",
"-d",
"/opt/airflow",
