This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch test-cache-refreshing
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 53f8aa6ccd2ce3ad38d79eccc11bffba822ea68e
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sat Mar 23 13:34:41 2024 +0100
Test cache refreshing in CI
---
.github/workflows/ci.yml | 597 +++--------------------------------------------
1 file changed, 27 insertions(+), 570 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 37b6dd3c7d..f4567c8dae 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -27,593 +27,50 @@ on: # yamllint disable-line rule:truthy
permissions:
# All other permissions are set to none
contents: read
- # Technically read access while waiting for images should be more than
enough. However,
- # there is a bug in GitHub Actions/Packages and in case private repositories
are used, you get a permission
- # denied error when attempting to just pull private image, changing the
token permission to write solves the
- # issue. This is not dangerous, because if it is for "apache/airflow", only
maintainers can push ci.yml
- # changes. If it is for a fork, then the token is read-only anyway.
packages: write
-env:
- DB_RESET: "true"
- VERBOSE: "true"
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_USERNAME: ${{ github.actor }}
- # You can override CONSTRAINTS_GITHUB_REPOSITORY by setting secret in your
repo but by default the
- # Airflow one is going to be used
- CONSTRAINTS_GITHUB_REPOSITORY: >-
- ${{ secrets.CONSTRAINTS_GITHUB_REPOSITORY != '' &&
- secrets.CONSTRAINTS_GITHUB_REPOSITORY || 'apache/airflow' }}
- # In builds from forks, this token is read-only. For scheduled/direct push
it is WRITE one
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}"
- USE_SUDO: "true"
- INCLUDE_NOT_READY_PROVIDERS: "true"
- AIRFLOW_ENABLE_AIP_44: "true"
- MOUNT_SOURCES: "skip"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
-
- build-info:
- name: "Build info"
- # The runs-on cannot refer to env. or secrets. context, so we have no
- # option but to specify a hard-coded list here. This is "safe", the
- # runner checks if the user is an owner or collaborator of the repo
- # before running the workflow.
- runs-on: ["ubuntu-22.04"]
- env:
- GITHUB_CONTEXT: ${{ toJson(github) }}
- outputs:
- image-tag: ${{ github.event.pull_request.head.sha || github.sha }}
- cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
- affected-providers-list-as-string: >-
- ${{ steps.selective-checks.outputs.affected-providers-list-as-string }}
- upgrade-to-newer-dependencies: ${{
steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
- python-versions: ${{ steps.selective-checks.outputs.python-versions }}
- python-versions-list-as-string: ${{
steps.selective-checks.outputs.python-versions-list-as-string }}
- all-python-versions-list-as-string: >-
- ${{ steps.selective-checks.outputs.all-python-versions-list-as-string
}}
- default-python-version: ${{
steps.selective-checks.outputs.default-python-version }}
- kubernetes-versions-list-as-string: >-
- ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string
}}
- kubernetes-combos-list-as-string: >-
- ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
- default-kubernetes-version: ${{
steps.selective-checks.outputs.default-kubernetes-version }}
- postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions
}}
- default-postgres-version: ${{
steps.selective-checks.outputs.default-postgres-version }}
- mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
- default-mysql-version: ${{
steps.selective-checks.outputs.default-mysql-version }}
- default-helm-version: ${{
steps.selective-checks.outputs.default-helm-version }}
- default-kind-version: ${{
steps.selective-checks.outputs.default-kind-version }}
- full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed
}}
- parallel-test-types-list-as-string: >-
- ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string
}}
- include-success-outputs: ${{
steps.selective-checks.outputs.include-success-outputs }}
- postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
- mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
- sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
- skip-provider-tests: ${{
steps.selective-checks.outputs.skip-provider-tests }}
- run-tests: ${{ steps.selective-checks.outputs.run-tests }}
- run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
- run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }}
- run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
- basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only
}}
- ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
- prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
- docs-build: ${{ steps.selective-checks.outputs.docs-build }}
- mypy-folders: ${{ steps.selective-checks.outputs.mypy-folders }}
- needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
- needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
- needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
- needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen
}}
- default-branch: ${{ steps.selective-checks.outputs.default-branch }}
- default-constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
- docs-list-as-string: ${{
steps.selective-checks.outputs.docs-list-as-string }}
- skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
- providers-compatibility-checks: ${{
steps.selective-checks.outputs.providers-compatibility-checks }}
- helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
- debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
- runs-on: ${{steps.selective-checks.outputs.runs-on}}
- is-self-hosted-runner: ${{
steps.selective-checks.outputs.is-self-hosted-runner }}
- is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner
}}
- is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
- is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
- is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
- is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
- latest-versions-only: ${{
steps.selective-checks.outputs.latest-versions-only }}
- chicken-egg-providers: ${{
steps.selective-checks.outputs.chicken-egg-providers }}
- has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
- source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }}
- pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
- in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }}
- build-job-description: ${{
steps.source-run-info.outputs.build-job-description }}
- canary-run: ${{ steps.source-run-info.outputs.canary-run }}
- run-coverage: ${{ steps.source-run-info.outputs.run-coverage }}
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- - name: Fetch incoming commit ${{ github.sha }} with its parent
- uses: actions/checkout@v4
- with:
- ref: ${{ github.sha }}
- fetch-depth: 2
- persist-credentials: false
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- - name: "Get information about the Workflow"
- id: source-run-info
- run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT}
- - name: Selective checks
- id: selective-checks
- env:
- PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}"
- COMMIT_REF: "${{ github.sha }}"
- VERBOSE: "false"
- run: breeze ci selective-check 2>> ${GITHUB_OUTPUT}
- - name: env
- run: printenv
- env:
- PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
- GITHUB_CONTEXT: ${{ toJson(github) }}
-
- basic-tests:
- name: "Basic tests"
- needs: [build-info]
- uses: ./.github/workflows/basic-tests.yml
- with:
- run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
- needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
- default-python-version:
${{needs.build-info.outputs.default-python-version}}
- basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
- skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}}
- canary-run: ${{needs.build-info.outputs.canary-run}}
- latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
-
- build-ci-images:
- name: >
- ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' ||
'Skip building' }}
- CI images in-workflow
- needs: [build-info]
- uses: ./.github/workflows/ci-image-build.yml
+ push-cache:
+ name: Push Image Cache
+ uses: ./.github/workflows/push-image-cache.yml
permissions:
contents: read
- # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
- # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
- # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
- # "in-workflow-build" condition
packages: write
secrets: inherit
with:
- do-build: ${{ needs.build-info.outputs.in-workflow-build }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- upload-constraints: "true"
- branch: ${{ needs.build-info.outputs.default-branch }}
+ runs-on: "[\"ubuntu-22.04\"]"
+ # runs-on: "[\"self-hosted\", \"Linux\", \"X64\"]"
+ include-prod-images: "false"
+ push-latest-images: "false"
+ image-tag: "test-tag1234"
+ python-versions: "[ \"3.8\" ]"
+ branch: "main"
use-uv: "true"
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
-
- wait-for-ci-images:
- timeout-minutes: 120
- name: "Wait for CI images"
- runs-on: ["ubuntu-22.04"]
- needs: [build-info, build-ci-images]
- if: needs.build-info.outputs.ci-image-build == 'true'
- env:
- BACKEND: sqlite
- # Force more parallelism for pull even on public images
- PARALLELISM: 6
- INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Login to ghcr.io
- run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{
needs.build-info.outputs.image-tag }}
- id: wait-for-images
- run: breeze ci-image pull --run-in-parallel --wait-for-image
--tag-as-latest
- env:
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- if: needs.build-info.outputs.in-workflow-build == 'false'
-
- additional-ci-image-checks:
- name: "Additional CI image checks"
- needs: [build-info, wait-for-ci-images]
- uses: ./.github/workflows/additional-ci-image-checks.yml
- with:
- runs-on: ${{needs.build-info.outputs.runs-on}}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- branch: ${{ needs.build-info.outputs.default-branch }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- canary-run: ${{ needs.build-info.outputs.canary-run }}
- mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
- needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
- latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only
}}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-
- static-checks-mypy-and-constraints-generation:
- name: "Static checks, mypy and constraints generation"
- needs: [build-info, wait-for-ci-images]
- uses: ./.github/workflows/static-checks-mypy-and-constraints-generation.yml
- with:
- runs-on: ${{needs.build-info.outputs.runs-on}}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- branch: ${{ needs.build-info.outputs.default-branch }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only
}}
- basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }}
- chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
- ci-image-build: ${{ needs.build-info.outputs.ci-image-build }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
-
- docs:
- name: "Docs"
- uses: ./.github/workflows/docs.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- if: >
- needs.build-info.outputs.docs-build == 'true' &&
- needs.build-info.outputs.latest-versions-only != 'true'
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- default-branch: ${{ needs.build-info.outputs.default-branch }}
- docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- canary-run: ${{ needs.build-info.outputs.canary-run }}
-
- providers:
- name: "Provider checks"
- uses: ./.github/workflows/check-providers.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- if: >
- needs.build-info.outputs.skip-providers-tests != 'true' &&
- needs.build-info.outputs.latest-versions-only != 'true'
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- affected-providers-list-as-string: ${{
needs.build-info.outputs.affected-providers-list-as-string }}
- providers-compatibility-checks: ${{
needs.build-info.outputs.providers-compatibility-checks }}
- skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
-
- tests-helm:
- name: "Helm tests"
- uses: ./.github/workflows/helm-tests.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- if: >
- needs.build-info.outputs.needs-helm-tests == 'true' &&
- needs.build-info.outputs.default-branch == 'main' &&
- needs.build-info.outputs.latest-versions-only != 'true'
-
- tests-postgres:
- name: "Postgres tests"
- uses: ./.github/workflows/run-unit-tests.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- backend: "postgres"
- test-name: "Postgres"
- test-scope: "DB"
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- backend-versions: ${{ needs.build-info.outputs.postgres-versions }}
- excludes: ${{ needs.build-info.outputs.postgres-exclude }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- run-migration-tests: "true"
- run-coverage: ${{ needs.build-info.outputs.run-coverage }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.run-tests == 'true'
-
- tests-mysql:
- name: "MySQL tests"
- uses: ./.github/workflows/run-unit-tests.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- backend: "mysql"
- test-name: "MySQL"
- test-scope: "DB"
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- backend-versions: ${{ needs.build-info.outputs.mysql-versions }}
- excludes: ${{ needs.build-info.outputs.mysql-exclude }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- run-coverage: ${{ needs.build-info.outputs.run-coverage }}
- run-migration-tests: "true"
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.run-tests == 'true'
+ include-success-outputs: "true"
+ constraints-branch: "constraints-main"
+ docker-cache: "registry"
- tests-sqlite:
- name: "Sqlite tests"
- uses: ./.github/workflows/run-unit-tests.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- backend: "sqlite"
- test-name: "Sqlite"
- test-name-separator: ""
- test-scope: "DB"
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- # No versions for sqlite
- backend-versions: "['']"
- excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- run-coverage: ${{ needs.build-info.outputs.run-coverage }}
- run-migration-tests: "true"
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.run-tests == 'true'
-
- tests-non-db:
- name: "Non-DB tests"
- uses: ./.github/workflows/run-unit-tests.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- backend: "sqlite"
- test-name: ""
- test-name-separator: ""
- test-scope: "Non-DB"
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- # No versions for non-db
- backend-versions: "['']"
- excludes: ${{ needs.build-info.outputs.sqlite-exclude }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- run-coverage: ${{ needs.build-info.outputs.run-coverage }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.run-tests == 'true'
-
- tests-special:
- name: "Special tests"
- uses: ./.github/workflows/special-tests.yml
- needs: [build-info, wait-for-ci-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
- run-coverage: ${{ needs.build-info.outputs.run-coverage }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- default-postgres-version: ${{
needs.build-info.outputs.default-postgres-version }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.run-tests == 'true'
-
- tests-integration:
- name: Integration Tests
- needs: [build-info, wait-for-ci-images]
- uses: ./.github/workflows/integration-tests.yml
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- parallel-test-types-list-as-string: ${{
needs.build-info.outputs.parallel-test-types-list-as-string }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- default-postgres-version: ${{
needs.build-info.outputs.default-postgres-version }}
- default-mysql-version: ${{
needs.build-info.outputs.default-mysql-version }}
- skip-provider-tests: ${{ needs.build-info.outputs.skip-provider-tests }}
- is-airflow-runner: ${{ needs.build-info.outputs.is-airflow-runner }}
- run-coverage: ${{ needs.build-info.outputs.run-coverage }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.run-tests == 'true'
-
- build-prod-images:
- name: >
- ${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' ||
'Skip building' }}
- PROD images in-workflow
- needs: [build-info, build-ci-images]
- uses: ./.github/workflows/prod-image-build.yml
+ build-ci-images:
+ name: Build CI images
+ needs: [push-cache]
+ uses: ./.github/workflows/ci-image-build.yml
permissions:
contents: read
- # This write is only given here for `push` events from "apache/airflow"
repo. It is not given for PRs
- # from forks. This is to prevent malicious PRs from creating images in
the "apache/airflow" repo.
- # For regular build for PRS this "build-prod-images" workflow will be
skipped anyway by the
- # "in-workflow-build" condition
packages: write
secrets: inherit
with:
- build-type: "Regular"
- do-build: ${{ needs.build-info.outputs.in-workflow-build }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- branch: ${{ needs.build-info.outputs.default-branch }}
- push-image: "true"
+ runs-on: "[\"ubuntu-22.04\"]"
+ # runs-on: "[\"self-hosted\", \"Linux\", \"X64\"]"
+ do-build: "true"
+ image-tag: "test-tag1234"
+ python-versions: "[ \"3.8\" ]"
+ upload-constraints: "true"
+ branch: "main"
use-uv: "true"
- build-provider-packages: ${{ needs.build-info.outputs.default-branch ==
'main' }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
-
- wait-for-prod-images:
- timeout-minutes: 80
- name: "Wait for PROD images"
- runs-on: ["ubuntu-22.04"]
- needs: [build-info, wait-for-ci-images, build-prod-images]
- if: needs.build-info.outputs.prod-image-build == 'true'
- env:
- BACKEND: sqlite
- PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- # Force more parallelism for pull on public images
- PARALLELISM: 6
- INCLUDE_SUCCESS_OUTPUTS:
"${{needs.build-info.outputs.include-success-outputs}}"
- IMAGE_TAG: ${{ needs.build-info.outputs.image-tag }}
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Cleanup docker
- uses: ./.github/actions/cleanup-docker
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Login to ghcr.io
- run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{
github.actor }} --password-stdin
- if: needs.build-info.outputs.in-workflow-build == 'false'
- - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{
needs.build-info.outputs.image-tag }}
- # We wait for the images to be available either from
"build-images.yml' run as pull_request_target
- # or from build-prod-images (or build-prod-images-release-branch)
above.
- # We are utilising single job to wait for all images because this job
merely waits
- # For the images to be available.
- run: breeze prod-image pull --wait-for-image --run-in-parallel
- env:
- PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
- if: needs.build-info.outputs.in-workflow-build == 'false'
-
- additional-prod-image-tests:
- name: "Additional PROD image tests"
- needs: [build-info, wait-for-prod-images,
static-checks-mypy-and-constraints-generation]
- uses: ./.github/workflows/additional-prod-image-tests.yml
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- default-branch: ${{ needs.build-info.outputs.default-branch }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- chicken-egg-providers: ${{
needs.build-info.outputs.chicken-egg-providers }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- canary-run: ${{ needs.build-info.outputs.canary-run }}
- if: needs.build-info.outputs.prod-image-build == 'true'
-
- tests-kubernetes:
- name: "Kubernetes tests"
- uses: ./.github/workflows/k8s-tests.yml
- needs: [build-info, wait-for-prod-images]
- permissions:
- contents: read
- packages: read
- secrets: inherit
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- kubernetes-versions-list-as-string: ${{
needs.build-info.outputs.kubernetes-versions-list-as-string }}
- kubernetes-combos-list-as-string: ${{
needs.build-info.outputs.kubernetes-combos-list-as-string }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- debug-resources: ${{ needs.build-info.outputs.debug-resources }}
- if: >
- ( needs.build-info.outputs.run-kubernetes-tests == 'true' ||
- needs.build-info.outputs.needs-helm-tests == 'true')
-
- finalize-tests:
- name: Finalize tests
- permissions:
- contents: write
- packages: write
- secrets: inherit
- needs:
- - build-info
- - docs
- - wait-for-ci-images
- - wait-for-prod-images
- - tests-sqlite
- - tests-mysql
- - tests-postgres
- - tests-non-db
- - tests-special
- - tests-integration
- - static-checks-mypy-and-constraints-generation
- uses: ./.github/workflows/finalize-tests.yml
- with:
- runs-on: ${{ needs.build-info.outputs.runs-on }}
- image-tag: ${{ needs.build-info.outputs.image-tag }}
- python-versions: ${{ needs.build-info.outputs.python-versions }}
- python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
- branch: ${{ needs.build-info.outputs.default-branch }}
- constraints-branch: ${{
needs.build-info.outputs.default-constraints-branch }}
- default-python-version: ${{
needs.build-info.outputs.default-python-version }}
- in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }}
- upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
- include-success-outputs: ${{
needs.build-info.outputs.include-success-outputs }}
- docker-cache: ${{ needs.build-info.outputs.cache-directive }}
- canary-run: ${{ needs.build-info.outputs.canary-run }}
+ upgrade-to-newer-dependencies: "false"
+ constraints-branch: "constraints-main"
+ docker-cache: "registry"
