This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 18d1b97b79a Refactor package checks to be test and remove duplication
(#44318)
18d1b97b79a is described below
commit 18d1b97b79adb27609489bfed1809120c9b51993
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Nov 24 11:34:18 2024 +0000
Refactor package checks to be test and remove duplication (#44318)
Package checks are already "tests" not only checks since we
introduced compatibility complete tests. This PR updates
names of files and also removes duplication of wheel/sdist
package build and verification and adding conditional steps.
---
.github/workflows/ci.yml | 10 +-
...ck-providers.yml => test-provider-packages.yml} | 111 ++++++---------------
dev/breeze/doc/ci/04_selective_checks.md | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
.../src/airflow_breeze/utils/selective_checks.py | 8 +-
dev/breeze/tests/test_selective_checks.py | 8 +-
6 files changed, 46 insertions(+), 95 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a1f46d74024..115d78a4991 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -109,7 +109,8 @@ jobs:
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions
}}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
- providers-compatibility-checks: ${{
steps.selective-checks.outputs.providers-compatibility-checks }}
+ # yamllint disable rule:line-length
+ providers-compatibility-tests-matrix: ${{
steps.selective-checks.outputs.providers-compatibility-tests-matrix }}
providers-test-types-list-as-string: >-
${{ steps.selective-checks.outputs.providers-test-types-list-as-string
}}
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
@@ -323,8 +324,8 @@ jobs:
docs-build: ${{ needs.build-info.outputs.docs-build }}
providers:
- name: "Provider checks"
- uses: ./.github/workflows/check-providers.yml
+ name: "Provider packages tests"
+ uses: ./.github/workflows/test-provider-packages.yml
needs: [build-info, wait-for-ci-images]
permissions:
contents: read
@@ -340,7 +341,8 @@ jobs:
default-python-version: ${{
needs.build-info.outputs.default-python-version }}
upgrade-to-newer-dependencies: ${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}
selected-providers-list-as-string: ${{
needs.build-info.outputs.selected-providers-list-as-string }}
- providers-compatibility-checks: ${{
needs.build-info.outputs.providers-compatibility-checks }}
+ # yamllint disable rule:line-length
+ providers-compatibility-tests-matrix: ${{
needs.build-info.outputs.providers-compatibility-tests-matrix }}
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests
}}
python-versions: ${{ needs.build-info.outputs.python-versions }}
providers-test-types-list-as-string: ${{
needs.build-info.outputs.providers-test-types-list-as-string }}
diff --git a/.github/workflows/check-providers.yml
b/.github/workflows/test-provider-packages.yml
similarity index 68%
rename from .github/workflows/check-providers.yml
rename to .github/workflows/test-provider-packages.yml
index a0bf2d316f8..08715af6b58 100644
--- a/.github/workflows/check-providers.yml
+++ b/.github/workflows/test-provider-packages.yml
@@ -44,9 +44,9 @@ on: # yamllint disable-line rule:truthy
description: "List of affected providers as string"
required: false
type: string
- providers-compatibility-checks:
+ providers-compatibility-tests-matrix:
description: >
- JSON-formatted array of providers compatibility checks in the form
of array of dicts
+ JSON-formatted array of providers compatibility tests in the form of
array of dicts
(airflow-version, python-versions, remove-providers, run-tests)
required: true
type: string
@@ -63,10 +63,14 @@ on: # yamllint disable-line rule:truthy
required: true
type: string
jobs:
- prepare-install-verify-provider-packages-wheel:
+ prepare-install-verify-provider-packages:
timeout-minutes: 80
- name: "Provider packages wheel build and verify"
+ name: "Providers ${{ matrix.package-format }} tests"
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ strategy:
+ fail-fast: false
+ matrix:
+ package-format: ["wheel", "sdist"]
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -94,120 +98,65 @@ jobs:
run: >
breeze release-management prepare-provider-documentation
--include-not-ready-providers
--non-interactive
- - name: "Prepare provider packages: wheel"
+ if: matrix.package-format == 'wheel'
+ - name: "Prepare provider packages: ${{ matrix.package-format }}"
run: >
breeze release-management prepare-provider-packages
--include-not-ready-providers
- --version-suffix-for-pypi dev0 --package-format wheel
- - name: "Prepare airflow package: wheel"
- run: breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0
- - name: "Verify wheel packages with twine"
+ --version-suffix-for-pypi dev0 --package-format ${{
matrix.package-format }}
+ - name: "Prepare airflow package: ${{ matrix.package-format }}"
+ run: >
+ breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0
+ --package-format ${{ matrix.package-format }}
+ - name: "Verify ${{ matrix.package-format }} packages with twine"
run: |
uv tool uninstall twine || true
- uv tool install twine && twine check dist/*.whl
+ uv tool install twine && twine check dist/*
- name: "Test providers issue generation automatically"
run: >
breeze release-management generate-issue-content-providers
--only-available-in-dist --disable-progress
+ if: matrix.package-format == 'wheel'
- name: Remove Python 3.9-incompatible provider packages
run: |
echo "Removing Python 3.9-incompatible provider: cloudant"
- rm -vf dist/apache_airflow_providers_cloudant*
+ rm -vf dist/*cloudant*
- name: "Generate source constraints from CI image"
shell: bash
run: >
breeze release-management generate-constraints
--airflow-constraints-mode constraints-source-providers --answer yes
- - name: "Install and verify all provider packages and airflow via wheel
files"
+ - name: "Install and verify wheel provider packages"
run: >
breeze release-management verify-provider-packages
--use-packages-from-dist
- --package-format wheel
- --use-airflow-version wheel
+ --package-format ${{ matrix.package-format }}
+ --use-airflow-version ${{ matrix.package-format }}
--airflow-constraints-reference default
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
env:
AIRFLOW_SKIP_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies
}}"
- - name: "Prepare airflow package: wheel without suffix and skipping the
tag check"
- run: >
- breeze release-management prepare-provider-packages --skip-tag-check
--package-format wheel
-
- prepare-install-provider-packages-sdist:
- timeout-minutes: 80
- name: "Provider packages sdist build and install"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
- env:
- GITHUB_REPOSITORY: ${{ github.repository }}
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- GITHUB_USERNAME: ${{ github.actor }}
- IMAGE_TAG: "${{ inputs.image-tag }}"
- INCLUDE_NOT_READY_PROVIDERS: "true"
- PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
- VERBOSE: "true"
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@v4
- with:
- persist-credentials: false
- - name: "Cleanup docker"
- run: ./scripts/ci/cleanup_docker.sh
- - name: >
- Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{
inputs.image-tag }}
- uses: ./.github/actions/prepare_breeze_and_image
- - name: "Cleanup dist files"
- run: rm -fv ./dist/*
- - name: "Prepare provider packages: sdist"
- run: >
- breeze release-management prepare-provider-packages
--include-not-ready-providers
- --version-suffix-for-pypi dev0 --package-format sdist
- ${{ inputs.selected-providers-list-as-string }}
- - name: "Prepare airflow package: sdist"
- run: >
- breeze release-management prepare-airflow-package
- --version-suffix-for-pypi dev0 --package-format sdist
- - name: "Verify sdist packages with twine"
- run: |
- uv tool uninstall twine || true
- uv tool install twine && twine check dist/*.tar.gz
- - name: "Generate source constraints from CI image"
- shell: bash
- run: >
- breeze release-management generate-constraints
- --airflow-constraints-mode constraints-source-providers --answer yes
- - name: "Install all provider packages and airflow via sdist files"
- run: >
- breeze release-management install-provider-packages
- --use-packages-from-dist
- --package-format sdist
- --use-airflow-version sdist
- --airflow-constraints-reference default
- --providers-constraints-location
-
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
- --run-in-parallel
- if: inputs.selected-providers-list-as-string == ''
- - name: "Install affected provider packages and airflow via sdist files"
+ if: matrix.package-format == 'wheel'
+ - name: "Install all sdist provider packages and airflow"
run: >
breeze release-management install-provider-packages
--use-packages-from-dist
- --package-format sdist
- --use-airflow-version sdist
+ --package-format ${{ matrix.package-format }}
+ --use-airflow-version ${{ matrix.package-format }}
--airflow-constraints-reference default
--providers-constraints-location
/files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
--run-in-parallel
- if: inputs.selected-providers-list-as-string != ''
+ if: matrix.package-format == 'sdist'
- providers-compatibility-checks:
+ providers-compatibility-tests-matrix:
timeout-minutes: 80
- name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }}
provider check
+ name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }}
providers test
runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
strategy:
fail-fast: false
matrix:
- include: ${{fromJSON(inputs.providers-compatibility-checks)}}
+ include: ${{fromJSON(inputs.providers-compatibility-tests-matrix)}}
env:
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index c39bc5df55f..08e5906745c 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -225,7 +225,7 @@ Github Actions to pass the list of parameters to a command
to execute
| postgres-exclude | Which versions of Postgres
to exclude for tests as JSON array |
[] | |
| postgres-versions | Which versions of Postgres
to use for tests as JSON array |
\['12'\] | |
| prod-image-build | Whether PROD image build is
needed |
true | |
-| providers-compatibility-checks | List of dicts:
(python_version, airflow_version, removed_providers) for compatibility checks
| \[{}\] | |
+| providers-compatibility-tests-matrix | Matrix of providers
compatibility tests: (python_version, airflow_version, removed_providers)
| \[{}\] | |
| providers-test-types-list-as-string | Which test types should be
run for unit tests for providers |
Providers Providers\[-google\] | * |
| pyproject-toml-changed | When pyproject.toml changed
in the PR. |
false | |
| python-versions | List of python versions to
use for that build |
\['3.9'\] | |
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py
b/dev/breeze/src/airflow_breeze/global_constants.py
index 4bc765f255a..87e9195be8e 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -609,7 +609,7 @@ DEFAULT_EXTRAS = [
CHICKEN_EGG_PROVIDERS = " ".join(["standard amazon common.sql"])
-BASE_PROVIDERS_COMPATIBILITY_CHECKS: list[dict[str, str | list[str]]] = [
+PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [
{
"python-version": "3.9",
"airflow-version": "2.8.4",
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index c5ecef55261..5f375e6dc7a 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -31,7 +31,6 @@ from airflow_breeze.branch_defaults import AIRFLOW_BRANCH,
DEFAULT_AIRFLOW_CONST
from airflow_breeze.global_constants import (
ALL_PYTHON_MAJOR_MINOR_VERSIONS,
APACHE_AIRFLOW_GITHUB_REPOSITORY,
- BASE_PROVIDERS_COMPATIBILITY_CHECKS,
CHICKEN_EGG_PROVIDERS,
COMMITTERS,
CURRENT_KUBERNETES_VERSIONS,
@@ -45,6 +44,7 @@ from airflow_breeze.global_constants import (
DISABLE_TESTABLE_INTEGRATIONS_FROM_CI,
HELM_VERSION,
KIND_VERSION,
+ PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
RUNS_ON_PUBLIC_RUNNER,
RUNS_ON_SELF_HOSTED_ASF_RUNNER,
RUNS_ON_SELF_HOSTED_RUNNER,
@@ -1376,12 +1376,12 @@ class SelectiveChecks:
return CHICKEN_EGG_PROVIDERS
@cached_property
- def providers_compatibility_checks(self) -> str:
- """Provider compatibility input checks for the current run. Filter out
python versions not built"""
+ def providers_compatibility_tests_matrix(self) -> str:
+ """Provider compatibility input matrix for the current run. Filter out
python versions not built"""
return json.dumps(
[
check
- for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS
+ for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX
if check["python-version"] in self.python_versions
]
)
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 2f00013f89f..f57b156181f 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -25,9 +25,9 @@ import pytest
from rich.console import Console
from airflow_breeze.global_constants import (
- BASE_PROVIDERS_COMPATIBILITY_CHECKS,
COMMITTERS,
DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+ PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
GithubEvents,
)
from airflow_breeze.utils.packages import get_available_packages
@@ -2291,10 +2291,10 @@ def test_has_migrations(files: tuple[str, ...],
has_migrations: bool):
pytest.param(
(),
{
- "providers-compatibility-checks": json.dumps(
+ "providers-compatibility-tests-matrix": json.dumps(
[
check
- for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS
+ for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX
if check["python-version"] ==
DEFAULT_PYTHON_MAJOR_MINOR_VERSION
]
),
@@ -2303,7 +2303,7 @@ def test_has_migrations(files: tuple[str, ...],
has_migrations: bool):
),
pytest.param(
("all versions",),
- {"providers-compatibility-checks":
json.dumps(BASE_PROVIDERS_COMPATIBILITY_CHECKS)},
+ {"providers-compatibility-tests-matrix":
json.dumps(PROVIDERS_COMPATIBILITY_TESTS_MATRIX)},
id="full tests",
),
],
