This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch v2-6-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-6-test by this push:
new 1ff6b27df8 Speed up verification and instalation of providers in CI
(#30839)
1ff6b27df8 is described below
commit 1ff6b27df8d4b739f5f6674d8655c2ca00cf8aa7
Author: Jarek Potiuk <[email protected]>
AuthorDate: Tue Apr 25 21:02:16 2023 +0200
Speed up verification and instalation of providers in CI (#30839)
For PRs that are just changing selected providers, there is no
need to install and verify all providers, just those affected
should be installed (and full verification should only be done
when all providers are being built - so generally speaking, when
Core PRs are run or when canary builds are run.
This PR distinguishes those two cases:
* installing and verification when all providers are selected
* installing only affected providers when only subset of them are
affected
Also a bug was found where core file changes did not trigger provider
docs build during implementation. This has been fixed
(cherry picked from commit 3af22f5bedad370ccb35ae887dff6daf995fd90c)
---
.github/workflows/ci.yml | 285 +++++++++++++++------
dev/breeze/SELECTIVE_CHECKS.md | 106 ++++----
.../src/airflow_breeze/utils/selective_checks.py | 108 ++++++--
dev/breeze/tests/test_selective_checks.py | 111 +++-----
4 files changed, 381 insertions(+), 229 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d1845680f0..084b3e3ed4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -128,6 +128,8 @@ jobs:
GITHUB_CONTEXT: ${{ toJson(github) }}
outputs:
cache-directive: ${{ steps.selective-checks.outputs.cache-directive }}
+ affected-providers-list-as-string: >
+ ${{ steps.selective-checks.outputs.affected-providers-list-as-string }}
upgrade-to-newer-dependencies: ${{
steps.selective-checks.outputs.upgrade-to-newer-dependencies }}
python-versions: ${{ steps.selective-checks.outputs.python-versions }}
python-versions-list-as-string: ${{
steps.selective-checks.outputs.python-versions-list-as-string }}
@@ -136,7 +138,8 @@ jobs:
default-python-version: ${{
steps.selective-checks.outputs.default-python-version }}
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string
}}
- kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos
}}
+ kubernetes-combos-list-as-string: >-
+ ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }}
default-kubernetes-version: ${{
steps.selective-checks.outputs.default-kubernetes-version }}
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions
}}
default-postgres-version: ${{
steps.selective-checks.outputs.default-postgres-version }}
@@ -146,12 +149,12 @@ jobs:
default-helm-version: ${{
steps.selective-checks.outputs.default-helm-version }}
default-kind-version: ${{
steps.selective-checks.outputs.default-kind-version }}
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed
}}
- parallel-test-types: ${{
steps.selective-checks.outputs.parallel-test-types }}
+ parallel-test-types-list-as-string: >-
+ ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string
}}
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
- providers-package-format-exclude: ${{
steps.selective-checks.outputs.providers-package-format-exclude }}
skip-provider-tests: ${{
steps.source-run-info.outputs.skip-provider-tests }}
run-tests: ${{ steps.selective-checks.outputs.run-tests }}
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }}
@@ -165,7 +168,7 @@ jobs:
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen
}}
default-branch: ${{ steps.selective-checks.outputs.default-branch }}
default-constraints-branch: ${{
steps.selective-checks.outputs.default-constraints-branch }}
- docs-filter: ${{ steps.selective-checks.outputs.docs-filter }}
+ docs-filter-list-as-string: ${{
steps.selective-checks.outputs.docs-filter-list-as-string }}
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }}
helm-test-packages: ${{
steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
@@ -552,7 +555,7 @@ jobs:
env:
PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
BACKEND: sqlite
- PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
- name: "Fix ownership"
run: breeze ci fix-ownership
if: always()
@@ -687,7 +690,7 @@ jobs:
docs-inventory-${{
hashFiles('setup.py','setup.cfg','pyproject.toml;') }}
docs-inventory-
- name: "Build docs"
- run: breeze build-docs ${{ needs.build-info.outputs.docs-filter }}
+ run: breeze build-docs ${{
needs.build-info.outputs.docs-filter-list-as-string }}
- name: Configure AWS credentials
uses: ./.github/actions/configure-aws-credentials
if: >
@@ -706,22 +709,16 @@ jobs:
run: breeze ci fix-ownership
if: always()
- prepare-test-provider-packages:
+ prepare-test-provider-packages-wheel:
timeout-minutes: 80
- name: "Provider packages ${{matrix.package-format}}"
+ name: "Provider packages wheel (verify)"
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
- strategy:
- matrix:
- package-format: ["sdist", "wheel"]
- exclude:
"${{fromJson(needs.build-info.outputs.providers-package-format-exclude)}}"
- fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
- PACKAGE_FORMAT: "${{matrix.package-format}}"
- USE_AIRFLOW_VERSION: "${{matrix.package-format}}"
- if: needs.build-info.outputs.image-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
+ if: >
+ needs.build-info.outputs.image-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
steps:
- name: Cleanup repo
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
@@ -735,50 +732,151 @@ jobs:
- name: "Cleanup dist files"
run: rm -fv ./dist/*
- name: "Prepare provider documentation"
- run: breeze release-management prepare-provider-documentation
- if: matrix.package-format == 'wheel'
- - name: "Prepare provider packages: ${{matrix.package-format}}"
- run: breeze release-management prepare-provider-packages
--version-suffix-for-pypi dev0
- - name: "Prepare airflow package: ${{matrix.package-format}}"
+ run: >
+ breeze release-management prepare-provider-documentation
+ ${{ needs.build-info.outputs.affected-providers-list-as-string }}
+ - name: "Prepare provider packages: wheel"
+ run: >
+ breeze release-management prepare-provider-packages
--version-suffix-for-pypi dev0
+ --package-format wheel ${{
needs.build-info.outputs.affected-providers-list-as-string }}
+ - name: "Prepare airflow package: wheel"
run: breeze release-management prepare-airflow-package
--version-suffix-for-pypi dev0
- name: "Verify wheel packages with twine"
- run: pipx install twine && twine check dist/*.whl
- if: matrix.package-format == 'wheel'
- - name: "Verify sdist packages with twine"
- run: pipx install twine && twine check dist/*.tar.gz
- if: matrix.package-format == 'sdist'
+ run: pipx install twine --force && twine check dist/*.whl
- name: "Test providers issue generation automatically"
run: >
breeze release-management generate-issue-content-providers
--only-available-in-dist --disable-progress
- if: matrix.package-format == 'wheel'
- - name: "Install and test provider packages and airflow via
${{matrix.package-format}} files"
- run: breeze release-management verify-provider-packages
--use-packages-from-dist
+ - name: "Install and verify all provider packages and airflow via wheel
files"
+ run: >
+ breeze release-management verify-provider-packages
--use-packages-from-dist
+ --package-format wheel --use-airflow-version wheel
+ if: needs.build-info.outputs.affected-providers-list-as-string == ''
+ env:
+ SKIP_CONSTRAINTS: "${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}"
+ - name: "Install affected provider packages and airflow via wheel files"
+ run: >
+ breeze release-management install-provider-packages
+ --package-format wheel --use-airflow-version wheel --run-in-parallel
+ if: needs.build-info.outputs.affected-providers-list-as-string != ''
env:
SKIP_CONSTRAINTS: "${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}"
- - name: "Remove airflow package and replace providers with 2.3-compliant
versions"
+ - name: "Fix ownership"
+ run: breeze ci fix-ownership
+ if: always()
+
+ provider-airflow-compatibility-check:
+ timeout-minutes: 80
+ name: "Providers Airflow 2.3 compatibility check"
+ runs-on: "${{needs.build-info.outputs.runs-on}}"
+ needs: [build-info, wait-for-ci-images]
+ env:
+ RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+ PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ if: >
+ needs.build-info.outputs.image-build == 'true' &&
needs.build-info.outputs.default-branch == 'main'
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: >
+ Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+ uses: ./.github/actions/prepare_breeze_and_image
+ - name: "Cleanup dist files"
+ run: rm -fv ./dist/*
+ - name: "Prepare provider packages: wheel"
+ run: >
+ breeze release-management prepare-provider-packages
--version-suffix-for-pypi dev0
+ --package-format wheel ${{
needs.build-info.outputs.affected-providers-list-as-string }}
+ - name: "Fix incompatible 2.3 provider packages"
run: |
- rm -vf dist/apache_airflow-*.whl
- # remove the provider packages that are not compatible with 2.3
- # rm -vf dist/apache_airflow_providers_docker*.whl
- # pip download --no-deps --dest dist
apache-airflow-providers-docker==3.1.0
- if: matrix.package-format == 'wheel'
+ # This step should remove the provider packages that are not
compatible with 2.3
+ # or replace them with 2.3 compatible versions. Sometimes we have
good reasons to bump
+ # the min airflow versions for some providers and then we need to
add exclusions here.
+ #
+ # The Removal can be done with:
+ #
+ # rm -vf dist/apache_airflow_providers_<PROVIDER>*.whl
+ #
+ # Then it can be followed by downloading a compatible version from
PyPI in case other
+ # providers depend on it and fail with import errors (you need to
download compatible version):
+ #
+ # pip download --no-deps --dest dist
apache-airflow-providers-<PROVIDER>==3.1.0
+ #
+ rm -vf dist/apache_airflow_providers_openlineage*.whl
- name: "Get all provider extras as AIRFLOW_EXTRAS env variable"
- run: >
- python -c 'from pathlib import Path; import json;
- providers =
json.loads(Path("generated/provider_dependencies.json").read_text());
- provider_keys = ",".join(providers.keys());
+ # Extras might be different on S3 so rather than relying on "all" we
should get the list of
+ # packages to be installed from the current provider_dependencies.json
file
+ run: |
+ python -c 'from pathlib import Path; import json
+ providers =
json.loads(Path("generated/provider_dependencies.json").read_text())
+ provider_keys = ",".join(providers.keys())
print("AIRFLOW_EXTRAS={}".format(provider_keys))' >> $GITHUB_ENV
- if: matrix.package-format == 'wheel'
- - name: "Install and test provider packages and airflow on Airflow 2.3
files"
+ - name: "Install and verify all provider packages and airflow on Airflow
2.3 files"
run: >
breeze release-management verify-provider-packages
--use-airflow-version 2.3.0
--use-packages-from-dist --airflow-constraints-reference
constraints-2.3.0
- if: matrix.package-format == 'wheel'
+ if: needs.build-info.outputs.affected-providers-list-as-string == ''
+ - name: "Install affected provider packages and airflow on Airflow 2.3
files"
+ run: >
+ breeze release-management install-provider-packages
--use-airflow-version 2.3.0
+ --airflow-constraints-reference constraints-2.3.0 --run-in-parallel
+ if: needs.build-info.outputs.affected-providers-list-as-string != ''
- name: "Fix ownership"
run: breeze ci fix-ownership
if: always()
+ prepare-install-provider-packages-sdist:
+ timeout-minutes: 80
+ name: "Provider packages sdist (install)"
+ runs-on: "${{needs.build-info.outputs.runs-on}}"
+ needs: [build-info, wait-for-ci-images]
+ env:
+ RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+ PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ USE_AIRFLOW_VERSION: "sdist"
+ # if: ${{ steps.source-run-info.outputs.canary-run }} == 'true'
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: >
+ Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+ uses: ./.github/actions/prepare_breeze_and_image
+ - name: "Cleanup dist files"
+ run: rm -fv ./dist/*
+ - name: "Prepare provider packages: sdist"
+ run: >
+ breeze release-management prepare-provider-packages
+ --version-suffix-for-pypi dev0 --package-format sdist
+ ${{ needs.build-info.outputs.affected-providers-list-as-string }}
+ - name: "Prepare airflow package: sdist"
+ run: >
+ breeze release-management prepare-airflow-package
+ --version-suffix-for-pypi dev0 --package-format sdist
+ - name: "Verify sdist packages with twine"
+ run: pipx install twine --force && twine check dist/*.tar.gz
+ - name: "Install all provider packages and airflow via sdist files"
+ run: >
+ breeze release-management install-provider-packages
+ --package-format sdist --use-airflow-version sdist --run-in-parallel
+ if: needs.build-info.outputs.affected-providers-list-as-string == ''
+ - name: "Install affected provider packages and airflow via sdist files"
+ run: >
+ breeze release-management install-provider-packages
+ --package-format sdist --use-airflow-version sdist --run-in-parallel
+ if: needs.build-info.outputs.affected-providers-list-as-string != ''
+ - name: "Fix ownership"
+ run: breeze ci fix-ownership
+ if: always()
+
+
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
@@ -854,7 +952,7 @@ jobs:
timeout-minutes: 130
name: >
Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
strategy:
@@ -865,7 +963,7 @@ jobs:
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
@@ -886,14 +984,19 @@ jobs:
persist-credentials: false
- name: "Prepare breeze & CI image:
${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- - name: "Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Migration Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- - name: "Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: breeze testing tests --run-in-parallel
- name: "Tests ARM Pytest collection: ${{matrix.python-version}}"
run: breeze testing tests --run-in-parallel --collect-only
--remove-arm-packages
if: matrix.postgres-version ==
needs.build-info.outputs.default-postgres-version
- - name: "Post Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Post Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-postgres-boto:
@@ -901,12 +1004,12 @@ jobs:
name: >
LatestBotoPostgres${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
@@ -934,11 +1037,11 @@ jobs:
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{needs.build-info.outputs.default-python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: breeze testing tests --run-in-parallel
- name: >
Post Tests: ${{needs.build-info.outputs.default-python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-postgres-in-progress-features-disabled:
@@ -946,12 +1049,12 @@ jobs:
name: >
InProgressDisabledPostgres${{needs.build-info.outputs.default-postgres-version}},
Py${{needs.build-info.outputs.default-python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
@@ -980,18 +1083,18 @@ jobs:
uses: ./.github/actions/prepare_breeze_and_image
- name: >
Tests: ${{needs.build-info.outputs.default-python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: breeze testing tests --run-in-parallel
- name: >
Post Tests: ${{needs.build-info.outputs.default-python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-mysql:
timeout-minutes: 130
name: >
MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
strategy:
@@ -1004,7 +1107,7 @@ jobs:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
BACKEND: "mysql"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
@@ -1022,18 +1125,23 @@ jobs:
persist-credentials: false
- name: "Prepare breeze & CI image:
${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- - name: "Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Migration Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- - name: "Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: breeze testing tests --run-in-parallel
- - name: "Post Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Post Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-mssql:
timeout-minutes: 130
name: >
MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}:
- ${{needs.build-info.outputs.parallel-test-types}}
+ ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
strategy:
@@ -1044,7 +1152,7 @@ jobs:
fail-fast: false
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
@@ -1065,17 +1173,22 @@ jobs:
persist-credentials: false
- name: "Prepare breeze & CI image:
${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- - name: "Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Migration Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- - name: "Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: breeze testing tests --run-in-parallel
- - name: "Post Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Post Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-sqlite:
timeout-minutes: 130
name: >
- Sqlite Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types}}
+ Sqlite Py${{matrix.python-version}}:
${{needs.build-info.outputs.parallel-test-types-list-as-string}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
needs: [build-info, wait-for-ci-images]
strategy:
@@ -1086,7 +1199,7 @@ jobs:
if: needs.build-info.outputs.run-tests == 'true'
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
@@ -1105,11 +1218,16 @@ jobs:
persist-credentials: false
- name: "Prepare breeze & CI image:
${{matrix.python-version}}:${{env.IMAGE_TAG}}"
uses: ./.github/actions/prepare_breeze_and_image
- - name: "Migration Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Migration Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/migration_tests
- - name: "Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
run: breeze testing tests --run-in-parallel
- - name: "Post Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Post Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-integration-postgres:
@@ -1119,7 +1237,7 @@ jobs:
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
@@ -1166,10 +1284,17 @@ jobs:
breeze testing integration-tests --integration trino --integration
kerberos
breeze stop
if: needs.build-info.outputs.runs-on != 'self-hosted'
+ - name: "Integration Tests Postgres: Kafka"
+ run: |
+ breeze testing integration-tests --integration kafka
+ breeze stop
+ if: needs.build-info.outputs.runs-on != 'self-hosted'
- name: "Integration Tests Postgres: all-testable"
run: breeze testing integration-tests --integration all-testable
if: needs.build-info.outputs.runs-on == 'self-hosted'
- - name: "Post Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Post Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
tests-integration-mysql:
@@ -1179,7 +1304,7 @@ jobs:
needs: [build-info, wait-for-ci-images]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
- PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types}}"
+ PARALLEL_TEST_TYPES:
"${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
@@ -1207,7 +1332,9 @@ jobs:
- name: "Integration Tests MySQL: all-testable"
run: breeze testing integration-tests --integration all-testable
if: needs.build-info.outputs.runs-on == 'self-hosted'
- - name: "Post Tests:
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types}}"
+ - name: >
+ Post Tests:
+
${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
uses: ./.github/actions/post_tests
if: needs.build-info.outputs.runs-on == 'self-hosted'
@@ -1332,7 +1459,8 @@ jobs:
fetch-depth: 2
persist-credentials: false
- name: >
- Prepare breeze & PROD image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+ Prepare breeze & PROD image:
+
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
uses: ./.github/actions/prepare_breeze_and_image
with:
pull-image-type: 'PROD'
@@ -1378,21 +1506,22 @@ jobs:
key: "\
k8s-env-${{
hashFiles('scripts/ci/kubernetes/k8s_requirements.txt','setup.cfg',\
'setup.py','pyproject.toml','generated/provider_dependencies.json') }}"
- - name: Run complete K8S tests
${{needs.build-info.outputs.kubernetes-combos}}
+ - name: Run complete K8S tests
${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
run: breeze k8s run-complete-tests --run-in-parallel --upgrade
env:
PYTHON_VERSIONS: ${{
needs.build-info.outputs.python-versions-list-as-string }}
KUBERNETES_VERSIONS:
${{needs.build-info.outputs.kubernetes-versions-list-as-string}}
EXECUTOR: ${{matrix.executor}}
VERBOSE: false
- - name: Upload KinD logs on failure
${{needs.build-info.outputs.kubernetes-combos}}
+ - name: Upload KinD logs on failure
${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v3
if: failure() || cancelled()
with:
name: kind-logs-${{matrix.executor}}
path: /tmp/kind_logs_*
retention-days: 7
- - name: Upload test resource logs on failure
${{needs.build-info.outputs.kubernetes-combos}}
+ - name: >
+ Upload test resource logs on failure
${{needs.build-info.outputs.kubernetes-combos-list-as-string}}
uses: actions/upload-artifact@v3
if: failure() || cancelled()
with:
diff --git a/dev/breeze/SELECTIVE_CHECKS.md b/dev/breeze/SELECTIVE_CHECKS.md
index 73827a807a..a3e64a6977 100644
--- a/dev/breeze/SELECTIVE_CHECKS.md
+++ b/dev/breeze/SELECTIVE_CHECKS.md
@@ -103,51 +103,61 @@ The logic implements the following rules:
* if `Image building` is disabled, only basic pre-commits are enabled - no
'image-depending` pre-commits
are enabled.
* If there are some setup files changed, `upgrade to newer dependencies` is
enabled.
-
-The selective check outputs available are described below:
-
-| Output | Meaning of the output
| Example value
|
-|------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------------------------------------------|
-| all-python-versions | List of all python versions there are
available in the form of JSON array | ['3.7',
'3.8', '3.9', '3.10'] |
-| all-python-versions-list-as-string | List of all python versions there are
available in the form of space separated string | 3.7 3.8 3.9
3.10 |
-| basic-checks-only | Whether to run all static checks
("false") or only basic set of static checks ("true") | false
|
-| cache-directive | Which cache should be be used for
images ("registry", "local" , "disabled") | registry
|
-| debug-resources | Whether resources usage should be
printed during parallel job execution ("true"/ "false") | false
|
-| default-branch | Which branch is default for the the
build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main
|
-| default-constraints-branch | Which branch is default for the the
build ("constraints-main" for main branch, "constraints-2-4" etc.) |
constraints-main |
-| default-helm-version | Which Helm version to use as default
| v3.9.4
|
-| default-kind-version | Which Kind version to use as default
| v0.16.0
|
-| default-kubernetes-version | Which Kubernetes version to use as
default | v1.25.2
|
-| default-mssql-version | Which MsSQL version to use as default
| 2017-latest
|
-| default-mysql-version | Which MySQL version to use as default
| 5.7
|
-| default-postgres-version | Which Postgres version to use as
default | 10
|
-| default-python-version | Which Python version to use as default
| 3.7
|
-| docs-build | Whether to build documentation
("true"/"false") | true
|
-| docs-filter | What filter to apply to docs building -
used in non-main branches to skip provider and chart docs. |
--package-filter apache-airflow --package-filter docker-stack |
-| full-tests-needed | Whether this build runs complete set of
tests or only subset (for faster PR builds). | false
|
-| helm-version | Which Helm version to use for tests
| v3.9.4
|
-| image-build | Whether CI image build is needed
| true
|
-| kind-version | Which Kind version to use for tests
| v0.16.0
|
-| kubernetes-combos | All combinations of Python version and
Kubernetes version to use for tests as space-separated string | 3.7-v1.25.2
3.8-v1.26.4 |
-| kubernetes-versions | All Kubernetes versions to use for
tests as JSON array |
['v1.25.2'] |
-| kubernetes-versions-list-as-string | All Kubernetes versions to use for
tests as space-separated string | v1.25.2
|
-| mssql-exclude | Which versions of MsSQL to exclude for
tests as JSON array | []
|
-| mssql-versions | Which versions of MsSQL to use for
tests as JSON array |
['2017-latest'] |
-| mysql-exclude | Which versions of MySQL to exclude for
tests as JSON array | []
|
-| mysql-versions | Which versions of MySQL to use for
tests as JSON array | ['5.7']
|
-| needs-api-codegen | Whether "api-codegen" are needed to run
("true"/"false") | true
|
-| needs-api-tests | Whether "api-tests" are needed to run
("true"/"false") | true
|
-| needs-helm-tests | Whether Helm tests are needed to run
("true"/"false") | true
|
-| needs-javascript-scans | Whether javascript CodeQL scans should
be run ("true"/"false") | true
|
-| needs-python-scans | Whether Python CodeQL scans should be
run ("true"/"false") | true
|
-| postgres-exclude | Which versions of Postgres to exclude
for tests as JSON array | []
|
-| postgres-versions | Which versions of Postgres to use for
tests as JSON array | ['10']
|
-| python-versions | Which versions of Python to use for
tests as JSON array | ['3.7']
|
-| python-versions-list-as-string | Which versions of MySQL to use for
tests as space-separated string | 3.7
|
-| run-kubernetes-tests | Whether Kubernetes tests should be run
("true"/"false") | true
|
-| run-tests | Whether unit tests should be run
("true"/"false") | true
|
-| run-www-tests | Whether WWW tests should be run
("true"/"false") | true
|
-| skip-pre-commits | Which pre-commits should be skipped
during the static-checks run | identity
|
-| sqlite-exclude | Which versions of Sqlite to exclude for
tests as JSON array | []
|
-| test-types | Which test types should be run for unit
tests | API Always CLI
Core Integration Other Providers WWW |
-| upgrade-to-newer-dependencies | Whether the image build should attempt
to upgrade all dependencies | false
|
+* If docs are build, the `docs-filter-list-as-string` will determine which
docs packages to build. This is based on
+ several criteria: if any of the airflow core, charts, docker-stack,
providers files or docs have changed,
+ then corresponding packages are build (including cross-dependent providers).
If any of the core files
+ changed, also providers docs are built because all providers depend on
airflow docs. If any of the docs
+ build python files changed or when build is "canary" type in main - all docs
packages are built.
+
+The selective check outputs available are described below. In case of
`list-as-string` values,
+empty string means `everything`, where lack of the output means `nothing` and
list elements are
+separated by spaces. This is to accommodate for the wau how outputs of this
kind can be easily used by
+Github Actions to pass the list of parameters to a command to execute
+
+| Output | Meaning of the output
| Example value
| List as string |
+|------------------------------------|---------------------------------------------------------------------------------------------------------|------------------------------------------------------------|----------------|
+| affected-providers-list-as-string | List of providers affected when they
are selectively affected. | airbyte
http | * |
+| all-python-versions | List of all python versions there are
available in the form of JSON array | ['3.7',
'3.8', '3.9', '3.10'] | |
+| all-python-versions-list-as-string | List of all python versions there are
available in the form of space separated string | 3.7 3.8 3.9
3.10 | * |
+| basic-checks-only | Whether to run all static checks
("false") or only basic set of static checks ("true") | false
| |
+| cache-directive | Which cache should be be used for
images ("registry", "local" , "disabled") |
registry | |
+| debug-resources | Whether resources usage should be
printed during parallel job execution ("true"/ "false") | false
| |
+| default-branch | Which branch is default for the the
build ("main" for main branch, "v2-4-test" for 2.4 line etc.) | main
| |
+| default-constraints-branch | Which branch is default for the the
build ("constraints-main" for main branch, "constraints-2-4" etc.) |
constraints-main | |
+| default-helm-version | Which Helm version to use as default
| v3.9.4
| |
+| default-kind-version | Which Kind version to use as default
| v0.16.0
| |
+| default-kubernetes-version | Which Kubernetes version to use as
default | v1.25.2
| |
+| default-mssql-version | Which MsSQL version to use as default
| 2017-latest
| |
+| default-mysql-version | Which MySQL version to use as default
| 5.7
| |
+| default-postgres-version | Which Postgres version to use as
default | 10
| |
+| default-python-version | Which Python version to use as default
| 3.7
| |
+| docs-build | Whether to build documentation
("true"/"false") | true
| |
+| docs-filter-list-as-string | What filter to apply to docs building -
based on which documentation packages should be built |
--package-filter apache-airflow -package-filter-helm-chart | |
+| full-tests-needed | Whether this build runs complete set of
tests or only subset (for faster PR builds) | false
| |
+| helm-version | Which Helm version to use for tests
| v3.9.4
| |
+| image-build | Whether CI image build is needed
| true
| |
+| kind-version | Which Kind version to use for tests
| v0.16.0
| |
+| kubernetes-combos-list-as-string | All combinations of Python version and
Kubernetes version to use for tests as space-separated string | 3.7-v1.25.2
3.8-v1.26.4 | * |
+| kubernetes-versions | All Kubernetes versions to use for
tests as JSON array |
['v1.25.2'] | |
+| kubernetes-versions-list-as-string | All Kubernetes versions to use for
tests as space-separated string | v1.25.2
| * |
+| mssql-exclude | Which versions of MsSQL to exclude for
tests as JSON array | []
| |
+| mssql-versions | Which versions of MsSQL to use for
tests as JSON array |
['2017-latest'] | |
+| mysql-exclude | Which versions of MySQL to exclude for
tests as JSON array | []
| |
+| mysql-versions | Which versions of MySQL to use for
tests as JSON array | ['5.7']
| |
+| needs-api-codegen | Whether "api-codegen" are needed to run
("true"/"false") | true
| |
+| needs-api-tests | Whether "api-tests" are needed to run
("true"/"false") | true
| |
+| needs-helm-tests | Whether Helm tests are needed to run
("true"/"false") | true
| |
+| needs-javascript-scans | Whether javascript CodeQL scans should
be run ("true"/"false") | true
| |
+| needs-python-scans | Whether Python CodeQL scans should be
run ("true"/"false") | true
| |
+| parallel-test-types-list-as-string | Which test types should be run for unit
tests | API Always
Providers\[amazon\] Providers\[-amazon\] | * |
+| postgres-exclude | Which versions of Postgres to exclude
for tests as JSON array | []
| |
+| postgres-versions | Which versions of Postgres to use for
tests as JSON array | ['10']
| |
+| python-versions | Which versions of Python to use for
tests as JSON array | ['3.7']
| |
+| python-versions-list-as-string | Which versions of MySQL to use for
tests as space-separated string | 3.7
| * |
+| run-kubernetes-tests | Whether Kubernetes tests should be run
("true"/"false") | true
| |
+| run-tests | Whether unit tests should be run
("true"/"false") | true
| |
+| run-www-tests | Whether WWW tests should be run
("true"/"false") | true
| |
+| skip-pre-commits | Which pre-commits should be skipped
during the static-checks run | true
| |
+| skip-provider-tests | When provider tests should be skipped
(on non-main branch or when no provider changes detected) | true
| |
+| sqlite-exclude | Which versions of Sqlite to exclude for
tests as JSON array | []
| |
+| upgrade-to-newer-dependencies | Whether the image build should attempt
to upgrade all dependencies (might be true/false or commit hash) | false
| |
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index d6012b6c36..9a6c1dee44 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -27,6 +27,7 @@ from airflow_breeze.utils.kubernetes_utils import
get_kubernetes_python_combos
from airflow_breeze.utils.path_utils import (
AIRFLOW_PROVIDERS_ROOT,
AIRFLOW_SOURCES_ROOT,
+ DOCS_DIR,
SYSTEM_TESTS_PROVIDERS_ROOT,
TESTS_PROVIDERS_ROOT,
)
@@ -41,6 +42,8 @@ from functools import lru_cache
from re import match
from typing import Any, Dict, List, TypeVar
+from typing_extensions import Literal
+
from airflow_breeze.global_constants import (
ALL_PYTHON_MAJOR_MINOR_VERSIONS,
CURRENT_KUBERNETES_VERSIONS,
@@ -195,7 +198,7 @@ TEST_TYPE_MATCHES = HashableDict(
)
-def find_provider_affected(changed_file: str) -> str | None:
+def find_provider_affected(changed_file: str, include_docs: bool) -> str |
None:
file_path = AIRFLOW_SOURCES_ROOT / changed_file
# is_relative_to is only available in Python 3.9 - we should simplify this
check when we are Python 3.9+
for provider_root in (TESTS_PROVIDERS_ROOT, SYSTEM_TESTS_PROVIDERS_ROOT,
AIRFLOW_PROVIDERS_ROOT):
@@ -206,6 +209,13 @@ def find_provider_affected(changed_file: str) -> str |
None:
except ValueError:
pass
else:
+ if include_docs:
+ try:
+ relative_path = file_path.relative_to(DOCS_DIR)
+ if
relative_path.parts[0].startswith("apache-airflow-providers-"):
+ return
relative_path.parts[0].replace("apache-airflow-providers-", "").replace("-",
".")
+ except ValueError:
+ pass
return None
for parent_dir_path in file_path.parents:
@@ -230,13 +240,15 @@ def add_dependent_providers(
providers.add(dep_name)
-def find_all_providers_affected(changed_files: tuple[str, ...]) -> set[str]:
+def find_all_providers_affected(
+ changed_files: tuple[str, ...], include_docs: bool
+) -> list[str] | Literal["ALL_PROVIDERS"] | None:
all_providers: set[str] = set()
dependencies = json.loads((AIRFLOW_SOURCES_ROOT / "generated" /
"provider_dependencies.json").read_text())
all_providers_affected = False
suspended_providers: set[str] = set()
for changed_file in changed_files:
- provider = find_provider_affected(changed_file)
+ provider = find_provider_affected(changed_file,
include_docs=include_docs)
if provider == "Providers":
all_providers_affected = True
elif provider is not None:
@@ -245,7 +257,7 @@ def find_all_providers_affected(changed_files: tuple[str,
...]) -> set[str]:
else:
all_providers.add(provider)
if all_providers_affected:
- return set()
+ return "ALL_PROVIDERS"
if suspended_providers:
# We check for suspended providers only after we have checked if all
providers are affected.
# No matter if we found that we are modifying a suspended provider
individually, if all providers are
@@ -265,9 +277,11 @@ def find_all_providers_affected(changed_files: tuple[str,
...]) -> set[str]:
)
get_console().print(f"Suspended providers: {suspended_providers}")
sys.exit(1)
+ if len(all_providers) == 0:
+ return None
for provider in list(all_providers):
add_dependent_providers(all_providers, provider, dependencies)
- return all_providers
+ return sorted(all_providers)
class SelectiveChecks:
@@ -304,7 +318,9 @@ class SelectiveChecks:
output = []
for field_name in dir(self):
if not field_name.startswith("_"):
- output.append(get_ga_output(field_name, getattr(self,
field_name)))
+ value = getattr(self, field_name)
+ if value is not None:
+ output.append(get_ga_output(field_name, value))
return "\n".join(output)
default_python_version = DEFAULT_PYTHON_MAJOR_MINOR_VERSION
@@ -387,11 +403,6 @@ class SelectiveChecks:
def helm_version(self) -> str:
return HELM_VERSION
- @cached_property
- def providers_package_format_exclude(self) -> list[dict[str, str]]:
- # Exclude sdist format unless full tests are run
- return [{"package-format": "sdist"}] if not self.full_tests_needed
else []
-
@cached_property
def postgres_exclude(self) -> list[dict[str, str]]:
if not self.full_tests_needed:
@@ -444,7 +455,7 @@ class SelectiveChecks:
return " ".join(self.kubernetes_versions)
@cached_property
- def kubernetes_combos(self) -> str:
+ def kubernetes_combos_list_as_string(self) -> str:
python_version_array: list[str] =
self.python_versions_list_as_string.split(" ")
kubernetes_version_array: list[str] =
self.kubernetes_versions_list_as_string.split(" ")
combo_titles, short_combo_titles, combos =
get_kubernetes_python_combos(
@@ -510,7 +521,12 @@ class SelectiveChecks:
@cached_property
def run_amazon_tests(self) -> bool:
- return "amazon" in self.parallel_test_types or "Providers" in
self.parallel_test_types.split(" ")
+ if self.parallel_test_types_list_as_string is None:
+ return False
+ return (
+ "amazon" in self.parallel_test_types_list_as_string
+ or "Providers" in self.parallel_test_types_list_as_string.split("
")
+ )
@cached_property
def run_kubernetes_tests(self) -> bool:
@@ -542,6 +558,11 @@ class SelectiveChecks:
get_console().print(f"[warning]{test_type} added because it
matched {count} files[/]")
return matched_files
+ def _are_all_providers_affected(self) -> bool:
+ # if "Providers" test is present in the list of tests, it means that
we should run all providers tests
+ # prepare all providers packages and build all providers documentation
+ return "Providers" in self._get_test_types_to_run()
+
def _get_test_types_to_run(self) -> list[str]:
candidate_test_types: set[str] = {"Always"}
matched_files: set[str] = set()
@@ -575,8 +596,10 @@ class SelectiveChecks:
candidate_test_types.update(all_selective_test_types())
else:
if "Providers" in candidate_test_types:
- affected_providers =
find_all_providers_affected(changed_files=self._files)
- if len(affected_providers) != 0:
+ affected_providers = find_all_providers_affected(
+ changed_files=self._files, include_docs=False
+ )
+ if affected_providers != "ALL_PROVIDERS" and
affected_providers is not None:
candidate_test_types.remove("Providers")
candidate_test_types.add(f"Providers[{','.join(sorted(affected_providers))}]")
get_console().print(
@@ -618,9 +641,9 @@ class SelectiveChecks:
current_test_types.add(f"Providers[{','.join(provider_tests_to_run)}]")
@cached_property
- def parallel_test_types(self) -> str:
+ def parallel_test_types_list_as_string(self) -> str | None:
if not self.run_tests:
- return ""
+ return None
if self.full_tests_needed:
current_test_types = set(all_selective_test_types())
else:
@@ -666,12 +689,33 @@ class SelectiveChecks:
) > 0 or self._github_event in [GithubEvents.PUSH,
GithubEvents.SCHEDULE]
@cached_property
- def docs_filter(self) -> str:
- return (
- ""
- if self._default_branch == "main"
- else "--package-filter apache-airflow --package-filter
docker-stack"
- )
+ def docs_filter_list_as_string(self) -> str | None:
+ _ALL_DOCS_LIST = ""
+ if not self.docs_build:
+ return None
+ if self._default_branch != "main":
+ return "--package-filter apache-airflow --package-filter
docker-stack"
+ if self.full_tests_needed:
+ return _ALL_DOCS_LIST
+ providers_affected =
find_all_providers_affected(changed_files=self._files, include_docs=True)
+ if (
+ providers_affected == "ALL_PROVIDERS"
+ or "docs/conf.py" in self._files
+ or "docs/build_docs.py" in self._files
+ or self._are_all_providers_affected()
+ ):
+ return _ALL_DOCS_LIST
+ packages = []
+ if any([file.startswith("airflow/") for file in self._files]):
+ packages.append("apache-airflow")
+ if any([file.startswith("chart/") or
file.startswith("docs/helm-chart") for file in self._files]):
+ packages.append("helm-chart")
+ if any([file.startswith("docs/docker-stack/") for file in
self._files]):
+ packages.append("docker-stack")
+ if providers_affected:
+ for provider in providers_affected:
+
packages.append(f"apache-airflow-providers-{provider.replace('.', '-')}")
+ return " ".join([f"--package-filter {package}" for package in
packages])
@cached_property
def skip_pre_commits(self) -> str:
@@ -679,7 +723,9 @@ class SelectiveChecks:
@cached_property
def skip_provider_tests(self) -> bool:
- return self._default_branch != "main"
+ return self._default_branch != "main" or not any(
+ test_type.startswith("Providers") for test_type in
self._get_test_types_to_run()
+ )
@cached_property
def cache_directive(self) -> str:
@@ -692,3 +738,17 @@ class SelectiveChecks:
@cached_property
def helm_test_packages(self) -> str:
return json.dumps(all_helm_test_packages())
+
+ @cached_property
+ def affected_providers_list_as_string(self) -> str | None:
+ _ALL_PROVIDERS_LIST = ""
+ if self.full_tests_needed:
+ return _ALL_PROVIDERS_LIST
+ if self._are_all_providers_affected():
+ return _ALL_PROVIDERS_LIST
+ affected_providers =
find_all_providers_affected(changed_files=self._files, include_docs=True)
+ if not affected_providers:
+ return None
+ if affected_providers == "ALL_PROVIDERS":
+ return _ALL_PROVIDERS_LIST
+ return " ".join(sorted(affected_providers))
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 7a23c3c551..6597150d5e 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -266,10 +266,10 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
("setup.py",),
{
"affected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10',
'3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10
3.11",
- "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
@@ -289,10 +289,10 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
("generated/provider_dependencies.json",),
{
"affected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10',
'3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10
3.11",
- "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
@@ -397,10 +397,10 @@ def test_expected_output_pull_request_main(
"main",
{
"affected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10',
'3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10
3.11",
- "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -424,10 +424,10 @@ def test_expected_output_pull_request_main(
"main",
{
"affected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10',
'3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10
3.11",
- "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -449,10 +449,10 @@ def test_expected_output_pull_request_main(
"main",
{
"affected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10',
'3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10
3.11",
- "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -474,10 +474,10 @@ def test_expected_output_pull_request_main(
"v2-3-stable",
{
"affected-providers-list-as-string":
ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10',
'3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10
3.11",
- "python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
+ "python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"run-tests": "true",
"docs-build": "true",
@@ -787,8 +787,8 @@ def test_expected_output_pull_request_target(
"main",
{
"affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
@@ -807,8 +807,8 @@ def test_expected_output_pull_request_target(
"v2-3-stable",
{
"affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"needs-helm-tests": "false",
"run-tests": "true",
@@ -826,8 +826,8 @@ def test_expected_output_pull_request_target(
"main",
{
"affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
@@ -878,8 +878,8 @@ def
test_no_commit_provided_trigger_full_build_for_any_event_type(github_event):
)
assert_outputs_are_printed(
{
- "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']",
- "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11",
+ "all-python-versions": "['3.7', '3.8', '3.9', '3.10']",
+ "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10",
"image-build": "true",
"needs-helm-tests": "true",
"run-tests": "true",
@@ -1071,50 +1071,3 @@ def test_docs_filter(files: tuple[str, ...],
expected_outputs: dict[str, str]):
default_branch="main",
)
assert_outputs_are_printed(expected_outputs, str(stderr))
-
-
[email protected](
- "files, labels, expected_outputs, should_fail",
- [
- pytest.param(
- ("airflow/providers/yandex/test.py",),
- (),
- None,
- True,
- id="Suspended provider changes should fail",
- ),
- pytest.param(
- ("airflow/providers/yandex/test.py",),
- ("allow suspended provider changes",),
- {"affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED},
- False,
- id="Suspended provider changes should not fail if appropriate
label is set",
- ),
- pytest.param(
- ("airflow/providers/yandex/test.py",
"airflow/providers/airbyte/test.py"),
- ("allow suspended provider changes",),
- {"affected-providers-list-as-string": "airbyte http"},
- False,
- id="Only non-suspended provider changes should be listed",
- ),
- ],
-)
-def test_suspended_providers(
- files: tuple[str, ...], labels: tuple[str], expected_outputs: dict[str,
str], should_fail: bool
-):
- failed = False
- try:
- stderr = str(
- SelectiveChecks(
- files=files,
- commit_ref="HEAD",
- github_event=GithubEvents.PULL_REQUEST,
- pr_labels=labels,
- default_branch="main",
- )
- )
- except SystemExit:
- failed = True
- assert failed == should_fail
- if not failed:
- assert_outputs_are_printed(expected_outputs, str(stderr))
