This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-2-test by this push:
new 4e06847c2cd [v3-2-test] Run non-provider mypy checks as regular prek
static checks instead of separate CI jobs (#64780) (#64810)
4e06847c2cd is described below
commit 4e06847c2cdeb17be1c947b7a9dffbfb34418835
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Apr 6 23:16:26 2026 +0200
[v3-2-test] Run non-provider mypy checks as regular prek static checks
instead of separate CI jobs (#64780) (#64810)
Non-provider mypy checks (airflow-core, task-sdk, airflow-ctl, dev, scripts,
devel-common) now run locally via uv as regular prek hooks in the pre-commit
stage, instead of running as separate mypy CI jobs in the CI image checks
workflow. This means they run as part of the regular static checks job in CI
and automatically on every local commit.
The folder-level mypy checks (which check entire directories at once for
comprehensive cross-file type checking) replace the previous file-level
incremental checks.
Provider mypy checks still run via breeze as a dedicated CI job, now
embedded
directly in the main CI workflow (ci-amd-arm.yml) instead of being
dispatched
through the ci-image-checks reusable workflow.
The selective checks logic skips non-provider mypy hooks when their relevant
files haven't changed, unless devel-common/pyproject.toml changes on main
(which affects all mypy configurations).
(cherry picked from commit 0ce1dd7)
---
.github/workflows/ci-amd-arm.yml | 51 +++-
.github/workflows/ci-image-checks.yml | 57 -----
.pre-commit-config.yaml | 26 +-
AGENTS.md | 5 +-
airflow-core/.pre-commit-config.yaml | 14 +-
.../airflow/utils/log/non_caching_file_handler.py | 2 +-
airflow-ctl/.pre-commit-config.yaml | 14 +-
contributing-docs/08_static_code_checks.rst | 48 ++--
.../airflow_breeze/commands/developer_commands.py | 8 +
.../airflow_breeze/commands/registry_commands.py | 2 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 2 +-
.../src/airflow_breeze/utils/selective_checks.py | 72 ++----
dev/breeze/tests/test_packages.py | 4 +-
dev/breeze/tests/test_selective_checks.py | 261 +++++++++------------
.../src/sphinx_exts/docs_build/package_filter.py | 2 +-
scripts/ci/prek/check_extra_packages_ref.py | 2 +-
.../prek/check_shared_distributions_structure.py | 2 +-
.../ci/prek/check_shared_distributions_usage.py | 2 +-
scripts/ci/prek/check_version_consistency.py | 2 +-
scripts/ci/prek/mypy_local_folder.py | 223 ++++++++++++++++++
scripts/ci/prek/update_airflow_pyproject_toml.py | 4 +-
scripts/ci/prek/update_providers_dependencies.py | 2 +-
scripts/tools/initialize_virtualenv.py | 2 +-
task-sdk/.pre-commit-config.yaml | 10 +-
24 files changed, 470 insertions(+), 347 deletions(-)
diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml
index 3642e3ea4ba..64e19d1981e 100644
--- a/.github/workflows/ci-amd-arm.yml
+++ b/.github/workflows/ci-amd-arm.yml
@@ -91,7 +91,6 @@ jobs:
kubernetes-versions-list-as-string: >-
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string
}}
latest-versions-only: ${{
steps.selective-checks.outputs.latest-versions-only }}
- mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }}
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
platform: ${{ steps.selective-checks.outputs.platform }}
@@ -115,7 +114,7 @@ jobs:
run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }}
run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }}
run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
- run-mypy: ${{ steps.selective-checks.outputs.run-mypy }}
+ run-mypy-providers: ${{
steps.selective-checks.outputs.run-mypy-providers }}
run-remote-logging-elasticsearch-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
run-remote-logging-s3-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
@@ -307,8 +306,6 @@ jobs:
with:
runners: ${{ needs.build-info.outputs.runner-type }}
platform: ${{ needs.build-info.outputs.platform }}
- run-mypy: ${{ needs.build-info.outputs.run-mypy }}
- mypy-checks: ${{ needs.build-info.outputs.mypy-checks }}
python-versions-list-as-string: ${{
needs.build-info.outputs.python-versions-list-as-string }}
branch: ${{ needs.build-info.outputs.default-branch }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
@@ -333,6 +330,51 @@ jobs:
DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+ mypy-providers:
+ timeout-minutes: 45
+ name: "MyPy providers checks"
+ needs: [build-info, build-ci-images]
+ runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }}
+ if: needs.build-info.outputs.run-mypy-providers == 'true'
+ env:
+ PYTHON_MAJOR_MINOR_VERSION: "${{
needs.build-info.outputs.default-python-version }}"
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ steps:
+ - name: "Cleanup repo"
+ shell: bash
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #
v6.0.2
+ with:
+ persist-credentials: false
+ - name: "Free up disk space"
+ shell: bash
+ run: ./scripts/tools/free_up_disk_space.sh
+ - name: "Prepare breeze & CI image: ${{
needs.build-info.outputs.default-python-version }}"
+ uses: ./.github/actions/prepare_breeze_and_image
+ with:
+ platform: ${{ needs.build-info.outputs.platform }}
+ python: "${{ needs.build-info.outputs.default-python-version }}"
+ use-uv: ${{ needs.build-info.outputs.use-uv }}
+ make-mnt-writeable-and-cleanup: true
+ id: breeze
+ - name: "Install prek"
+ uses: ./.github/actions/install-prek
+ id: prek
+ with:
+ python-version: ${{steps.breeze.outputs.host-python-version}}
+ platform: ${{ needs.build-info.outputs.platform }}
+ save-cache: false
+ - name: "MyPy checks for providers"
+ run: prek --color always --verbose --stage manual mypy-providers
--all-files
+ env:
+ VERBOSE: "false"
+ COLUMNS: "202"
+ SKIP_GROUP_OUTPUT: "true"
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ RUFF_FORMAT: "github"
+ INCLUDE_MYPY_VOLUME: "false"
+
providers:
name: "provider distributions tests"
uses: ./.github/workflows/test-providers.yml
@@ -895,6 +937,7 @@ jobs:
- build-prod-images
- ci-image-checks
- generate-constraints
+ - mypy-providers
- providers
- tests-helm
- tests-integration-system
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index 9d21ad2f92c..1f259432041 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -28,14 +28,6 @@ on: # yamllint disable-line rule:truthy
description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
required: true
type: string
- run-mypy:
- description: "Whether to run mypy checks (true/false)"
- required: true
- type: string
- mypy-checks:
- description: "List of folders to run mypy checks on"
- required: false
- type: string
python-versions-list-as-string:
description: "The list of python versions as string separated by
spaces"
required: true
@@ -169,55 +161,6 @@ jobs:
run: cat ~/.cache/prek/prek.log || true
if: failure()
- mypy:
- timeout-minutes: 45
- name: "MyPy checks"
- runs-on: ${{ fromJSON(inputs.runners) }}
- if: inputs.run-mypy == 'true'
- strategy:
- fail-fast: false
- matrix:
- mypy-check: ${{ fromJSON(inputs.mypy-checks) }}
- env:
- PYTHON_MAJOR_MINOR_VERSION: "${{inputs.default-python-version}}"
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #
v6.0.2
- with:
- persist-credentials: false
- - name: "Free up disk space"
- shell: bash
- run: ./scripts/tools/free_up_disk_space.sh
- - name: "Prepare breeze & CI image: ${{ inputs.default-python-version }}"
- uses: ./.github/actions/prepare_breeze_and_image
- with:
- platform: ${{ inputs.platform }}
- python: "${{ inputs.default-python-version }}"
- use-uv: ${{ inputs.use-uv }}
- make-mnt-writeable-and-cleanup: true
- id: breeze
- - name: "Install prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
- platform: ${{ inputs.platform }}
- save-cache: false
- - name: "MyPy checks for ${{ matrix.mypy-check }}"
- run: prek --color always --verbose --stage manual "$MYPY_CHECK"
--all-files
- env:
- VERBOSE: "false"
- COLUMNS: "202"
- SKIP_GROUP_OUTPUT: "true"
- DEFAULT_BRANCH: ${{ inputs.branch }}
- RUFF_FORMAT: "github"
- INCLUDE_MYPY_VOLUME: "false"
- MYPY_CHECK: ${{ matrix.mypy-check }}
-
build-docs:
timeout-minutes: 150
name: "Build documentation"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3112a512a74..c8bb00b2ff9 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1013,39 +1013,23 @@ repos:
^uv\.lock$
pass_filenames: false
require_serial: true
- ## ADD MOST PREK HOOK ABOVE THAT LINE
- # The below prek hooks are those requiring CI image to be built
- ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
- id: mypy-dev
- stages: ['pre-push']
name: Run mypy for dev
language: python
- entry: ./scripts/ci/prek/mypy.py
- files: ^dev/.*\.py$|^scripts/.*\.py$
- require_serial: true
- - id: mypy-dev
- stages: ['manual']
- name: Run mypy for dev (manual)
- language: python
- entry: ./scripts/ci/prek/mypy_folder.py dev scripts
+ entry: ./scripts/ci/prek/mypy_local_folder.py dev scripts
pass_filenames: false
files: ^.*\.py$
require_serial: true
- id: mypy-devel-common
- stages: ['pre-push']
name: Run mypy for devel-common
language: python
- entry: ./scripts/ci/prek/mypy.py
- files: ^devel-common/.*\.py$
- require_serial: true
- - id: mypy-devel-common
- stages: ['manual']
- name: Run mypy for devel-common (manual)
- language: python
- entry: ./scripts/ci/prek/mypy_folder.py devel-common
+ entry: ./scripts/ci/prek/mypy_local_folder.py devel-common
pass_filenames: false
files: ^.*\.py$
require_serial: true
+ ## ADD MOST PREK HOOK ABOVE THAT LINE
+ # The below prek hooks are those requiring CI image to be built
+ ## ONLY ADD PREK HOOKS HERE THAT REQUIRE CI IMAGE
- id: check-template-fields-valid
name: Check templated fields mapped in operators/sensors
language: python
diff --git a/AGENTS.md b/AGENTS.md
index f01a0112733..894c7e08cc8 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -30,7 +30,8 @@
- **Run other suites of tests** `breeze testing <test_group>` (test groups:
`airflow-ctl-tests`, `docker-compose-tests`, `task-sdk-tests`)
- **Run scripts tests:** `uv run --project scripts pytest scripts/tests/ -xvs`
- **Run Airflow CLI:** `breeze run airflow dags list`
-- **Type-check:** `breeze run mypy path/to/code`
+- **Type-check (non-providers):** `uv run --project <PROJECT> --with
"apache-airflow-devel-common[mypy]" mypy path/to/code`
+- **Type-check (providers):** `breeze run mypy path/to/code`
- **Lint with ruff only:** `prek run ruff --from-ref <target_branch>`
- **Format with ruff only:** `prek run ruff-format --from-ref <target_branch>`
- **Run regular (fast) static checks:** `prek run --from-ref <target_branch>
--stage pre-commit`
@@ -146,7 +147,7 @@ code review checklist in
[`.github/instructions/code-review.instructions.md`](.g
3. Confirm the code follows the project's coding standards and architecture
boundaries
described in this file.
4. Run regular (fast) static checks (`prek run --from-ref <target_branch>
--stage pre-commit`)
- and fix any failures.
+ and fix any failures. This includes mypy checks for non-provider projects
(airflow-core, task-sdk, airflow-ctl, dev, scripts, devel-common).
5. Run manual (slower) checks (`prek run --from-ref <target_branch> --stage
manual`) and fix any failures.
6. Run relevant individual tests and confirm they pass.
7. Find which tests to run for the changes with selective-checks and run those
tests in parallel to confirm they pass and check for CI-specific issues.
diff --git a/airflow-core/.pre-commit-config.yaml
b/airflow-core/.pre-commit-config.yaml
index d995d79d3c1..299c3612c7f 100644
--- a/airflow-core/.pre-commit-config.yaml
+++ b/airflow-core/.pre-commit-config.yaml
@@ -221,23 +221,15 @@ repos:
additional_dependencies: ['[email protected]']
pass_filenames: true
require_serial: true
- ## ADD MOST PREK HOOK ABOVE THAT LINE
- # The below prek hooks are those requiring CI image to be built
- id: mypy-airflow-core
- stages: ['pre-push']
name: Run mypy for airflow-core
language: python
- entry: ../scripts/ci/prek/mypy.py
- files: ^.*\.py$
- require_serial: true
- - id: mypy-airflow-core
- stages: ['manual']
- name: Run mypy for airflow-core (manual)
- language: python
- entry: ../scripts/ci/prek/mypy_folder.py airflow-core
+ entry: ../scripts/ci/prek/mypy_local_folder.py airflow-core
pass_filenames: false
files: ^.*\.py$
require_serial: true
+ ## ADD MOST PREK HOOK ABOVE THAT LINE
+ # The below prek hooks are those requiring CI image to be built
- id: generate-openapi-spec
name: Generate the FastAPI API spec
language: python
diff --git a/airflow-core/src/airflow/utils/log/non_caching_file_handler.py
b/airflow-core/src/airflow/utils/log/non_caching_file_handler.py
index aa0ca9864e2..ad3c0dbe279 100644
--- a/airflow-core/src/airflow/utils/log/non_caching_file_handler.py
+++ b/airflow-core/src/airflow/utils/log/non_caching_file_handler.py
@@ -25,7 +25,7 @@ from typing import IO
def make_file_io_non_caching(io: IO[str]) -> IO[str]:
try:
fd = io.fileno()
- os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
+ os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) # type:
ignore[attr-defined]
except Exception:
# in case either file descriptor cannot be retrieved or fadvise is not
available
# we should simply return the wrapper retrieved by FileHandler's open
method
diff --git a/airflow-ctl/.pre-commit-config.yaml
b/airflow-ctl/.pre-commit-config.yaml
index e63268b077e..c45a1985ec1 100644
--- a/airflow-ctl/.pre-commit-config.yaml
+++ b/airflow-ctl/.pre-commit-config.yaml
@@ -25,21 +25,9 @@ repos:
- repo: local
hooks:
- id: mypy-airflow-ctl
- stages: ['pre-push']
name: Run mypy for airflow-ctl
language: python
- entry: ../scripts/ci/prek/mypy.py
- files:
- (?x)
- ^src/airflowctl/.*\.py$|
- ^tests/.*\.py$
- exclude: .*generated.py
- require_serial: true
- - id: mypy-airflow-ctl
- stages: ['manual']
- name: Run mypy for airflow-ctl (manual)
- language: python
- entry: ../scripts/ci/prek/mypy_folder.py airflow-ctl
+ entry: ../scripts/ci/prek/mypy_local_folder.py airflow-ctl
pass_filenames: false
files: ^.*\.py$
require_serial: true
diff --git a/contributing-docs/08_static_code_checks.rst
b/contributing-docs/08_static_code_checks.rst
index 1d009b92cef..23256c6c0e0 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -173,17 +173,18 @@ But you can run prek hooks manually as needed.
prek
- Run only mypy check on your staged airflow and dev files by specifying the
- ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be
specified):
+ ``mypy-airflow-core`` and ``mypy-dev`` prek hooks (more hooks can be
specified).
+ For non-provider projects, mypy runs locally via ``uv`` (no breeze image
needed):
.. code-block:: bash
- prek mypy-airflow-core mypy-dev --stage pre-push
+ prek mypy-airflow-core mypy-dev
- Run only mypy airflow checks on all "airflow-core" files by using:
.. code-block:: bash
- prek mypy-airflow-core --all-files --stage pre-push
+ prek mypy-airflow-core --all-files
- Run all pre-commit stage hooks on all files by using:
@@ -279,41 +280,40 @@ them manually by running ``prek --stage manual
<hook-id>``.
Mypy checks
-----------
-When we run mypy checks locally when pushing a change to PR, the ``mypy-*``
checks is run, ``mypy-airflow``,
-``mypy-dev``, ``mypy-providers``, ``mypy-airflow-ctl``, depending on the files
you are changing. The mypy checks
-are run by passing those changed files to mypy. This is way faster than
running checks for all files (even
-if mypy cache is used - especially when you change a file in Airflow core that
is imported and used by many
-files). You also need to have ``breeze ci-image build --python 3.10`` built
locally to run the mypy checks.
+When we run mypy checks locally, the ``mypy-*`` checks run depending on the
files you are changing:
+``mypy-airflow-core``, ``mypy-dev``, ``mypy-providers``, ``mypy-task-sdk``,
``mypy-airflow-ctl``, etc.
-However, in some cases, it produces different results than when running checks
for the whole set
-of files, because ``mypy`` does not even know that some types are defined in
other files and it might not
-be able to follow imports properly if they are dynamic. Therefore in CI we run
``mypy`` check for whole
-directories (``airflow`` - excluding providers, ``providers``, ``dev`` and
``docs``) to make sure
-that we catch all ``mypy`` errors - so you can experience different results
when running mypy locally and
-in CI. If you want to run mypy checks for all files locally, you can do it by
running the following
-command (example for ``airflow`` files):
+For **non-provider projects** (airflow-core, task-sdk, airflow-ctl, dev,
scripts, devel-common), mypy
+runs locally using the ``uv`` virtualenv — no breeze CI image is needed. These
checks run as regular
+prek hooks in the ``pre-commit`` stage, checking whole directories at once.
This means they run both
+as part of local commits and as part of regular static checks in CI (not as
separate mypy CI jobs).
+You can also run mypy directly. Use ``--frozen`` to avoid updating ``uv.lock``:
.. code-block:: bash
- prek --stage manual mypy-<FOLDER> --all-files
+ uv run --frozen --project <PROJECT> --with
"apache-airflow-devel-common[mypy]" mypy path/to/code
-For example:
+To run the prek hook for a specific project (example for ``airflow-core``
files):
.. code-block:: bash
- prek --stage manual mypy-airflow --all-files
+ prek mypy-airflow-core --all-files
To show unused mypy ignores for any providers/airflow etc, eg: run below
command:
.. code-block:: bash
+
export SHOW_UNUSED_MYPY_WARNINGS=true
- prek --stage manual mypy-airflow --all-files
+ prek mypy-airflow-core --all-files
+
+For non-provider projects, the local mypy cache is stored in ``.mypy_cache``
at the repo root.
+
+For **providers**, mypy still runs via breeze (``breeze run mypy``) as a
separate CI job and requires
+``breeze ci-image build --python 3.10`` to be built locally. Providers use a
separate docker-volume
+(called ``mypy-cache-volume``) that keeps the cache of last MyPy execution.
-MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps
the cache of last MyPy
-execution in order to speed MyPy checks up (sometimes by order of magnitude).
While in most cases MyPy
-will handle refreshing the cache when and if needed, there are some cases when
it won't (cache invalidation
-is the hard problem in computer science). This might happen for example when
we upgrade MyPY. In such
-cases you might need to manually remove the cache volume by running ``breeze
down --cleanup-mypy-cache``.
+To clear all mypy caches (both local ``.mypy_cache`` and the Docker volume),
run
+``breeze down --cleanup-mypy-cache``.
-----------
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index 5c70a495afc..f10ed0ec0bd 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -932,6 +932,10 @@ def down(preserve_volumes: bool, cleanup_mypy_cache: bool,
cleanup_build_cache:
if cleanup_mypy_cache:
command_to_execute = ["docker", "volume", "rm", "--force",
"mypy-cache-volume"]
run_command(command_to_execute)
+ local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache"
+ if local_mypy_cache.exists():
+ console_print(f"\n[info]Removing local mypy cache:
{local_mypy_cache}\n")
+ shutil.rmtree(local_mypy_cache)
if cleanup_build_cache:
command_to_execute = ["docker", "volume", "rm", "--force",
"airflow-cache-volume"]
run_command(command_to_execute)
@@ -1070,6 +1074,10 @@ def doctor(ctx):
console_print("\n[info]Cleaning mypy cache...\n")
command_to_execute = ["docker", "volume", "rm", "--force",
"mypy-cache-volume"]
run_command(command_to_execute)
+ local_mypy_cache = AIRFLOW_ROOT_PATH / ".mypy_cache"
+ if local_mypy_cache.exists():
+ console_print(f"\n[info]Removing local mypy cache:
{local_mypy_cache}\n")
+ shutil.rmtree(local_mypy_cache)
console_print("\n[info]Cleaning build cache...\n")
command_to_execute = ["docker", "volume", "rm", "--force",
"airflow-cache-volume"]
diff --git a/dev/breeze/src/airflow_breeze/commands/registry_commands.py
b/dev/breeze/src/airflow_breeze/commands/registry_commands.py
index f818a16ea01..0655deb51d1 100644
--- a/dev/breeze/src/airflow_breeze/commands/registry_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/registry_commands.py
@@ -160,7 +160,7 @@ def _read_provider_yaml_info(provider_id: str) ->
tuple[str, list[str]]:
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
provider_yaml_path = _find_provider_yaml(provider_id)
with open(provider_yaml_path) as f:
diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py
b/dev/breeze/src/airflow_breeze/utils/packages.py
index 25f9a945473..553f0379b4c 100644
--- a/dev/breeze/src/airflow_breeze/utils/packages.py
+++ b/dev/breeze/src/airflow_breeze/utils/packages.py
@@ -533,7 +533,7 @@ def load_pyproject_toml(pyproject_toml_file_path: Path) ->
dict[str, Any]:
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
toml_content = pyproject_toml_file_path.read_text()
syntax = Syntax(toml_content, "toml", theme="ansi_dark", line_numbers=True)
try:
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 99e74fcfa50..6a756289db6 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -870,58 +870,21 @@ class SelectiveChecks:
return False
@cached_property
- def mypy_checks(self) -> list[str]:
- checks_to_run: list[str] = []
- if (
- self._matching_files(FileGroupForCi.DEVEL_TOML_FILES,
CI_FILE_GROUP_MATCHES)
- and self._default_branch == "main"
- ):
- return [
- "mypy-airflow-core",
- "mypy-providers",
- "mypy-dev",
- "mypy-task-sdk",
- "mypy-devel-common",
- "mypy-airflow-ctl",
- ]
- if (
- self._matching_files(FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES,
CI_FILE_GROUP_MATCHES)
- or self.full_tests_needed
- ):
- checks_to_run.append("mypy-airflow-core")
- if (
+ def run_mypy_providers(self) -> bool:
+ # Non-provider mypy checks run as part of regular static checks (prek
hooks).
+ # Only provider mypy needs a separate CI job (requires the CI Docker
image with breeze).
+ return (
self._matching_files(FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES,
CI_FILE_GROUP_MATCHES)
or self._matching_files(
FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES,
CI_FILE_GROUP_MATCHES
)
or self._are_all_providers_affected()
- ) and self._default_branch == "main":
- checks_to_run.append("mypy-providers")
- if (
- self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES,
CI_FILE_GROUP_MATCHES)
- or self.full_tests_needed
- ):
- checks_to_run.append("mypy-dev")
- if (
- self._matching_files(FileGroupForCi.TASK_SDK_FILES,
CI_FILE_GROUP_MATCHES)
- or self.full_tests_needed
- ):
- checks_to_run.append("mypy-task-sdk")
- if (
- self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES,
CI_FILE_GROUP_MATCHES)
- or self.full_tests_needed
- ):
- checks_to_run.append("mypy-devel-common")
- if (
- self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES,
CI_FILE_GROUP_MATCHES)
+ or (
+ self._matching_files(FileGroupForCi.DEVEL_TOML_FILES,
CI_FILE_GROUP_MATCHES)
+ and self._default_branch == "main"
+ )
or self.full_tests_needed
- ):
- checks_to_run.append("mypy-airflow-ctl")
- return checks_to_run
-
- @cached_property
- def run_mypy(self) -> bool:
- return self.mypy_checks != []
+ ) and self._default_branch == "main"
@cached_property
def run_python_scans(self) -> bool:
@@ -1499,6 +1462,23 @@ class SelectiveChecks:
# only skip provider validation if none of the provider.yaml and
provider
# python files changed because validation also walks through all
the provider python files
prek_hooks_to_skip.add("check-provider-yaml-valid")
+ # Non-provider mypy checks run as prek hooks in static checks.
+ # Skip them when their relevant files haven't changed, unless
devel-common
+ # pyproject.toml changes on main (which affects all mypy checks).
+ if not (
+ self._matching_files(FileGroupForCi.DEVEL_TOML_FILES,
CI_FILE_GROUP_MATCHES)
+ and self._default_branch == "main"
+ ):
+ if not
self._matching_files(FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES,
CI_FILE_GROUP_MATCHES):
+ prek_hooks_to_skip.add("mypy-airflow-core")
+ if not self._matching_files(FileGroupForCi.ALL_DEV_PYTHON_FILES,
CI_FILE_GROUP_MATCHES):
+ prek_hooks_to_skip.add("mypy-dev")
+ if not self._matching_files(FileGroupForCi.TASK_SDK_FILES,
CI_FILE_GROUP_MATCHES):
+ prek_hooks_to_skip.add("mypy-task-sdk")
+ if not
self._matching_files(FileGroupForCi.ALL_DEVEL_COMMON_PYTHON_FILES,
CI_FILE_GROUP_MATCHES):
+ prek_hooks_to_skip.add("mypy-devel-common")
+ if not
self._matching_files(FileGroupForCi.ALL_AIRFLOW_CTL_PYTHON_FILES,
CI_FILE_GROUP_MATCHES):
+ prek_hooks_to_skip.add("mypy-airflow-ctl")
return ",".join(sorted(prek_hooks_to_skip))
@cached_property
diff --git a/dev/breeze/tests/test_packages.py
b/dev/breeze/tests/test_packages.py
index b48062f93d1..2a11a4ef29d 100644
--- a/dev/breeze/tests/test_packages.py
+++ b/dev/breeze/tests/test_packages.py
@@ -384,7 +384,7 @@ def test_apply_version_suffix_to_provider_pyproject_toml(
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
from unittest.mock import patch
# Get the original provider details
@@ -474,7 +474,7 @@ def
test_apply_version_suffix_to_non_provider_pyproject_tomls(
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
distribution_paths = [AIRFLOW_ROOT_PATH / distribution for distribution in
distributions]
original_pyproject_toml_paths = [path / "pyproject.toml" for path in
distribution_paths]
original_contents = [path.read_text() for path in
original_pyproject_toml_paths]
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 5d6ff376872..90e5551ce33 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -98,59 +98,57 @@ LIST_OF_ALL_PROVIDER_TESTS_AS_JSON = json.dumps(
_get_test_list_as_json(_split_list(sorted(LIST_OF_ALL_PROVIDER_TESTS), 5))
)
-ALL_MYPY_CHECKS_ARRAY = [
- "mypy-airflow-core",
- "mypy-providers",
- "mypy-dev",
- "mypy-task-sdk",
- "mypy-devel-common",
- "mypy-airflow-ctl",
-]
-
-ALL_MYPY_CHECKS = str(ALL_MYPY_CHECKS_ARRAY)
-
-ALL_MYPY_CHECKS_EXCEPT_PROVIDERS = str(
- [check for check in ALL_MYPY_CHECKS_ARRAY if check != "mypy-providers"]
-)
ALL_SKIPPED_COMMITS_ON_NO_CI_IMAGE = (
"check-provider-yaml-valid,flynt,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED = "identity,update-uv-lock"
ALL_SKIPPED_COMMITS_IF_NO_UI = (
-
"identity,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+
"identity,mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+ "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+)
+ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS = (
+ "identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock"
)
-ALL_SKIPPED_COMMITS_IF_NO_HELM_TESTS =
"identity,lint-helm-chart,update-uv-lock"
ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS = (
-
"identity,lint-helm-chart,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ "identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+ "ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_AND_UI = (
"check-provider-yaml-valid,identity,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS = (
"check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS = (
"check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS = (
- "check-provider-yaml-valid,flynt,identity,lint-helm-chart,update-uv-lock"
+ "check-provider-yaml-valid,flynt,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,update-uv-lock"
)
ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED = (
"check-provider-yaml-valid,flynt,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
)
@@ -244,8 +242,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"core-test-types-list-as-strings-in-json": None,
"providers-test-types-list-as-strings-in-json": None,
"individual-providers-test-types-list-as-strings-in-json":
None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="No tests on simple change",
)
@@ -278,8 +275,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
"individual-providers-test-types-list-as-strings-in-json":
LIST_OF_ALL_PROVIDER_TESTS_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests should be run when API file changed",
)
@@ -303,8 +299,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
"individual-providers-test-types-list-as-strings-in-json":
LIST_OF_ALL_PROVIDER_TESTS_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests should be run when fastapi files change",
)
@@ -328,8 +323,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
"individual-providers-test-types-list-as-strings-in-json":
LIST_OF_ALL_PROVIDER_TESTS_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests should run when API test files change",
)
@@ -349,7 +343,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-unit-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS,
+ "skip-prek-hooks": (
+ "check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
_get_test_list_as_json(
@@ -358,8 +356,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
),
"providers-test-types-list-as-strings-in-json": None,
"individual-providers-test-types-list-as-strings-in-json":
None,
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
"skip-providers-tests": "true",
},
id="Only Serialization tests",
@@ -388,8 +385,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
"individual-providers-test-types-list-as-strings-in-json":
LIST_OF_ALL_PROVIDER_TESTS_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests and docs should run on API change",
)
@@ -431,8 +427,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Selected Providers and docs should run",
)
@@ -474,8 +469,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
"skip-providers-tests": "false",
},
id="Selected Providers should run when system tests are
modified",
@@ -521,8 +515,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
"skip-providers-tests": "false",
},
id="Selected Providers and docs should run when both system
tests and tests are modified",
@@ -568,8 +561,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
"skip-providers-tests": "false",
},
id="Selected Providers and docs should run when both system
tests and tests are modified for more than one provider",
@@ -595,8 +587,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": None,
"providers-test-types-list-as-strings-in-json": None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Only docs builds should run - no tests needed",
)
@@ -619,13 +610,16 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-task-sdk-integration-tests": "true",
"docs-build": "true",
"full-tests-needed": "false",
- "skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS,
+ "skip-prek-hooks": (
+ "check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"skip-providers-tests": "false",
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers', 'mypy-task-sdk']",
+ "run-mypy-providers": "true",
},
id="Task SDK source file changed - Task SDK, Core and provider
tests should run",
)
@@ -651,8 +645,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS,
"skip-providers-tests": "true",
"upgrade-to-newer-dependencies": "false",
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Task SDK integration tests files changed - "
"Task SDK integration tests and prod image build should run
but no other tests",
@@ -676,11 +669,14 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-airflow-ctl-integration-tests": "true",
"docs-build": "true",
"full-tests-needed": "false",
- "skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS,
+ "skip-prek-hooks": (
+ "check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-core,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"skip-providers-tests": "true",
"upgrade-to-newer-dependencies": "false",
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-ctl']",
+ "run-mypy-providers": "false",
},
id="Airflow CTL source file changed - Airflow CTL tests should
run",
)
@@ -706,8 +702,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS,
"skip-providers-tests": "true",
"upgrade-to-newer-dependencies": "false",
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Airflow CTL integration tests files changed - "
"Airflow CTL integration tests and prod image build should run
but no other tests",
@@ -748,8 +743,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Helm tests, providers (both upstream and downstream),"
"kubernetes tests and docs should run",
@@ -812,8 +806,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
},
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Helm tests, http and all relevant providers, kubernetes
tests and "
"docs should run even if unimportant files were added",
@@ -880,8 +873,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"providers-test-types-list-as-strings-in-json": None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Docs should run even if unimportant files were added and
prod image "
"should be build for chart changes",
@@ -907,8 +899,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"upgrade-to-newer-dependencies": "true",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run and upgrading to newer requirements
as dependencies change",
)
@@ -947,8 +938,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Providers tests run including amazon tests if only amazon
provider.yaml files changed",
),
@@ -980,8 +970,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Providers tests run without amazon tests if no amazon file
changed",
),
@@ -1018,8 +1007,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Providers tests run including amazon tests if amazon provider
files changed",
),
@@ -1042,7 +1030,11 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-amazon-tests": "false",
"docs-build": "false",
"run-kubernetes-tests": "false",
- "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI_AND_HELM_TESTS,
+ "skip-prek-hooks": (
+ "identity,lint-helm-chart,"
+
"mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
[{"description": "Always", "test_types": "Always"}]
@@ -1055,8 +1047,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core', 'mypy-providers']",
+ "run-mypy-providers": "true",
},
id="Only Always and common providers tests should run when only
common.io and tests/always changed",
),
@@ -1078,8 +1069,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests to run when standard operator changed",
),
@@ -1103,8 +1093,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests should be run when tests/utils/ change",
)
@@ -1132,8 +1121,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"testable-core-integrations": "['kerberos', 'otel',
'redis']",
"testable-providers-integrations": "['celery',
'cassandra', 'drill', 'elasticsearch', 'tinkerpop', 'kafka', "
"'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']",
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All tests should be run when devel-common/ change",
)
@@ -1153,8 +1141,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"full-tests-needed": "false",
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_CODE_PROVIDERS_AND_HELM_TESTS,
"upgrade-to-newer-dependencies": "false",
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
"run-helm-tests": "false",
"run-ui-tests": "true",
"run-ui-e2e-tests": "true",
@@ -1184,8 +1171,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"core-test-types-list-as-strings-in-json": None,
"providers-test-types-list-as-strings-in-json": None,
"individual-providers-test-types-list-as-strings-in-json":
None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Run docs-build for RELEASE_NOTES.rst",
),
@@ -1202,13 +1188,16 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-unit-tests": "false",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-prek-hooks":
"check-provider-yaml-valid,flynt,identity,ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock",
+ "skip-prek-hooks": (
+ "check-provider-yaml-valid,flynt,identity,"
+
"mypy-airflow-core,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": None,
"providers-test-types-list-as-strings-in-json": None,
"individual-providers-test-types-list-as-strings-in-json":
None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Run docs-build for chart/RELEASE_NOTES.rst",
),
@@ -1230,8 +1219,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"core-test-types-list-as-strings-in-json": None,
"providers-test-types-list-as-strings-in-json": None,
"individual-providers-test-types-list-as-strings-in-json":
None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Run docs-build for SECURITY.md",
),
@@ -1244,8 +1232,7 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
pytest.param(
("devel-common/pyproject.toml",),
{
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="All mypy checks should run when
devel-common/pyproject.toml changes",
)
@@ -1526,8 +1513,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run including all providers when git
provider is changed"
"(special case for now)",
@@ -1561,8 +1547,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run including all providers when full
tests are needed, "
"and all versions are required.",
@@ -1596,8 +1581,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run including all providers when full
tests are needed "
"but with single python and kubernetes if `default versions
only` label is set",
@@ -1631,8 +1615,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
"run-ui-e2e-tests": "true",
},
id="Everything should run including all providers when full
tests are needed "
@@ -1668,8 +1651,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run including all providers when full
tests are needed "
"but with single python and kubernetes if `latest versions
only` label is set",
@@ -1704,8 +1686,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run including full providers when full "
"tests are needed even with different label set as well",
@@ -1738,8 +1719,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
"providers-test-types-list-as-strings-in-json":
ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON,
"individual-providers-test-types-list-as-strings-in-json":
LIST_OF_ALL_PROVIDER_TESTS_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="Everything should run including full providers when "
"full tests are needed even if no files are changed",
@@ -1766,8 +1746,7 @@ def test_full_test_needed_when_scripts_changes(files:
tuple[str, ...], expected_
"skip-prek-hooks": All_SKIPPED_COMMITS_IF_NON_MAIN_BRANCH,
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS_EXCEPT_PROVIDERS,
+ "run-mypy-providers": "false",
},
id="Everything should run except Providers and lint prek "
"when full tests are needed for non-main branch",
@@ -1809,8 +1788,7 @@ def test_expected_output_full_tests_needed(
"full-tests-needed": "false",
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Nothing should run if only non-important files changed",
),
@@ -1835,8 +1813,7 @@ def test_expected_output_full_tests_needed(
"core-test-types-list-as-strings-in-json": json.dumps(
[{"description": "Always", "test_types": "Always"}]
),
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="No Helm tests, No providers no lint charts, should run if "
"only chart/providers changed in non-main but PROD image should be
built",
@@ -1863,8 +1840,7 @@ def test_expected_output_full_tests_needed(
"core-test-types-list-as-strings-in-json": json.dumps(
[{"description": "Always...CLI", "test_types": "Always
CLI"}]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
},
id="Only CLI tests and Kubernetes tests should run if cli/chart
files changed in non-main branch",
),
@@ -1887,8 +1863,7 @@ def test_expected_output_full_tests_needed(
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
},
id="All tests except Providers and helm lint prek "
"should run if core file changed in non-main branch",
@@ -1930,8 +1905,7 @@ def test_expected_output_pull_request_v2_7(
"upgrade-to-newer-dependencies": "false",
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED,
"core-test-types-list-as-strings-in-json": None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="No tests run on push if only text non-doc files changed",
),
@@ -1951,8 +1925,7 @@ def test_expected_output_pull_request_v2_7(
"docs-list-as-string": None,
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="No tests run on push if only text non-doc files changed in
non-main branch",
),
@@ -1973,8 +1946,7 @@ def test_expected_output_pull_request_v2_7(
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
"run-ui-e2e-tests": "false",
},
id="All tests run on push if core file changed",
@@ -2029,8 +2001,7 @@ def test_expected_output_push(
"upgrade-to-newer-dependencies": "false",
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NOT_IMPORTANT_FILES_CHANGED,
"core-test-types-list-as-strings-in-json": None,
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Nothing should run if only non-important files changed",
),
@@ -2047,13 +2018,16 @@ def test_expected_output_push(
"skip-providers-tests": "true",
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
- "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS,
+ "skip-prek-hooks": (
+ "check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
[{"description": "Always", "test_types": "Always"}]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
},
id="Only Always and docs build should run if only system tests
changed",
),
@@ -2080,7 +2054,10 @@ def test_expected_output_push(
"apache.kafka cncf.kubernetes common.compat common.messaging
common.sql databricks facebook google hashicorp http microsoft.azure "
"microsoft.mssql mysql openlineage oracle postgres "
"presto salesforce samba sftp ssh standard trino",
- "skip-prek-hooks": ALL_SKIPPED_COMMITS_IF_NO_UI,
+ "skip-prek-hooks": (
+
"identity,mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json": json.dumps(
@@ -2099,8 +2076,7 @@ def test_expected_output_push(
}
]
),
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core', 'mypy-providers']",
+ "run-mypy-providers": "true",
},
id="CLI tests and Google-related provider tests should run if
cli/chart files changed but "
"prod image should be build too and k8s tests too",
@@ -2117,12 +2093,15 @@ def test_expected_output_push(
"skip-providers-tests": "true",
"docs-build": "true",
"docs-list-as-string": "apache-airflow",
- "skip-prek-hooks":
ALL_SKIPPED_COMMITS_IF_NO_PROVIDERS_UI_AND_HELM_TESTS,
+ "skip-prek-hooks": (
+ "check-provider-yaml-valid,identity,lint-helm-chart,"
+
"mypy-airflow-ctl,mypy-dev,mypy-devel-common,mypy-task-sdk,"
+
"ts-compile-lint-simple-auth-manager-ui,ts-compile-lint-ui,update-uv-lock"
+ ),
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
},
id="Tests for all airflow core types except providers should run
if model file changed",
),
@@ -2141,8 +2120,7 @@ def test_expected_output_push(
"upgrade-to-newer-dependencies": "false",
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED,
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
id="pre commit ts-compile-format-lint should not be ignored if
openapi spec changed.",
),
@@ -2179,8 +2157,7 @@ def test_expected_output_push(
}
]
),
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
id="Trigger openlineage and related providers tests when Assets
files changed",
),
@@ -2243,8 +2220,7 @@ def
test_no_commit_provided_trigger_full_build_for_any_event_type(mock_get, gith
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED,
"upgrade-to-newer-dependencies": ("true" if github_event ==
GithubEvents.SCHEDULE else "false"),
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
str(stderr),
)
@@ -2293,8 +2269,7 @@ def
test_files_provided_trigger_full_build_for_any_event_type(mock_get, github_e
"skip-prek-hooks":
ALL_SKIPPED_COMMITS_BY_DEFAULT_ON_ALL_TESTS_NEEDED,
"upgrade-to-newer-dependencies": ("true" if github_event ==
GithubEvents.SCHEDULE else "false"),
"core-test-types-list-as-strings-in-json":
ALL_CI_SELECTIVE_TEST_TYPES_AS_JSON,
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
str(stderr),
)
@@ -2607,8 +2582,7 @@ def test_provider_compatibility_checks(labels: tuple[str,
...], expected_outputs
pytest.param(
("README.md",),
{
- "run-mypy": "false",
- "mypy-checks": "[]",
+ "run-mypy-providers": "false",
},
"main",
(),
@@ -2617,8 +2591,7 @@ def test_provider_compatibility_checks(labels: tuple[str,
...], expected_outputs
pytest.param(
("airflow-core/src/airflow/cli/file.py",),
{
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
},
"main",
(),
@@ -2627,8 +2600,7 @@ def test_provider_compatibility_checks(labels: tuple[str,
...], expected_outputs
pytest.param(
("airflow-core/src/airflow/models/file.py",),
{
- "run-mypy": "true",
- "mypy-checks": "['mypy-airflow-core']",
+ "run-mypy-providers": "false",
},
"main",
(),
@@ -2637,8 +2609,7 @@ def test_provider_compatibility_checks(labels: tuple[str,
...], expected_outputs
pytest.param(
("task-sdk/src/airflow/sdk/a_file.py",),
{
- "run-mypy": "true",
- "mypy-checks": "['mypy-providers', 'mypy-task-sdk']",
+ "run-mypy-providers": "true",
},
"main",
(),
@@ -2647,8 +2618,7 @@ def test_provider_compatibility_checks(labels: tuple[str,
...], expected_outputs
pytest.param(
("dev/a_package/a_file.py",),
{
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
"main",
(),
@@ -2657,8 +2627,7 @@ def test_provider_compatibility_checks(labels: tuple[str,
...], expected_outputs
pytest.param(
("readme.md",),
{
- "run-mypy": "true",
- "mypy-checks": ALL_MYPY_CHECKS,
+ "run-mypy-providers": "true",
},
"main",
("full tests needed",),
diff --git a/devel-common/src/sphinx_exts/docs_build/package_filter.py
b/devel-common/src/sphinx_exts/docs_build/package_filter.py
index f4d7c037232..3cd83951be8 100644
--- a/devel-common/src/sphinx_exts/docs_build/package_filter.py
+++ b/devel-common/src/sphinx_exts/docs_build/package_filter.py
@@ -64,7 +64,7 @@ def find_packages_to_build(available_packages: list[str],
package_filters: list[
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
read_toml = tomllib.loads(pyproject_toml_path.read_text())
package_name = read_toml["project"]["name"]
if package_name == "apache-airflow":
diff --git a/scripts/ci/prek/check_extra_packages_ref.py
b/scripts/ci/prek/check_extra_packages_ref.py
index 6e3bd8bcd6b..009e4d6c662 100755
--- a/scripts/ci/prek/check_extra_packages_ref.py
+++ b/scripts/ci/prek/check_extra_packages_ref.py
@@ -39,7 +39,7 @@ from tabulate import tabulate
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
EXTRA_PACKAGES_REF_FILE = AIRFLOW_ROOT_PATH / "airflow-core" / "docs" /
"extra-packages-ref.rst"
PYPROJECT_TOML_FILE_PATH = AIRFLOW_ROOT_PATH / "pyproject.toml"
diff --git a/scripts/ci/prek/check_shared_distributions_structure.py
b/scripts/ci/prek/check_shared_distributions_structure.py
index ac08c28924b..3a7366bd300 100755
--- a/scripts/ci/prek/check_shared_distributions_structure.py
+++ b/scripts/ci/prek/check_shared_distributions_structure.py
@@ -35,7 +35,7 @@ from pathlib import Path
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
from common_prek_utils import AIRFLOW_ROOT_PATH, console
diff --git a/scripts/ci/prek/check_shared_distributions_usage.py
b/scripts/ci/prek/check_shared_distributions_usage.py
index c5759d76d73..5a734fa255a 100755
--- a/scripts/ci/prek/check_shared_distributions_usage.py
+++ b/scripts/ci/prek/check_shared_distributions_usage.py
@@ -39,7 +39,7 @@ from pathlib import Path
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
from common_prek_utils import AIRFLOW_ROOT_PATH, console, insert_documentation
diff --git a/scripts/ci/prek/check_version_consistency.py
b/scripts/ci/prek/check_version_consistency.py
index ebcf457d0da..2711b55cccd 100755
--- a/scripts/ci/prek/check_version_consistency.py
+++ b/scripts/ci/prek/check_version_consistency.py
@@ -32,7 +32,7 @@ import sys
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
from common_prek_utils import (
AIRFLOW_CORE_SOURCES_PATH,
diff --git a/scripts/ci/prek/mypy_local_folder.py
b/scripts/ci/prek/mypy_local_folder.py
new file mode 100755
index 00000000000..e578079b11f
--- /dev/null
+++ b/scripts/ci/prek/mypy_local_folder.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# /// script
+# requires-python = ">=3.10,<3.11"
+# dependencies = [
+# "rich>=13.6.0",
+# ]
+# ///
+"""Run mypy on entire folders using local virtualenv (uv) instead of breeze.
+
+Used for non-provider projects: airflow-core, task-sdk, airflow-ctl, dev,
scripts, devel-common.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import shlex
+import subprocess
+import sys
+
+from common_prek_utils import (
+ AIRFLOW_ROOT_PATH,
+)
+
+CI = os.environ.get("CI")
+
+try:
+ from rich.console import Console
+
+ console = Console(width=400, color_system="standard")
+except ImportError:
+ console = None # type: ignore[assignment]
+
+if __name__ not in ("__main__", "__mp_main__"):
+ raise SystemExit(
+ "This file is intended to be executed as an executable program. You
cannot use it as a module."
+ f"To run this script, run the ./{__file__} command"
+ )
+
+ALLOWED_FOLDERS = [
+ "airflow-core",
+ "dev",
+ "scripts",
+ "devel-common",
+ "task-sdk",
+ "airflow-ctl",
+]
+
+# Map folder(s) to the uv project to use for running mypy.
+# When multiple folders are checked together (e.g. dev + scripts), the first
folder's project is used.
+FOLDER_TO_PROJECT = {
+ "airflow-core": "airflow-core",
+ "task-sdk": "task-sdk",
+ "airflow-ctl": "airflow-ctl",
+ "devel-common": "devel-common",
+ "dev": "dev",
+ "scripts": "scripts",
+}
+
+if len(sys.argv) < 2:
+ if console:
+ console.print(f"[yellow]You need to specify the folder to test as
parameter: {ALLOWED_FOLDERS}\n")
+ else:
+ print(f"You need to specify the folder to test as parameter:
{ALLOWED_FOLDERS}")
+ sys.exit(1)
+
+mypy_folders = sys.argv[1:]
+
+show_unused_warnings = os.environ.get("SHOW_UNUSED_MYPY_WARNINGS", "false")
+show_unreachable_warnings = os.environ.get("SHOW_UNREACHABLE_MYPY_WARNINGS",
"false")
+
+for mypy_folder in mypy_folders:
+ if mypy_folder not in ALLOWED_FOLDERS:
+ if console:
+ console.print(
+ f"\n[red]ERROR: Folder `{mypy_folder}` is wrong.[/]\n\n"
+ f"All folders passed should be one of those:
{ALLOWED_FOLDERS}\n"
+ )
+ else:
+ print(
+ f"\nERROR: Folder `{mypy_folder}` is wrong.\n\n"
+ f"All folders passed should be one of those:
{ALLOWED_FOLDERS}\n"
+ )
+ sys.exit(1)
+
+exclude_regexps = [
+ re.compile(x)
+ for x in [
+ r"^.*/node_modules/.*",
+ r"^.*\\..*",
+ r"^.*/src/airflow/__init__.py$",
+ ]
+]
+
+
+def get_all_files(folder: str) -> list[str]:
+ files_to_check = []
+ python_file_paths = (AIRFLOW_ROOT_PATH / folder).resolve().rglob("*.py")
+ for file in python_file_paths:
+ if (
+ file.name not in ("conftest.py",)
+ and not any(x.match(file.as_posix()) for x in exclude_regexps)
+ and not any(part.startswith(".") for part in file.parts)
+ ):
+
files_to_check.append(file.relative_to(AIRFLOW_ROOT_PATH).as_posix())
+ return files_to_check
+
+
+all_files_to_check: list[str] = []
+for mypy_folder in mypy_folders:
+ all_files_to_check.extend(get_all_files(mypy_folder))
+
+if not all_files_to_check:
+ print("No files to test. Quitting")
+ sys.exit(0)
+
+# Write file list
+mypy_file_list = AIRFLOW_ROOT_PATH / "files" / "mypy_files.txt"
+mypy_file_list.parent.mkdir(parents=True, exist_ok=True)
+mypy_file_list.write_text("\n".join(all_files_to_check))
+
+if console:
+ console.print(f"[info]You can check the list of files in:[/]
{mypy_file_list}")
+else:
+ print(f"You can check the list of files in: {mypy_file_list}")
+
+file_argument_local = f"@{mypy_file_list}"
+file_argument_ci = "@/files/mypy_files.txt"
+
+project = FOLDER_TO_PROJECT.get(mypy_folders[0], "devel-common")
+
+mypy_extra_args: list[str] = []
+
+if show_unused_warnings == "true":
+ if console:
+ console.print("[info]Running mypy with --warn-unused-ignores")
+ else:
+ print("Running mypy with --warn-unused-ignores")
+ mypy_extra_args.append("--warn-unused-ignores")
+
+if show_unreachable_warnings == "true":
+ if console:
+ console.print("[info]Running mypy with --warn-unreachable")
+ else:
+ print("Running mypy with --warn-unreachable")
+ mypy_extra_args.append("--warn-unreachable")
+
+if console:
+ console.print(f"[magenta]Running mypy for folders: {mypy_folders}[/]")
+else:
+ print(f"Running mypy for folders: {mypy_folders}")
+
+if CI:
+ # In CI, run inside the breeze Docker image to avoid needing a local
environment
+ # and to not change uv.lock or synchronize dependencies.
+ from common_prek_utils import (
+ initialize_breeze_prek,
+ run_command_via_breeze_shell,
+ )
+
+ initialize_breeze_prek(__name__, __file__)
+
+ mypy_cmd = f"TERM=ansi mypy {shlex.quote(file_argument_ci)} {'
'.join(mypy_extra_args)}"
+ result = run_command_via_breeze_shell(
+ cmd=["bash", "-c", mypy_cmd],
+ warn_image_upgrade_needed=True,
+ extra_env={
+ "INCLUDE_MYPY_VOLUME": "false",
+ "MOUNT_SOURCES": "selected",
+ },
+ )
+else:
+ # Locally, run via uv with --frozen to not update the lock file.
+ cmd = [
+ "uv",
+ "run",
+ "--frozen",
+ "--project",
+ project,
+ "--with",
+ "apache-airflow-devel-common[mypy]",
+ "mypy",
+ file_argument_local,
+ *mypy_extra_args,
+ ]
+
+ result = subprocess.run(
+ cmd,
+ cwd=str(AIRFLOW_ROOT_PATH),
+ check=False,
+ env={**os.environ, "TERM": "ansi"},
+ )
+
+if result.returncode != 0:
+ msg = (
+ "Mypy check failed. You can run mypy locally with:\n"
+ f" prek run mypy-{mypy_folders[0]} --all-files\n"
+ "Or directly with:\n"
+ f' uv run --project {project} --with
"apache-airflow-devel-common[mypy]" mypy <files>\n'
+ "You can also clear the mypy cache with:\n"
+ " breeze down --cleanup-mypy-cache\n"
+ )
+ if console:
+ console.print(f"[yellow]{msg}")
+ else:
+ print(msg)
+sys.exit(result.returncode)
diff --git a/scripts/ci/prek/update_airflow_pyproject_toml.py
b/scripts/ci/prek/update_airflow_pyproject_toml.py
index cccae46867e..ff3ee1e1fe2 100755
--- a/scripts/ci/prek/update_airflow_pyproject_toml.py
+++ b/scripts/ci/prek/update_airflow_pyproject_toml.py
@@ -85,7 +85,7 @@ def get_optional_dependencies(pyproject_toml_path: Path) ->
list[str]:
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text())
return airflow_core_toml_dict["project"]["optional-dependencies"].keys()
@@ -112,7 +112,7 @@ def _read_toml(path: Path) -> dict[str, Any]:
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
return tomllib.loads(path.read_text())
diff --git a/scripts/ci/prek/update_providers_dependencies.py
b/scripts/ci/prek/update_providers_dependencies.py
index 9e89084f7b0..b114e55a047 100755
--- a/scripts/ci/prek/update_providers_dependencies.py
+++ b/scripts/ci/prek/update_providers_dependencies.py
@@ -69,7 +69,7 @@ def load_pyproject_toml(pyproject_toml_file_path: Path) ->
dict[str, Any]:
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
return tomllib.loads(pyproject_toml_file_path.read_text())
diff --git a/scripts/tools/initialize_virtualenv.py
b/scripts/tools/initialize_virtualenv.py
index fff6fbb4b62..a6edf0268dc 100755
--- a/scripts/tools/initialize_virtualenv.py
+++ b/scripts/tools/initialize_virtualenv.py
@@ -62,7 +62,7 @@ def get_dependency_groups(pyproject_toml_path: Path) ->
list[str]:
try:
import tomllib
except ImportError:
- import tomli as tomllib
+ import tomli as tomllib # type: ignore[no-redef]
airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text())
return airflow_core_toml_dict["dependency-groups"].keys()
diff --git a/task-sdk/.pre-commit-config.yaml b/task-sdk/.pre-commit-config.yaml
index 315e0ea8a13..df1be38500f 100644
--- a/task-sdk/.pre-commit-config.yaml
+++ b/task-sdk/.pre-commit-config.yaml
@@ -56,17 +56,9 @@ repos:
pass_filenames: false
files:
^src/airflow/sdk/definitions/dag\.py$|^src/airflow/sdk/definitions/decorators/task_group\.py$
- id: mypy-task-sdk
- stages: ['pre-push']
name: Run mypy for task-sdk
language: python
- entry: ../scripts/ci/prek/mypy.py
- files: ^.*\.py$
- require_serial: true
- - id: mypy-task-sdk
- stages: ['manual']
- name: Run mypy for task-sdk (manual)
- language: python
- entry: ../scripts/ci/prek/mypy_folder.py task-sdk
+ entry: ../scripts/ci/prek/mypy_local_folder.py task-sdk
pass_filenames: false
files: ^.*\.py$
require_serial: true
