This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f7b663d9af Run mypy checks for full packages in CI (#36638)
f7b663d9af is described below
commit f7b663d9aff472d0a419e16c262fbae2a8a69ce1
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Jan 7 13:53:23 2024 +0100
Run mypy checks for full packages in CI (#36638)
MyPy as used in our static checks has slightly different heuristics
when running on on individual files and whole packages. This sometimes
causes semi-random failures when different set of files is produced
when pre-commits split the files between parallel processes.
The regular `mypy-*` pre-commits work by passing filenames to mypy
checks, and when `--all-files` flag is passed to mypy, this means
that 2700 files are passed. In this case pre-commit will split such
long list of files to several sequential muypy executions. This
is not very good because depending on the list of files passed,
mypy can split the list diferently and results will be different
when just list of files changes - so mypy might start detecting
problems that were not present before.
This PR introduces new `mypy` check that runs mypy for packages
rather than individual files. We cannot run them for local
pre-commit runs, because in many cases, such package based
mypy check will run for minutes when a single file changes,
due to cache invalidation rules - and we do not want to penalise
commits that are changing common airflow code (because such PRs
would invalidate a lot of mypy cache every time such common file
changes). So we still want to run file-based mypy for local
commits. But we do not want to pass 2700 files in CI, rather than
that on CI we want to run mypy checks "per package".
This PR introduces a new "manual" stage mypy pre-commit check that
will run "package" based mypy checks and adds selective check rules
that will decide properly when to run such tests and separate,
matrix-based CI job that will run such mypy checks - separately
for each of the packages: "airflow", "providers", "docs", "dev".
Also this job will skip providers checks in non-main branch and
will run all tests when "full tests needed" are requested.
This PR ignores some errors resulted from 3rd-party libraries used
that are randomply appearing when some files are modified (and fixes
the current main failures)
---
.github/workflows/ci.yml | 56 ++++-
.pre-commit-config.yaml | 13 +-
STATIC_CODE_CHECKS.rst | 32 ++-
airflow/decorators/__init__.pyi | 20 +-
airflow/models/taskreschedule.py | 3 +-
airflow/operators/latest_only.py | 2 +-
airflow/providers/apache/druid/hooks/druid.py | 4 +-
.../providers/databricks/hooks/databricks_sql.py | 4 +-
airflow/providers/exasol/hooks/exasol.py | 6 +-
airflow/providers/google/cloud/hooks/cloud_run.py | 2 +-
.../providers/google/cloud/operators/bigquery.py | 2 +-
.../providers/google/cloud/triggers/cloud_run.py | 2 +-
.../google/cloud/utils/credentials_provider.py | 2 +-
.../providers/google/common/hooks/base_google.py | 2 +-
.../google/common/utils/id_token_credentials.py | 4 +-
airflow/providers/grpc/hooks/grpc.py | 2 +-
airflow/providers/postgres/hooks/postgres.py | 4 +-
airflow/providers/snowflake/hooks/snowflake.py | 6 +-
airflow/providers/trino/operators/trino.py | 8 +-
airflow/providers/vertica/hooks/vertica.py | 2 +-
airflow/providers_manager.py | 10 +-
airflow/sensors/base.py | 4 +-
airflow/utils/operator_helpers.py | 2 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 +
.../src/airflow_breeze/utils/selective_checks.py | 63 ++++--
dev/breeze/tests/test_selective_checks.py | 234 +++++++++++++++++----
images/breeze/output_static-checks.svg | 24 +--
images/breeze/output_static-checks.txt | 2 +-
scripts/ci/pre_commit/common_precommit_utils.py | 26 ++-
scripts/ci/pre_commit/pre_commit_mypy.py | 31 ++-
30 files changed, 443 insertions(+), 130 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a622e74111..9e9d7a788a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -104,6 +104,8 @@ jobs:
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
docs-build: ${{ steps.selective-checks.outputs.docs-build }}
+ mypy-packages: ${{ steps.selective-checks.outputs.mypy-packages }}
+ needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen
}}
@@ -510,7 +512,6 @@ jobs:
retention-days: 7
if-no-files-found: error
-
static-checks:
timeout-minutes: 45
name: "Static checks"
@@ -550,6 +551,57 @@ jobs:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
RUFF_FORMAT: "github"
+ # Runs static checks for groups of files in the repository in a single
process without passing .
+ # List of files
+ mypy:
+ timeout-minutes: 45
+ name: "MyPy checks"
+ runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+ needs: [build-info, wait-for-ci-images]
+ strategy:
+ fail-fast: false
+ matrix:
+ mypy-package: ${{fromJson(needs.build-info.outputs.mypy-packages)}}
+ env:
+ RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+ PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ UPGRADE_TO_NEWER_DEPENDENCIES: "${{
needs.build-info.outputs.upgrade-to-newer-dependencies }}"
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ if: needs.build-info.outputs.needs-mypy == 'true'
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ if: needs.build-info.outputs.needs-mypy == 'true'
+ - name: >
+ Prepare breeze & CI image:
${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+ uses: ./.github/actions/prepare_breeze_and_image
+ id: breeze
+ if: needs.build-info.outputs.needs-mypy == 'true'
+ - name: Cache pre-commit envs
+ uses: actions/cache@v3
+ with:
+ path: ~/.cache/pre-commit
+ # yamllint disable-line rule:line-length
+ key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{
hashFiles('.pre-commit-config.yaml') }}"
+ restore-keys: |
+ pre-commit-${{steps.breeze.outputs.host-python-version}}-
+ if: needs.build-info.outputs.needs-mypy == 'true'
+ - name: "MyPy checks for ${{ matrix.mypy-package }}"
+ run: |
+ pip install pre-commit
+ pre-commit run --color always --verbose --hook-stage manual mypy
--all-files
+ env:
+ VERBOSE: "false"
+ COLUMNS: "250"
+ SKIP_GROUP_OUTPUT: "true"
+ DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+ RUFF_FORMAT: "github"
+ MYPY_PACKAGES: ${{ matrix.mypy-package }}
+ if: needs.build-info.outputs.needs-mypy == 'true'
+
# Those checks are run if no image needs to be built for checks. This is for
simple changes that
# Do not touch any of the python code or any of the important files that
might require building
# The CI Docker image and they can be run entirely using the pre-commit
virtual environments on host
@@ -2041,6 +2093,7 @@ jobs:
- wait-for-ci-images
- wait-for-prod-images
- static-checks
+ - mypy
- tests-sqlite
- tests-mysql
- tests-postgres
@@ -2195,6 +2248,7 @@ jobs:
- build-docs
- spellcheck-docs
- static-checks
+ - mypy
- tests-sqlite
- tests-mysql
- tests-postgres
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index fcd9284002..4923e4e7ea 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -736,7 +736,7 @@ repos:
# Keep dependency versions in sync w/ airflow/www/package.json
additional_dependencies: ['[email protected]',
'[email protected]', '[email protected]']
- id: compile-www-assets
- name: Compile www assets
+ name: Compile www assets (manual)
language: node
stages: ['manual']
'types_or': [javascript, ts, tsx]
@@ -745,7 +745,7 @@ repos:
pass_filenames: false
additional_dependencies: ['[email protected]']
- id: compile-www-assets-dev
- name: Compile www assets in dev mode
+ name: Compile www assets in dev mode (manual)
language: node
stages: ['manual']
'types_or': [javascript, ts, tsx]
@@ -1105,6 +1105,15 @@ repos:
exclude: ^docs/rtd-deprecation
require_serial: true
additional_dependencies: ['rich>=12.4.4']
+ - id: mypy
+ stages: ['manual']
+ name: Run mypy for specified packages (manual)
+ language: python
+ entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
+ pass_filenames: false
+ files: ^.*\.py$
+ require_serial: true
+ additional_dependencies: ['rich>=12.4.4']
- id: check-provider-yaml-valid
name: Validate provider.yaml files
entry: ./scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index bfe03692ca..7e093aa0c4 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -114,6 +114,13 @@ Available pre-commit checks
This table lists pre-commit hooks used by Airflow. The ``Image`` column
indicates which hooks
require Breeze Docker image to be built locally.
+.. note:: Manual pre-commits
+
+ Most of the checks we run are configured to run automatically when you
commit the code. However,
+ there are some checks that are not run automatically and you need to run
them manually. Those
+ checks are marked with ``manual`` in the ``Description`` column in the table
below. You can run
+ them manually by running ``pre-commit run --hook-stage manual <hook-id>``.
+
.. note:: Disabling particular checks
In case you have a problem with running particular ``pre-commit`` check you
can still continue using the
@@ -127,6 +134,24 @@ require Breeze Docker image to be built locally.
the image by setting ``SKIP_BREEZE_PRE_COMMITS`` to "true". This will mark
the tests as "green" automatically
when run locally (note that those checks will anyway run in CI).
+.. note:: Mypy checks
+
+ When we run mypy checks locally when committing a change, one of the
``mypy-*`` checks is run, ``mypy-core``,
+ ``mypy-dev``, ``mypy-providers``, ``mypy-docs``, depending on the files you
are changing. The mypy checks
+ are run by passing those changed files to mypy. This is way faster than
running checks for all files (even
+ if mypy cache is used - especially when you change a file in airflow core
that is imported and used by many
+ files). However, in some cases, it produces different results than when
running checks for the whole set
+ of files, because ``mypy`` does not even know that some types are defined in
other files and it might not
+ be able to follow imports properly if they are dynamic. Therefore in CI we
run ``mypy`` check for whole
+ directories (``airflow`` - excluding providers, ``airflow/providers``,
``dev`` and ``docs``) to make sure
+ that we catch all ``mypy`` errors - so you can experience different results
when running mypy locally and
+ in CI. If you want to run mypy checks for all files locally, you can do it
by running the following
+ command (example for ``airflow`` files):
+
+ .. code-block:: bash
+
+ MYPY_PACKAGES="airflow" pre-commit run --hook-stage manual mypy
--all-files
+
.. note:: Mypy volume cache
MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps
the cache of last MyPy
@@ -135,6 +160,7 @@ require Breeze Docker image to be built locally.
is the hard problem in computer science). This might happen for example when
we upgrade MyPY. In such
cases you might need to manually remove the cache volume by running ``breeze
down --cleanup-mypy-cache``.
+
.. BEGIN AUTO-GENERATED STATIC CHECK LIST
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
@@ -258,9 +284,9 @@ require Breeze Docker image to be built locally.
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| codespell | Run codespell to
check for common misspellings in files | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| compile-www-assets | Compile www
assets | |
+| compile-www-assets | Compile www
assets (manual) | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| compile-www-assets-dev | Compile www
assets in dev mode | |
+| compile-www-assets-dev | Compile www
assets in dev mode (manual) | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| create-missing-init-py-files-tests | Create missing
init.py files in tests | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
@@ -316,6 +342,8 @@ require Breeze Docker image to be built locally.
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| mixed-line-ending | Detect if mixed
line ending is used (\r vs. \r\n) | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
+| mypy | Run mypy for
specified packages (manual) | * |
++-----------------------------------------------------------+--------------------------------------------------------------+---------+
| mypy-core | Run mypy for
core | * |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| mypy-dev | Run mypy for dev
| * |
diff --git a/airflow/decorators/__init__.pyi b/airflow/decorators/__init__.pyi
index 9c6f19c311..5574fa361e 100644
--- a/airflow/decorators/__init__.pyi
+++ b/airflow/decorators/__init__.pyi
@@ -62,7 +62,7 @@ __all__ = [
class TaskDecoratorCollection:
@overload
- def python(
+ def python( # type: ignore[misc]
self,
*,
multiple_outputs: bool | None = None,
@@ -90,7 +90,7 @@ class TaskDecoratorCollection:
def python(self, python_callable: Callable[FParams, FReturn]) ->
Task[FParams, FReturn]: ...
# [END mixin_for_typing]
@overload
- def __call__(
+ def __call__( # type: ignore[misc]
self,
*,
multiple_outputs: bool | None = None,
@@ -103,7 +103,7 @@ class TaskDecoratorCollection:
def __call__(self, python_callable: Callable[FParams, FReturn]) ->
Task[FParams, FReturn]:
"""Aliasing ``python``; signature should match exactly."""
@overload
- def virtualenv(
+ def virtualenv( # type: ignore[misc]
self,
*,
multiple_outputs: bool | None = None,
@@ -189,7 +189,9 @@ class TaskDecoratorCollection:
such as transmission a large amount of XCom to TaskAPI.
"""
@overload
- def branch(self, *, multiple_outputs: bool | None = None, **kwargs) ->
TaskDecorator:
+ def branch( # type: ignore[misc]
+ self, *, multiple_outputs: bool | None = None, **kwargs
+ ) -> TaskDecorator:
"""Create a decorator to wrap the decorated callable into a
BranchPythonOperator.
For more information on how to use this decorator, see
:ref:`concepts:branching`.
@@ -201,7 +203,7 @@ class TaskDecoratorCollection:
@overload
def branch(self, python_callable: Callable[FParams, FReturn]) ->
Task[FParams, FReturn]: ...
@overload
- def branch_virtualenv(
+ def branch_virtualenv( # type: ignore[misc]
self,
*,
multiple_outputs: bool | None = None,
@@ -294,7 +296,7 @@ class TaskDecoratorCollection:
self, python_callable: Callable[FParams, FReturn]
) -> Task[FParams, FReturn]: ...
@overload
- def short_circuit(
+ def short_circuit( # type: ignore[misc]
self,
*,
multiple_outputs: bool | None = None,
@@ -629,7 +631,7 @@ class TaskDecoratorCollection:
:param progress_callback: Callback function for receiving k8s
container logs.
"""
@overload
- def sensor(
+ def sensor( # type: ignore[misc]
self,
*,
poke_interval: float = ...,
@@ -666,7 +668,7 @@ class TaskDecoratorCollection:
@overload
def sensor(self, python_callable: Callable[FParams, FReturn] | None =
None) -> Task[FParams, FReturn]: ...
@overload
- def pyspark(
+ def pyspark( # type: ignore[misc]
self,
*,
multiple_outputs: bool | None = None,
@@ -688,7 +690,7 @@ class TaskDecoratorCollection:
self, python_callable: Callable[FParams, FReturn] | None = None
) -> Task[FParams, FReturn]: ...
@overload
- def bash(
+ def bash( # type: ignore[misc]
self,
*,
env: dict[str, str] | None = None,
diff --git a/airflow/models/taskreschedule.py b/airflow/models/taskreschedule.py
index a098001773..69356661e2 100644
--- a/airflow/models/taskreschedule.py
+++ b/airflow/models/taskreschedule.py
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
from airflow.models.operator import Operator
from airflow.models.taskinstance import TaskInstance
+ from airflow.serialization.pydantic.taskinstance import
TaskInstancePydantic
class TaskReschedule(TaskInstanceDependencies):
@@ -103,7 +104,7 @@ class TaskReschedule(TaskInstanceDependencies):
@classmethod
def stmt_for_task_instance(
cls,
- ti: TaskInstance,
+ ti: TaskInstance | TaskInstancePydantic,
*,
try_number: int | None = None,
descending: bool = False,
diff --git a/airflow/operators/latest_only.py b/airflow/operators/latest_only.py
index bca66dc2d4..7b4341b101 100644
--- a/airflow/operators/latest_only.py
+++ b/airflow/operators/latest_only.py
@@ -45,7 +45,7 @@ class LatestOnlyOperator(BaseBranchOperator):
def choose_branch(self, context: Context) -> str | Iterable[str]:
# If the DAG Run is externally triggered, then return without
# skipping downstream tasks
- dag_run: DagRun = context["dag_run"]
+ dag_run: DagRun = context["dag_run"] # type: ignore[assignment]
if dag_run.external_trigger:
self.log.info("Externally triggered DAG_Run: allowing execution to
proceed.")
return
list(context["task"].get_direct_relative_ids(upstream=False))
diff --git a/airflow/providers/apache/druid/hooks/druid.py
b/airflow/providers/apache/druid/hooks/druid.py
index 455c780f76..92d4aa6408 100644
--- a/airflow/providers/apache/druid/hooks/druid.py
+++ b/airflow/providers/apache/druid/hooks/druid.py
@@ -200,7 +200,7 @@ class DruidDbApiHook(DbApiHook):
endpoint = conn.extra_dejson.get("endpoint", "druid/v2/sql")
return f"{conn_type}://{host}/{endpoint}"
- def set_autocommit(self, conn: connect, autocommit: bool) ->
NotImplementedError:
+ def set_autocommit(self, conn: connect, autocommit: bool) -> None:
raise NotImplementedError()
def insert_rows(
@@ -211,5 +211,5 @@ class DruidDbApiHook(DbApiHook):
commit_every: int = 1000,
replace: bool = False,
**kwargs: Any,
- ) -> NotImplementedError:
+ ) -> None:
raise NotImplementedError()
diff --git a/airflow/providers/databricks/hooks/databricks_sql.py
b/airflow/providers/databricks/hooks/databricks_sql.py
index 6c31691c45..d18d804b04 100644
--- a/airflow/providers/databricks/hooks/databricks_sql.py
+++ b/airflow/providers/databricks/hooks/databricks_sql.py
@@ -174,7 +174,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
)
return self._sql_conn
- @overload
+ @overload # type: ignore[override]
def run(
self,
sql: str | Iterable[str],
@@ -249,7 +249,7 @@ class DatabricksSqlHook(BaseDatabricksHook, DbApiHook):
self.set_autocommit(conn, autocommit)
with closing(conn.cursor()) as cur:
- self._run_command(cur, sql_statement, parameters)
+ self._run_command(cur, sql_statement, parameters) # type:
ignore[attr-defined]
if handler is not None:
raw_result = handler(cur)
if self.return_tuple:
diff --git a/airflow/providers/exasol/hooks/exasol.py
b/airflow/providers/exasol/hooks/exasol.py
index 6d52b5122b..81c9381b3a 100644
--- a/airflow/providers/exasol/hooks/exasol.py
+++ b/airflow/providers/exasol/hooks/exasol.py
@@ -162,7 +162,7 @@ class ExasolHook(DbApiHook):
)
return cols
- @overload
+ @overload # type: ignore[override]
def run(
self,
sql: str | Iterable[str],
@@ -232,7 +232,9 @@ class ExasolHook(DbApiHook):
with closing(conn.execute(sql_statement, parameters)) as
exa_statement:
self.log.info("Running statement: %s, parameters: %s",
sql_statement, parameters)
if handler is not None:
- result =
self._make_common_data_structure(handler(exa_statement))
+ result = self._make_common_data_structure( # type:
ignore[attr-defined]
+ handler(exa_statement)
+ )
if return_single_query_results(sql, return_last,
split_statements):
_last_result = result
_last_columns = self.get_description(exa_statement)
diff --git a/airflow/providers/google/cloud/hooks/cloud_run.py
b/airflow/providers/google/cloud/hooks/cloud_run.py
index 8741aa1d4d..61a6fadb76 100644
--- a/airflow/providers/google/cloud/hooks/cloud_run.py
+++ b/airflow/providers/google/cloud/hooks/cloud_run.py
@@ -31,7 +31,7 @@ from google.cloud.run_v2 import (
RunJobRequest,
UpdateJobRequest,
)
-from google.longrunning import operations_pb2
+from google.longrunning import operations_pb2 # type: ignore[attr-defined]
from airflow.exceptions import AirflowException
from airflow.providers.google.common.consts import CLIENT_INFO
diff --git a/airflow/providers/google/cloud/operators/bigquery.py
b/airflow/providers/google/cloud/operators/bigquery.py
index c9fdee102e..18bd065cb2 100644
--- a/airflow/providers/google/cloud/operators/bigquery.py
+++ b/airflow/providers/google/cloud/operators/bigquery.py
@@ -449,7 +449,7 @@ class BigQueryValueCheckOperator(_BigQueryDbHookMixin,
SQLValueCheckOperator):
# job.result() returns a RowIterator. Mypy expects an instance of
SupportsNext[Any] for
# the next() call which the RowIterator does not resemble to.
Hence, ignore the arg-type error.
records = next(job.result()) # type: ignore[arg-type]
- self.check_value(records)
+ self.check_value(records) # type: ignore[attr-defined]
self.log.info("Current state of job %s is %s", job.job_id,
job.state)
@staticmethod
diff --git a/airflow/providers/google/cloud/triggers/cloud_run.py
b/airflow/providers/google/cloud/triggers/cloud_run.py
index f47a7ac1b3..c0d3458c12 100644
--- a/airflow/providers/google/cloud/triggers/cloud_run.py
+++ b/airflow/providers/google/cloud/triggers/cloud_run.py
@@ -25,7 +25,7 @@ from airflow.providers.google.cloud.hooks.cloud_run import
CloudRunAsyncHook
from airflow.triggers.base import BaseTrigger, TriggerEvent
if TYPE_CHECKING:
- from google.longrunning import operations_pb2
+ from google.longrunning import operations_pb2 # type: ignore[attr-defined]
DEFAULT_BATCH_LOCATION = "us-central1"
diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py
b/airflow/providers/google/cloud/utils/credentials_provider.py
index 18c00b0844..df83358e4b 100644
--- a/airflow/providers/google/cloud/utils/credentials_provider.py
+++ b/airflow/providers/google/cloud/utils/credentials_provider.py
@@ -30,7 +30,7 @@ from urllib.parse import urlencode
import google.auth
import google.auth.credentials
import google.oauth2.service_account
-from google.auth import impersonated_credentials
+from google.auth import impersonated_credentials # type: ignore[attr-defined]
from google.auth.environment_vars import CREDENTIALS, LEGACY_PROJECT, PROJECT
from airflow.exceptions import AirflowException
diff --git a/airflow/providers/google/common/hooks/base_google.py
b/airflow/providers/google/common/hooks/base_google.py
index 72c51212ac..b03eaf7547 100644
--- a/airflow/providers/google/common/hooks/base_google.py
+++ b/airflow/providers/google/common/hooks/base_google.py
@@ -36,7 +36,7 @@ import requests
import tenacity
from asgiref.sync import sync_to_async
from google.api_core.exceptions import Forbidden, ResourceExhausted,
TooManyRequests
-from google.auth import _cloud_sdk, compute_engine
+from google.auth import _cloud_sdk, compute_engine # type:
ignore[attr-defined]
from google.auth.environment_vars import CLOUD_SDK_CONFIG_DIR, CREDENTIALS
from google.auth.exceptions import RefreshError
from google.auth.transport import _http_client
diff --git a/airflow/providers/google/common/utils/id_token_credentials.py
b/airflow/providers/google/common/utils/id_token_credentials.py
index 6a41438a22..c2b78bb2e1 100644
--- a/airflow/providers/google/common/utils/id_token_credentials.py
+++ b/airflow/providers/google/common/utils/id_token_credentials.py
@@ -36,7 +36,7 @@ from typing import TYPE_CHECKING
import google.auth.transport
from google.auth import credentials as google_auth_credentials,
environment_vars, exceptions
-from google.oauth2 import credentials as oauth2_credentials, service_account
+from google.oauth2 import credentials as oauth2_credentials, service_account
# type: ignore[attr-defined]
if TYPE_CHECKING:
import google.oauth2
@@ -146,7 +146,7 @@ def _get_gcloud_sdk_credentials(
target_audience: str | None,
) -> google_auth_credentials.Credentials | None:
"""Gets the credentials and project ID from the Cloud SDK."""
- from google.auth import _cloud_sdk
+ from google.auth import _cloud_sdk # type: ignore[attr-defined]
# Check if application default credentials exist.
credentials_filename =
_cloud_sdk.get_application_default_credentials_path()
diff --git a/airflow/providers/grpc/hooks/grpc.py
b/airflow/providers/grpc/hooks/grpc.py
index b0bec1b9ce..a6a3ca1271 100644
--- a/airflow/providers/grpc/hooks/grpc.py
+++ b/airflow/providers/grpc/hooks/grpc.py
@@ -21,7 +21,7 @@ from typing import Any, Callable, Generator
import grpc
from google import auth as google_auth
-from google.auth import jwt as google_auth_jwt
+from google.auth import jwt as google_auth_jwt # type: ignore[attr-defined]
from google.auth.transport import (
grpc as google_auth_transport_grpc,
requests as google_auth_transport_requests,
diff --git a/airflow/providers/postgres/hooks/postgres.py
b/airflow/providers/postgres/hooks/postgres.py
index 76ce4dacef..481733ece9 100644
--- a/airflow/providers/postgres/hooks/postgres.py
+++ b/airflow/providers/postgres/hooks/postgres.py
@@ -331,7 +331,9 @@ class PostgresHook(DbApiHook):
if is_redshift:
authority =
self._get_openlineage_redshift_authority_part(connection)
else:
- authority = DbApiHook.get_openlineage_authority_part(connection,
default_port=5432)
+ authority = DbApiHook.get_openlineage_authority_part( # type:
ignore[attr-defined]
+ connection, default_port=5432
+ )
return DatabaseInfo(
scheme="postgres" if not is_redshift else "redshift",
diff --git a/airflow/providers/snowflake/hooks/snowflake.py
b/airflow/providers/snowflake/hooks/snowflake.py
index 4b0d13b5e5..cd1dc87165 100644
--- a/airflow/providers/snowflake/hooks/snowflake.py
+++ b/airflow/providers/snowflake/hooks/snowflake.py
@@ -300,7 +300,7 @@ class SnowflakeHook(DbApiHook):
def get_autocommit(self, conn):
return getattr(conn, "autocommit_mode", False)
- @overload
+ @overload # type: ignore[override]
def run(
self,
sql: str | Iterable[str],
@@ -385,10 +385,10 @@ class SnowflakeHook(DbApiHook):
with self._get_cursor(conn, return_dictionaries) as cur:
results = []
for sql_statement in sql_list:
- self._run_command(cur, sql_statement, parameters)
+ self._run_command(cur, sql_statement, parameters) # type:
ignore[attr-defined]
if handler is not None:
- result = self._make_common_data_structure(handler(cur))
+ result =
self._make_common_data_structure(handler(cur)) # type: ignore[attr-defined]
if return_single_query_results(sql, return_last,
split_statements):
_last_result = result
_last_description = cur.description
diff --git a/airflow/providers/trino/operators/trino.py
b/airflow/providers/trino/operators/trino.py
index 7f90bf9947..20798977b6 100644
--- a/airflow/providers/trino/operators/trino.py
+++ b/airflow/providers/trino/operators/trino.py
@@ -65,11 +65,11 @@ class TrinoOperator(SQLExecuteQueryOperator):
)
def on_kill(self) -> None:
- if self._hook is not None and isinstance(self._hook, TrinoHook):
- query_id = "'" + self._hook.query_id + "'"
+ if self._hook is not None and isinstance(self._hook, TrinoHook): #
type: ignore[attr-defined]
+ query_id = "'" + self._hook.query_id + "'" # type:
ignore[attr-defined]
try:
- self.log.info("Stopping query run with queryId - %s",
self._hook.query_id)
- self._hook.run(
+ self.log.info("Stopping query run with queryId - %s",
self._hook.query_id) # type: ignore[attr-defined]
+ self._hook.run( # type: ignore[attr-defined]
sql=f"CALL system.runtime.kill_query(query_id =>
{query_id},message => 'Job "
f"killed by "
f"user');",
diff --git a/airflow/providers/vertica/hooks/vertica.py
b/airflow/providers/vertica/hooks/vertica.py
index 91672e2aec..191fa10a8b 100644
--- a/airflow/providers/vertica/hooks/vertica.py
+++ b/airflow/providers/vertica/hooks/vertica.py
@@ -132,7 +132,7 @@ class VerticaHook(DbApiHook):
conn = connect(**conn_config)
return conn
- @overload
+ @overload # type: ignore[override]
def run(
self,
sql: str | Iterable[str],
diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index 162f2d32af..979bd2ecf1 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -404,7 +404,7 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
return ProvidersManager._initialized
@staticmethod
- def initialization_stack_trace() -> str:
+ def initialization_stack_trace() -> str | None:
return ProvidersManager._initialization_stack_trace
def __init__(self):
@@ -418,7 +418,7 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
# Keeps dict of hooks keyed by connection type
self._hooks_dict: dict[str, HookInfo] = {}
self._fs_set: set[str] = set()
- self._taskflow_decorators: dict[str, Callable] = LazyDictWithCache()
+ self._taskflow_decorators: dict[str, Callable] = LazyDictWithCache()
# type: ignore[assignment]
# keeps mapping between connection_types and hook class, package they
come from
self._hook_provider_dict: dict[str, HookClassProvider] = {}
# Keeps dict of hooks keyed by connection type. They are lazy
evaluated at access time
@@ -1120,7 +1120,9 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
"""Retrieve all configs defined in the providers."""
for provider_package, provider in self._provider_dict.items():
if provider.data.get("config"):
- self._provider_configs[provider_package] =
provider.data.get("config")
+ self._provider_configs[provider_package] = (
+ provider.data.get("config") # type: ignore[assignment]
+ )
def _discover_plugins(self) -> None:
"""Retrieve all plugins defined in the providers."""
@@ -1196,7 +1198,7 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
@property
def taskflow_decorators(self) -> dict[str, TaskDecorator]:
self.initialize_providers_taskflow_decorator()
- return self._taskflow_decorators
+ return self._taskflow_decorators # type: ignore[return-value]
@property
def extra_links_class_names(self) -> list[str]:
diff --git a/airflow/sensors/base.py b/airflow/sensors/base.py
index 8bf40c913b..d1a01ac87e 100644
--- a/airflow/sensors/base.py
+++ b/airflow/sensors/base.py
@@ -212,7 +212,9 @@ class BaseSensorOperator(BaseOperator, SkipMixin):
if self.reschedule:
# If reschedule, use the start date of the first try (first try
can be either the very
# first execution of the task, or the first execution after the
task was cleared.)
- first_try_number = context["ti"].max_tries - self.retries + 1
+ max_tries: int = context["ti"].max_tries or 0
+ retries: int = self.retries or 0
+ first_try_number = max_tries - retries + 1
with create_session() as session:
start_date = session.scalar(
TaskReschedule.stmt_for_task_instance(
diff --git a/airflow/utils/operator_helpers.py
b/airflow/utils/operator_helpers.py
index ef1f05e304..5f92467147 100644
--- a/airflow/utils/operator_helpers.py
+++ b/airflow/utils/operator_helpers.py
@@ -172,7 +172,7 @@ class KeywordParameters:
def unpacking(self) -> Mapping[str, Any]:
"""Dump the kwargs mapping to unpack with ``**`` in a function call."""
- if self._wildcard and isinstance(self._kwargs, Context):
+ if self._wildcard and isinstance(self._kwargs, Context): # type:
ignore[misc]
return lazy_mapping_from_context(self._kwargs)
return self._kwargs
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index 00b94a86e7..469ecd9cc9 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -103,6 +103,7 @@ PRE_COMMIT_LIST = [
"lint-markdown",
"lint-openapi",
"mixed-line-ending",
+ "mypy",
"mypy-core",
"mypy-dev",
"mypy-docs",
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 9a184b822b..79bae9f8ce 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -563,6 +563,43 @@ class SelectiveChecks:
)
return False
+ @cached_property
+ def mypy_packages(self) -> list[str]:
+ packages_to_run: list[str] = []
+ if (
+ self._matching_files(
+ FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES,
CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
+ )
+ or self.full_tests_needed
+ ):
+ packages_to_run.append("airflow")
+ if (
+ self._matching_files(
+ FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES,
CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES
+ )
+ or self._are_all_providers_affected()
+ ) and self._default_branch == "main":
+ packages_to_run.append("airflow/providers")
+ if (
+ self._matching_files(
+ FileGroupForCi.ALL_DOCS_PYTHON_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
+ )
+ or self.full_tests_needed
+ ):
+ packages_to_run.append("docs")
+ if (
+ self._matching_files(
+ FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
+ )
+ or self.full_tests_needed
+ ):
+ packages_to_run.append("dev")
+ return packages_to_run
+
+ @cached_property
+ def needs_mypy(self) -> bool:
+ return self.mypy_packages != []
+
@cached_property
def needs_python_scans(self) -> bool:
return self._should_be_run(FileGroupForCi.PYTHON_PRODUCTION_FILES)
@@ -802,6 +839,14 @@ class SelectiveChecks:
def skip_pre_commits(self) -> str:
pre_commits_to_skip = set()
pre_commits_to_skip.add("identity")
+ # Skip all mypy "individual" file checks if we are running mypy checks
in CI
+ # In the CI we always run mypy for the whole "package" rather than for
`--all-files` because
+ # The pre-commit will semi-randomly skip such list of files into
several groups and we want
+ # to make sure that such checks are always run in CI for whole "group"
of files - i.e.
+ # whole package rather than for individual files. That's why we skip
those checks in CI
+ # and run them via `mypy-all` command instead and dedicated CI job in
matrix
+ # This will also speed up static-checks job usually as the jobs will
be running in parallel
+ pre_commits_to_skip.update({"mypy-providers", "mypy-core",
"mypy-docs", "mypy-dev"})
if self._default_branch != "main":
# Skip those tests on all "release" branches
pre_commits_to_skip.update(
@@ -810,29 +855,13 @@ class SelectiveChecks:
"check-extra-packages-references",
"check-provider-yaml-valid",
"lint-helm-chart",
- "mypy-providers",
)
)
+
if self.full_tests_needed:
# when full tests are needed, we do not want to skip any checks
and we should
# run all the pre-commits just to be sure everything is ok when
some structural changes occurred
return ",".join(sorted(pre_commits_to_skip))
- if not self._matching_files(
- FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
- ):
- pre_commits_to_skip.add("mypy-providers")
- if not self._matching_files(
- FileGroupForCi.ALL_AIRFLOW_PYTHON_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
- ):
- pre_commits_to_skip.add("mypy-core")
- if not self._matching_files(
- FileGroupForCi.ALL_DOCS_PYTHON_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
- ):
- pre_commits_to_skip.add("mypy-docs")
- if not self._matching_files(
- FileGroupForCi.ALL_DEV_PYTHON_FILES, CI_FILE_GROUP_MATCHES,
CI_FILE_GROUP_EXCLUDES
- ):
- pre_commits_to_skip.add("mypy-dev")
if not self._matching_files(FileGroupForCi.WWW_FILES,
CI_FILE_GROUP_MATCHES, CI_FILE_GROUP_EXCLUDES):
pre_commits_to_skip.add("ts-compile-format-lint-www")
if not self._matching_files(
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 8c11e2628e..667342267e 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -75,7 +75,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
print_in_color("\nOutput received:")
print_in_color(received_output_as_dict)
print_in_color()
- assert expected_value == received_value
+ assert received_value == expected_value
else:
print(
f"\n[red]ERROR: The key '{expected_key}' missing but "
@@ -111,6 +111,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": None,
+ "needs-mypy": "false",
+ "mypy-packages": "[]",
},
id="No tests on simple change",
)
@@ -130,10 +132,12 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,"
"mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "API Always",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Only API tests and DOCS should run",
)
@@ -153,10 +157,12 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,"
"mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always Operators",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Only Operator tests and DOCS should run",
)
@@ -176,11 +182,13 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,"
"mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
BranchExternalPython BranchPythonVenv "
"ExternalPython Operators PythonVenv",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Only Python tests",
)
@@ -200,10 +208,12 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,"
"mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Serialization",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Only Serialization tests",
)
@@ -227,11 +237,13 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"identity,lint-helm-chart,mypy-dev,mypy-docs,"
+ "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,"
"ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "API Always
Providers[amazon] "
"Providers[common.sql,openlineage,pgvector,postgres]
Providers[google]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers']",
},
id="API and providers tests and docs should run",
)
@@ -251,11 +263,13 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "false",
- "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,"
+ "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,"
"ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Providers[apache.beam] Providers[google]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Selected Providers and docs should run",
)
@@ -280,6 +294,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": None,
+ "needs-mypy": "false",
+ "mypy-packages": "[]",
},
id="Only docs builds should run - no tests needed",
)
@@ -303,11 +319,13 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Providers[amazon] "
"Providers[common.sql,openlineage,pgvector,postgres]
Providers[google]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Helm tests, providers (both upstream and downstream),"
"kubernetes tests and docs should run",
@@ -333,11 +351,13 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always "
"Providers[airbyte,apache.livy,dbt.cloud,dingding,discord,http]
Providers[amazon]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Helm tests, http and all relevant providers, kubernetes
tests and "
"docs should run even if unimportant files were added",
@@ -362,10 +382,12 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "true",
- "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Providers[airbyte,http]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Helm tests, airbyte/http providers, kubernetes tests and "
"docs should run even if unimportant files were added",
@@ -389,12 +411,14 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"needs-helm-tests": "true",
"run-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,mypy-dev,"
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,mypy-core,mypy-dev,"
"mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-amazon-tests": "false",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Docs should run even if unimportant files were added and
prod image "
"should be build for chart changes",
@@ -416,9 +440,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-amazon-tests": "true",
"docs-build": "true",
"full-tests-needed": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "true",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="Everything should run - including all providers and
upgrading to "
"newer requirements as setup.py changed and all Python
versions",
@@ -440,9 +466,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-amazon-tests": "true",
"docs-build": "true",
"full-tests-needed": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "true",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="Everything should run and upgrading to newer requirements
as dependencies change",
)
@@ -462,13 +490,15 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"run-amazon-tests": "true",
"parallel-test-types-list-as-string": "Always
Providers[amazon] "
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,"
"imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh]
Providers[google]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Providers tests run including amazon tests if amazon provider
files changed",
),
@@ -486,10 +516,12 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "false",
"docs-build": "false",
- "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Providers[airbyte,http]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Providers tests run without amazon tests if no amazon file
changed",
),
@@ -509,12 +541,14 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
"run-tests": "true",
"run-amazon-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Providers[amazon] "
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,"
"imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh]
Providers[google]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
},
id="Providers tests run including amazon tests if amazon provider
files changed",
),
@@ -537,8 +571,11 @@ def assert_outputs_are_printed(expected_outputs: dict[str,
str], stderr: str):
"run-amazon-tests": "false",
"docs-build": "false",
"run-kubernetes-tests": "false",
+ "skip-pre-commits":
"identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Always
Providers[common.io,openlineage]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers']",
},
id="Only Always and Common.IO tests should run when only common.io
and tests/always changed",
),
@@ -578,9 +615,11 @@ def test_expected_output_pull_request_main(
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"full-tests-needed": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="Everything should run including all providers when full
tests are needed",
)
@@ -605,9 +644,11 @@ def test_expected_output_pull_request_main(
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"full-tests-needed": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="Everything should run including full providers when full "
"tests are needed even with different label set as well",
@@ -630,9 +671,11 @@ def test_expected_output_pull_request_main(
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"full-tests-needed": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="Everything should run including full providers when"
"full tests are needed even if no files are changed",
@@ -655,14 +698,14 @@ def test_expected_output_pull_request_main(
"docs-build": "true",
"docs-list-as-string": "apache-airflow docker-stack",
"full-tests-needed": "true",
- "skip-pre-commits": "check-airflow-provider-compatibility,"
-
"check-extra-packages-references,check-provider-yaml-valid,identity,"
- "lint-helm-chart,mypy-providers",
+ "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"skip-provider-tests": "true",
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "API Always
BranchExternalPython "
"BranchPythonVenv CLI Core ExternalPython Operators Other
PlainAsserts "
"PythonVenv Serialization WWW",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'docs', 'dev']",
},
id="Everything should run except Providers and lint pre-commit
"
"when full tests are needed for non-main branch",
@@ -707,6 +750,8 @@ def test_expected_output_full_tests_needed(
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "true",
"parallel-test-types-list-as-string": None,
+ "needs-mypy": "false",
+ "mypy-packages": "[]",
},
id="Nothing should run if only non-important files changed",
),
@@ -735,6 +780,8 @@ def test_expected_output_full_tests_needed(
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "true",
"parallel-test-types-list-as-string": "Always",
+ "needs-mypy": "false",
+ "mypy-packages": "[]",
},
id="No Helm tests, No providers no lint charts, should run if "
"only chart/providers changed in non-main but PROD image should be
built",
@@ -759,13 +806,13 @@ def test_expected_output_full_tests_needed(
"docs-build": "true",
"docs-list-as-string": "apache-airflow docker-stack",
"full-tests-needed": "false",
- "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,"
- "check-provider-yaml-valid,identity,lint-helm-chart,"
- "mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "true",
"parallel-test-types-list-as-string": "Always CLI",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Only CLI tests and Kubernetes tests should run if cli/chart
files changed in non-main branch",
),
@@ -788,11 +835,11 @@ def test_expected_output_full_tests_needed(
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "true",
- "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,"
- "check-provider-yaml-valid,identity,lint-helm-chart,"
- "mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"parallel-test-types-list-as-string": "API Always
BranchExternalPython BranchPythonVenv "
"CLI Core ExternalPython Operators Other PlainAsserts
PythonVenv Serialization WWW",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="All tests except Providers and helm lint pre-commit "
"should run if core file changed in non-main branch",
@@ -832,6 +879,8 @@ def test_expected_output_pull_request_v2_7(
"mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"skip-provider-tests": "true",
"parallel-test-types-list-as-string": None,
+ "needs-mypy": "false",
+ "mypy-packages": "[]",
},
id="Nothing should run if only non-important files changed",
),
@@ -847,11 +896,12 @@ def test_expected_output_pull_request_v2_7(
"run-tests": "true",
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,"
- "mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "true",
"parallel-test-types-list-as-string": "Always",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="Only Always and docs build should run if only system tests
changed",
),
@@ -877,7 +927,7 @@ def test_expected_output_pull_request_v2_7(
"cncf.kubernetes common.sql facebook google hashicorp
microsoft.azure "
"microsoft.mssql mysql openlineage oracle postgres "
"presto salesforce samba sftp ssh trino",
- "skip-pre-commits":
"identity,mypy-dev,mypy-docs,ts-compile-format-lint-www",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "true",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "false",
@@ -885,6 +935,8 @@ def test_expected_output_pull_request_v2_7(
"Providers[apache.beam,apache.cassandra,cncf.kubernetes,common.sql,facebook,hashicorp,"
"microsoft.azure,microsoft.mssql,mysql,openlineage,oracle,postgres,presto,salesforce,"
"samba,sftp,ssh,trino] Providers[google]",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers']",
},
id="CLI tests and Google-related provider tests should run if
cli/chart files changed but "
"prod image should be build too and k8s tests too",
@@ -906,12 +958,13 @@ def test_expected_output_pull_request_v2_7(
"run-tests": "true",
"docs-build": "true",
"docs-list-as-string": "apache-airflow",
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
- "mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "true",
"parallel-test-types-list-as-string": "API Always CLI
Operators WWW",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
},
id="No providers tests should run if only CLI/API/Operators/WWW
file changed",
),
@@ -927,12 +980,13 @@ def test_expected_output_pull_request_v2_7(
"run-tests": "true",
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
- "mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "false",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers']",
},
id="Tests for all providers should run if model file changed",
),
@@ -948,12 +1002,13 @@ def test_expected_output_pull_request_v2_7(
"run-tests": "true",
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
- "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-dev,"
- "mypy-docs,mypy-providers,ts-compile-format-lint-www",
+ "skip-pre-commits":
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers,ts-compile-format-lint-www",
"run-kubernetes-tests": "false",
"upgrade-to-newer-dependencies": "false",
"skip-provider-tests": "false",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers']",
},
id="Tests for all providers should run if any other than
API/WWW/CLI/Operators file changed.",
),
@@ -990,9 +1045,11 @@ def test_expected_output_pull_request_target(
"run-tests": "true",
"docs-build": "true",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "true",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="All tests run on push even if unimportant file changed",
),
@@ -1009,12 +1066,13 @@ def test_expected_output_pull_request_target(
"needs-helm-tests": "false",
"run-tests": "true",
"docs-build": "true",
- "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,"
-
"check-provider-yaml-valid,identity,lint-helm-chart,mypy-providers",
+ "skip-pre-commits":
"check-airflow-provider-compatibility,check-extra-packages-references,check-provider-yaml-valid,identity,lint-helm-chart,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"docs-list-as-string": "apache-airflow docker-stack",
"upgrade-to-newer-dependencies": "true",
"parallel-test-types-list-as-string": "API Always
BranchExternalPython BranchPythonVenv "
"CLI Core ExternalPython Operators Other PlainAsserts
PythonVenv Serialization WWW",
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'docs', 'dev']",
},
id="All tests except Providers and Helm run on push"
" even if unimportant file changed in non-main branch",
@@ -1032,10 +1090,12 @@ def test_expected_output_pull_request_target(
"needs-helm-tests": "true",
"run-tests": "true",
"docs-build": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"docs-list-as-string": ALL_DOCS_SELECTED_FOR_BUILD,
"upgrade-to-newer-dependencies": "true",
"parallel-test-types-list-as-string":
ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
},
id="All tests run on push if core file changed",
),
@@ -1084,11 +1144,13 @@ def
test_no_commit_provided_trigger_full_build_for_any_event_type(github_event):
"needs-helm-tests": "true",
"run-tests": "true",
"docs-build": "true",
- "skip-pre-commits": "identity",
+ "skip-pre-commits":
"identity,mypy-core,mypy-dev,mypy-docs,mypy-providers",
"upgrade-to-newer-dependencies": "true"
if github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE]
else "false",
"parallel-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES,
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs', 'dev']",
},
str(stderr),
)
@@ -1577,3 +1639,91 @@ def test_provider_compatibility_checks(labels:
tuple[str, ...], expected_outputs
default_branch="main",
)
assert_outputs_are_printed(expected_outputs, str(stderr))
+
+
[email protected](
+ "files, expected_outputs, default_branch, pr_labels",
+ [
+ pytest.param(
+ ("README.md",),
+ {
+ "needs-mypy": "false",
+ "mypy-packages": "[]",
+ },
+ "main",
+ (),
+ id="No mypy checks on non-python files",
+ ),
+ pytest.param(
+ ("airflow/cli/file.py",),
+ {
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow']",
+ },
+ "main",
+ (),
+ id="Airflow mypy checks on airflow regular files",
+ ),
+ pytest.param(
+ ("airflow/models/file.py",),
+ {
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers']",
+ },
+ "main",
+ (),
+ id="Airflow mypy checks on airflow files that can trigger provider
tests",
+ ),
+ pytest.param(
+ ("airflow/providers/a_file.py",),
+ {
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow/providers']",
+ },
+ "main",
+ (),
+ id="Airflow mypy checks on provider files",
+ ),
+ pytest.param(
+ ("docs/a_file.py",),
+ {
+ "needs-mypy": "true",
+ "mypy-packages": "['docs']",
+ },
+ "main",
+ (),
+ id="Doc checks on doc files",
+ ),
+ pytest.param(
+ ("dev/a_package/a_file.py",),
+ {
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
+ },
+ "main",
+ (),
+ id="All mypy checks on def files changed (full tests needed are
implicit)",
+ ),
+ pytest.param(
+ ("readme.md",),
+ {
+ "needs-mypy": "true",
+ "mypy-packages": "['airflow', 'airflow/providers', 'docs',
'dev']",
+ },
+ "main",
+ ("full tests needed",),
+ id="All mypy checks on full tests needed",
+ ),
+ ],
+)
+def test_mypy_matches(
+ files: tuple[str, ...], expected_outputs: dict[str, str], default_branch:
str, pr_labels: tuple[str, ...]
+):
+ stderr = SelectiveChecks(
+ files=files,
+ commit_ref="HEAD",
+ default_branch=default_branch,
+ github_event=GithubEvents.PULL_REQUEST,
+ pr_labels=pr_labels,
+ )
+ assert_outputs_are_printed(expected_outputs, str(stderr))
diff --git a/images/breeze/output_static-checks.svg
b/images/breeze/output_static-checks.svg
index 6d16c9a61b..be61aa2829 100644
--- a/images/breeze/output_static-checks.svg
+++ b/images/breeze/output_static-checks.svg
@@ -343,18 +343,18 @@
</text><text class="breeze-static-checks-r5" x="0" y="922.8" textLength="12.2"
clip-path="url(#breeze-static-checks-line-37)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="922.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-37)">doctoc | end-of-file-fixer | fix-encoding-pragma | flynt |                       </t
[...]
</text><text class="breeze-static-checks-r5" x="0" y="947.2" textLength="12.2"
clip-path="url(#breeze-static-checks-line-38)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="947.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-38)">generate-airflow-diagrams | generate-pypi-readme | identity | insert-license |   </text><text
class="breeze-static-checks-r5" x="1451.8" y="947.2" textLength="12.2"
clip-path="url(#bre [...]
</text><text class="breeze-static-checks-r5" x="0" y="971.6" textLength="12.2"
clip-path="url(#breeze-static-checks-line-39)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="971.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-39)">lint-chart-schema | lint-css | lint-dockerfile | lint-helm-chart |               </text><text
class="breeze-static-checks-r5 [...]
-</text><text class="breeze-static-checks-r5" x="0" y="996" textLength="12.2"
clip-path="url(#breeze-static-checks-line-40)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="996" textLength="988.2"
clip-path="url(#breeze-static-checks-line-40)">lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy-core |</text><text
class="breeze-static-checks-r5" x="1451.8" y="996" textLength="12.2"
clip-path="url(#breeze-static- [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1020.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-41)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1020.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-41)">mypy-dev | mypy-docs | mypy-providers | pretty-format-json | python-no-log-warn |</text><text
class="breeze-static-checks-r5" x="1451.8" y="1020.4" textLength="12.2"
clip-path="url(#breez [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1044.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-42)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1044.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-42)">replace-bad-characters | rst-backticks | ruff | ruff-format | shellcheck |       </text><text
class="breeze-static-checks-r5" x="1451.8" y="1044.8" text [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1069.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-43)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1069.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-43)">trailing-whitespace | ts-compile-format-lint-www | update-black-version |        </text><text
class="breeze-static-checks-r5" x="1451.8" y="1069.2" textLength="12.2" c [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1093.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-44)">update-breeze-cmd-output | update-breeze-readme-config-hash |                    </text><text
class="breeze-static-ch [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-static-checks-line-45)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2"
clip-path="url(#breeze-static-checks-line-45)">update-common-sql-api-stubs | update-er-diagram | update-extras |                </text><text
class="breeze-static-checks-r5" x="14 [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1142.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-46)">update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            </text><text
class="breeze-static-checks-r5" x="1451.8" y="1142.4" textLengt [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1166.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-47)">update-installed-providers-to-be-sorted | update-local-yml-file |                </text><text
class="breeze-static-checks-r5" x="1451.8" [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1191.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-48)">update-migration-references | update-providers-dependencies |                    </text><text
class="breeze-static-ch [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1215.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-49)">update-spelling-wordlist-to-be-sorted | update-supported-versions |              </text><text
class="breeze-static-checks-r5" x="1451.8" y="1215.6" [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2"
clip-path="url(#breeze-static-checks-line-50)">update-vendored-in-k8s-json-schema | update-version | validate-pyproject |       </text><text
class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2"
clip-path="u [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1264.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-51)">yamllint)                                     &
[...]
+</text><text class="breeze-static-checks-r5" x="0" y="996" textLength="12.2"
clip-path="url(#breeze-static-checks-line-40)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="996" textLength="988.2"
clip-path="url(#breeze-static-checks-line-40)">lint-json-schema | lint-markdown | lint-openapi | mixed-line-ending | mypy |     </text><text
class="breeze-static-checks-r5" x="1451.8" y="996" textLength="12.2" clip-
[...]
+</text><text class="breeze-static-checks-r5" x="0" y="1020.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-41)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1020.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-41)">mypy-core | mypy-dev | mypy-docs | mypy-providers | pretty-format-json |         </text><text
class="breeze-static-checks-r5" x="1451.8" y="10 [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1044.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-42)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1044.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-42)">python-no-log-warn | replace-bad-characters | rst-backticks | ruff | ruff-format </text><text
class="breeze-static-checks-r5" x="1451.8" y="1044.8" textLength="12.2"
clip-path="url(#breez [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1069.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-43)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1069.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-43)">| shellcheck | trailing-whitespace | ts-compile-format-lint-www |                </text><text
class="breeze-static-checks- [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1093.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-44)">update-black-version | update-breeze-cmd-output |                            
[...]
+</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-static-checks-line-45)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2"
clip-path="url(#breeze-static-checks-line-45)">update-breeze-readme-config-hash | update-common-sql-api-stubs |                 </text><text
class="breeze-static-checks-r5" x="1451.8" [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1142.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-46)">update-er-diagram | update-extras | update-in-the-wild-to-be-sorted |            </text><text
class="breeze-static-checks-r5" x="1451.8" y="1142.4" [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1166.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-47)">update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |    </text><text
class="breeze-static-checks-r5" x="1451.8" y="1166.8" textLength="12.2"
clip-path="url(#breeze-static-c [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1191.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-48)">update-local-yml-file | update-migration-references |                           &#
[...]
+</text><text class="breeze-static-checks-r5" x="0" y="1215.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-49)">update-providers-dependencies | update-spelling-wordlist-to-be-sorted |          </text><text
class="breeze-static-checks-r5" x="1451.8" y="1215.6" textLength="12.2" c [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2"
clip-path="url(#breeze-static-checks-line-50)">update-supported-versions | update-vendored-in-k8s-json-schema | update-version |</text><text
class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)"> [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1264.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-51)">validate-pyproject | yamllint)                                
[...]
</text><text class="breeze-static-checks-r5" x="0" y="1288.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">│</text><text
class="breeze-static-checks-r4" x="24.4" y="1288.8" textLength="12.2"
clip-path="url(#breeze-static-checks-line-52)">-</text><text
class="breeze-static-checks-r4" x="36.6" y="1288.8" textLength="61"
clip-path="url(#breeze-static-checks-line-52)">-show</text><text
class="breeze-static-checks-r4" x="97.6" y="1288.8" textLength="195.2"
clip-path="url(# [...]
</text><text class="breeze-static-checks-r5" x="0" y="1313.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text
class="breeze-static-checks-r4" x="24.4" y="1313.2" textLength="12.2"
clip-path="url(#breeze-static-checks-line-53)">-</text><text
class="breeze-static-checks-r4" x="36.6" y="1313.2" textLength="134.2"
clip-path="url(#breeze-static-checks-line-53)">-initialize</text><text
class="breeze-static-checks-r4" x="170.8" y="1313.2" textLength="146.4" clip-p
[...]
</text><text class="breeze-static-checks-r5" x="0" y="1337.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">│</text><text
class="breeze-static-checks-r4" x="24.4" y="1337.6" textLength="12.2"
clip-path="url(#breeze-static-checks-line-54)">-</text><text
class="breeze-static-checks-r4" x="36.6" y="1337.6" textLength="48.8"
clip-path="url(#breeze-static-checks-line-54)">-max</text><text
class="breeze-static-checks-r4" x="85.4" y="1337.6" textLength="292.8"
clip-path="url( [...]
diff --git a/images/breeze/output_static-checks.txt
b/images/breeze/output_static-checks.txt
index cba6394a9c..9ae25e18c1 100644
--- a/images/breeze/output_static-checks.txt
+++ b/images/breeze/output_static-checks.txt
@@ -1 +1 @@
-cc154a8e6d64f6034782bac9898f3a05
+d5902ce9c52ac5338e019056b557da73
diff --git a/scripts/ci/pre_commit/common_precommit_utils.py
b/scripts/ci/pre_commit/common_precommit_utils.py
index 76370b15ad..8926bc1823 100644
--- a/scripts/ci/pre_commit/common_precommit_utils.py
+++ b/scripts/ci/pre_commit/common_precommit_utils.py
@@ -45,12 +45,32 @@ def read_airflow_version() -> str:
raise RuntimeError("Couldn't find __version__ in AST")
-def filter_out_providers_on_non_main_branch(files: list[str]) -> list[str]:
- """When running build on non-main branch do not take providers into
account"""
+def pre_process_files(files: list[str]) -> list[str]:
+ """Pre-process files passed to mypy.
+
+ * When running build on non-main branch do not take providers into account.
+ * When running "airflow/providers" package, then we need to add
--namespace-packages flag.
+ * When running "airflow" package, then we need to exclude providers.
+ """
default_branch = os.environ.get("DEFAULT_BRANCH")
if not default_branch or default_branch == "main":
return files
- return [file for file in files if not
file.startswith(f"airflow{os.sep}providers")]
+ result = [file for file in files if not
file.startswith(f"airflow{os.sep}providers")]
+ if "airflow/providers" in files:
+ if len(files) > 1:
+ raise RuntimeError(
+ "When running `airflow/providers` package, you cannot run any
other packages because only "
+ "airflow/providers package requires --namespace-packages flag
to be set"
+ )
+ result.append("--namespace-packages")
+ if "airflow" in files:
+ if len(files) > 1:
+ raise RuntimeError(
+ "When running `airflow` package, you cannot run any other
packages because only "
+ "airflow/providers package requires --exclude
airflow/providers/.* flag to be set"
+ )
+ result.extend(["--exclude", "airflow/providers/.*"])
+ return result
def insert_documentation(file_path: Path, content: list[str], header: str,
footer: str):
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py
b/scripts/ci/pre_commit/pre_commit_mypy.py
index 61fc4baede..8f38f27ac4 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -18,6 +18,7 @@
from __future__ import annotations
import os
+import shlex
import sys
from pathlib import Path
@@ -25,15 +26,18 @@ sys.path.insert(0, str(Path(__file__).parent.resolve()))
from common_precommit_utils import (
console,
- filter_out_providers_on_non_main_branch,
initialize_breeze_precommit,
+ pre_process_files,
run_command_via_breeze_shell,
)
initialize_breeze_precommit(__name__, __file__)
-files_to_test = filter_out_providers_on_non_main_branch(sys.argv[1:])
-if files_to_test == ["--namespace-packages"]:
+files_to_test = pre_process_files(sys.argv[1:])
+mypy_packages = os.environ.get("MYPY_PACKAGES")
+if mypy_packages:
+ files_to_test += shlex.split(mypy_packages)
+if files_to_test == ["--namespace-packages"] or files_to_test == []:
print("No files to tests. Quitting")
sys.exit(0)
@@ -51,19 +55,26 @@ res = run_command_via_breeze_shell(
"MOUNT_SOURCES": "selected",
},
)
+ci_environment = os.environ.get("CI")
if res.returncode != 0:
+ if mypy_packages and ci_environment:
+ console.print(
+ "[yellow]You are running mypy with the packages selected. If you
want to"
+ "reproduce it locally, you need to run the following command:\n"
+ )
+ console.print(
+ f'MYPY_PACKAGES="{mypy_packages}" pre-commit run --hook-stage
manual mypy --all-files\n'
+ )
upgrading = os.environ.get("UPGRADE_TO_NEWER_DEPENDENCIES", "false") !=
"false"
if upgrading:
console.print(
- "[yellow]You are running mypy with the image that has dependencies
upgraded automatically."
+ "[yellow]You are running mypy with the image that has dependencies
upgraded automatically.\n"
)
flag = " --upgrade-to-newer-dependencies" if upgrading else ""
console.print(
- "[yellow]If you see strange stacktraces above, "
- f"run `breeze ci-image build --python 3.8{flag}` and try again. "
- "You can also run `breeze down --cleanup-mypy-cache` to clean up the
cache used. "
- "Still sometimes diff heuristic in mypy is behaving abnormal, to
double check you can "
- "call `breeze static-checks --type mypy-[dev|core|providers|docs]
--all-files` "
- 'and then commit via `git commit --no-verify -m "commit message"`. CI
will do a full check.'
+ "[yellow]If you see strange stacktraces above, and can't reproduce it,
please run"
+ " this command and try again:\n"
)
+ console.print(f"breeze ci-image build --python 3.8{flag}\n")
+ console.print("[yellow]You can also run `breeze down --cleanup-mypy-cache`
to clean up the cache used.\n")
sys.exit(res.returncode)
