This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b82ce61285 Improve modules import in Airflow core by some of them into
a type-checking block (#33755)
b82ce61285 is described below
commit b82ce61285f3f4f0c7eccb2c3effaef53c9fb84e
Author: Hussein Awala <[email protected]>
AuthorDate: Mon Aug 28 16:54:08 2023 +0200
Improve modules import in Airflow core by some of them into a type-checking
block (#33755)
* Improve modules import in Airflow core by some of them into a
type-checking block
* Restore lazy import in models and fix import in utils.sessions
* fix unit tests
* fix unit tests
* fix static checks
---
airflow/configuration.py | 9 ++++++---
airflow/exceptions.py | 3 ++-
airflow/executors/base_executor.py | 8 +++++---
airflow/executors/local_executor.py | 6 ++++--
airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py | 4 +++-
airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py | 5 ++++-
airflow/plugins_manager.py | 13 ++++++-------
airflow/providers_manager.py | 3 ++-
airflow/settings.py | 6 ++++--
airflow/stats.py | 5 ++++-
airflow/templates.py | 5 ++++-
airflow/timetables/_cron.py | 6 ++++--
airflow/timetables/base.py | 4 ++--
airflow/timetables/events.py | 10 +++++++---
airflow/timetables/interval.py | 7 +++++--
airflow/timetables/simple.py | 3 ++-
airflow/timetables/trigger.py | 12 ++++++++----
airflow/triggers/external_task.py | 11 ++++++++---
18 files changed, 80 insertions(+), 40 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 41dc8d718e..7f04c6926b 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -35,22 +35,25 @@ from configparser import ConfigParser, NoOptionError,
NoSectionError
from contextlib import contextmanager
from copy import deepcopy
from json.decoder import JSONDecodeError
-from typing import IO, Any, Dict, Generator, Iterable, Pattern, Set, Tuple,
Union
+from typing import IO, TYPE_CHECKING, Any, Dict, Generator, Iterable, Pattern,
Set, Tuple, Union
from urllib.parse import urlsplit
import re2
from packaging.version import parse as parse_version
from typing_extensions import overload
-from airflow.auth.managers.base_auth_manager import BaseAuthManager
from airflow.exceptions import AirflowConfigException
-from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend
+from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH
from airflow.utils import yaml
from airflow.utils.empty_set import _get_empty_set_for_configuration
from airflow.utils.module_loading import import_string
from airflow.utils.providers_configuration_loader import
providers_configuration_loaded
from airflow.utils.weight_rule import WeightRule
+if TYPE_CHECKING:
+ from airflow.auth.managers.base_auth_manager import BaseAuthManager
+ from airflow.secrets import BaseSecretsBackend
+
log = logging.getLogger(__name__)
# show Airflow's deprecation warnings
diff --git a/airflow/exceptions.py b/airflow/exceptions.py
index b471297cd9..0840e801a1 100644
--- a/airflow/exceptions.py
+++ b/airflow/exceptions.py
@@ -20,7 +20,6 @@
"""Exceptions used by Airflow."""
from __future__ import annotations
-import datetime
import warnings
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, NamedTuple, Sized
@@ -28,6 +27,8 @@ from typing import TYPE_CHECKING, Any, NamedTuple, Sized
from airflow.utils.trigger_rule import TriggerRule
if TYPE_CHECKING:
+ import datetime
+
from airflow.models import DAG, DagRun
diff --git a/airflow/executors/base_executor.py
b/airflow/executors/base_executor.py
index 909a955cbf..2791c938a4 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -17,18 +17,16 @@
"""Base executor - this is the base class for all the implemented executors."""
from __future__ import annotations
-import argparse
import logging
import sys
import warnings
from collections import defaultdict
from dataclasses import dataclass, field
-from datetime import datetime
from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple
import pendulum
-from airflow.cli.cli_config import DefaultHelpParser, GroupCommand
+from airflow.cli.cli_config import DefaultHelpParser
from airflow.configuration import conf
from airflow.exceptions import RemovedInAirflow3Warning
from airflow.stats import Stats
@@ -38,8 +36,12 @@ from airflow.utils.state import TaskInstanceState
PARALLELISM: int = conf.getint("core", "PARALLELISM")
if TYPE_CHECKING:
+ import argparse
+ from datetime import datetime
+
from airflow.callbacks.base_callback_sink import BaseCallbackSink
from airflow.callbacks.callback_requests import CallbackRequest
+ from airflow.cli.cli_config import GroupCommand
from airflow.models.taskinstance import TaskInstance
from airflow.models.taskinstancekey import TaskInstanceKey
diff --git a/airflow/executors/local_executor.py
b/airflow/executors/local_executor.py
index 9dcfdb629f..fd3cc6a629 100644
--- a/airflow/executors/local_executor.py
+++ b/airflow/executors/local_executor.py
@@ -29,8 +29,7 @@ import os
import subprocess
from abc import abstractmethod
from multiprocessing import Manager, Process
-from multiprocessing.managers import SyncManager
-from queue import Empty, Queue
+from queue import Empty
from typing import TYPE_CHECKING, Any, Optional, Tuple
from setproctitle import getproctitle, setproctitle
@@ -42,6 +41,9 @@ from airflow.utils.log.logging_mixin import LoggingMixin
from airflow.utils.state import TaskInstanceState
if TYPE_CHECKING:
+ from multiprocessing.managers import SyncManager
+ from queue import Queue
+
from airflow.executors.base_executor import CommandType
from airflow.models.taskinstance import TaskInstanceStateType
from airflow.models.taskinstancekey import TaskInstanceKey
diff --git a/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
b/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
index 8280a3265f..cff12d057b 100644
--- a/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
+++ b/airflow/kubernetes/pre_7_4_0_compatibility/k8s_model.py
@@ -19,8 +19,10 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from functools import reduce
+from typing import TYPE_CHECKING
-from kubernetes.client import models as k8s
+if TYPE_CHECKING:
+ from kubernetes.client import models as k8s
class K8SModel(ABC):
diff --git a/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
index 1fe8d50d76..4576197863 100644
--- a/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
+++ b/airflow/kubernetes/pre_7_4_0_compatibility/pod_generator.py
@@ -27,13 +27,13 @@ is supported and no serialization need be written.
from __future__ import annotations
import copy
-import datetime
import logging
import os
import secrets
import string
import warnings
from functools import reduce
+from typing import TYPE_CHECKING
import re2
from dateutil import parser
@@ -54,6 +54,9 @@ from airflow.utils import yaml
from airflow.utils.hashlib_wrapper import md5
from airflow.version import version as airflow_version
+if TYPE_CHECKING:
+ import datetime
+
log = logging.getLogger(__name__)
MAX_LABEL_LEN = 63
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 1b117101d7..0970e2ca3c 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -28,19 +28,18 @@ import sys
import types
from typing import TYPE_CHECKING, Any, Iterable
-try:
- import importlib_metadata
-except ImportError:
- from importlib import metadata as importlib_metadata # type:
ignore[no-redef]
-
-from types import ModuleType
-
from airflow import settings
from airflow.utils.entry_points import entry_points_with_dist
from airflow.utils.file import find_path_from_directory
from airflow.utils.module_loading import import_string, qualname
if TYPE_CHECKING:
+ try:
+ import importlib_metadata
+ except ImportError:
+ from importlib import metadata as importlib_metadata # type:
ignore[no-redef]
+ from types import ModuleType
+
from airflow.hooks.base import BaseHook
from airflow.listeners.listener import ListenerManager
from airflow.timetables.base import Timetable
diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index 002747c332..3623be2843 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -33,7 +33,6 @@ from time import perf_counter
from typing import TYPE_CHECKING, Any, Callable, MutableMapping, NamedTuple,
TypeVar, cast
from packaging.utils import canonicalize_name
-from typing_extensions import Literal
from airflow.exceptions import AirflowOptionalProviderFeatureException
from airflow.hooks.filesystem import FSHook
@@ -85,6 +84,8 @@ def _ensure_prefix_for_placeholders(field_behaviors:
dict[str, Any], conn_type:
if TYPE_CHECKING:
+ from typing_extensions import Literal
+
from airflow.decorators.base import TaskDecorator
from airflow.hooks.base import BaseHook
diff --git a/airflow/settings.py b/airflow/settings.py
index 0c4624777a..aff84d9e46 100644
--- a/airflow/settings.py
+++ b/airflow/settings.py
@@ -30,8 +30,7 @@ import pendulum
import pluggy
import sqlalchemy
from sqlalchemy import create_engine, exc, text
-from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session as SASession, scoped_session, sessionmaker
+from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
from airflow import policies
@@ -43,6 +42,9 @@ from airflow.utils.orm_event_handlers import
setup_event_handlers
from airflow.utils.state import State
if TYPE_CHECKING:
+ from sqlalchemy.engine import Engine
+ from sqlalchemy.orm import Session as SASession
+
from airflow.www.utils import UIAlert
log = logging.getLogger(__name__)
diff --git a/airflow/stats.py b/airflow/stats.py
index 5b84def6db..f6a6e97c54 100644
--- a/airflow/stats.py
+++ b/airflow/stats.py
@@ -22,7 +22,10 @@ import socket
from typing import TYPE_CHECKING, Callable
from airflow.configuration import conf
-from airflow.metrics.base_stats_logger import NoStatsLogger, StatsLogger
+from airflow.metrics.base_stats_logger import NoStatsLogger
+
+if TYPE_CHECKING:
+ from airflow.metrics.base_stats_logger import StatsLogger
log = logging.getLogger(__name__)
diff --git a/airflow/templates.py b/airflow/templates.py
index 8cd113054d..95851253a7 100644
--- a/airflow/templates.py
+++ b/airflow/templates.py
@@ -17,11 +17,14 @@
# under the License.
from __future__ import annotations
-import datetime
+from typing import TYPE_CHECKING
import jinja2.nativetypes
import jinja2.sandbox
+if TYPE_CHECKING:
+ import datetime
+
class _AirflowEnvironmentMixin:
def __init__(self, **kwargs):
diff --git a/airflow/timetables/_cron.py b/airflow/timetables/_cron.py
index 89cae4bdcb..6787628888 100644
--- a/airflow/timetables/_cron.py
+++ b/airflow/timetables/_cron.py
@@ -18,17 +18,19 @@ from __future__ import annotations
import datetime
from functools import cached_property
-from typing import Any
+from typing import TYPE_CHECKING, Any
from cron_descriptor import CasingTypeEnum, ExpressionDescriptor,
FormatException, MissingFieldException
from croniter import CroniterBadCronError, CroniterBadDateError, croniter
-from pendulum import DateTime
from pendulum.tz.timezone import Timezone
from airflow.exceptions import AirflowTimetableInvalid
from airflow.utils.dates import cron_presets
from airflow.utils.timezone import convert_to_utc, make_aware, make_naive
+if TYPE_CHECKING:
+ from pendulum import DateTime
+
def _is_schedule_fixed(expression: str) -> bool:
"""Figures out if the schedule has a fixed time (e.g. 3 AM every day).
diff --git a/airflow/timetables/base.py b/airflow/timetables/base.py
index c02f700233..b5e95ef5f4 100644
--- a/airflow/timetables/base.py
+++ b/airflow/timetables/base.py
@@ -19,11 +19,11 @@ from __future__ import annotations
from typing import TYPE_CHECKING, Any, NamedTuple, Sequence
from warnings import warn
-from pendulum import DateTime
-
from airflow.typing_compat import Protocol, runtime_checkable
if TYPE_CHECKING:
+ from pendulum import DateTime
+
from airflow.utils.types import DagRunType
diff --git a/airflow/timetables/events.py b/airflow/timetables/events.py
index ce8fa9527f..62cf3dce4e 100644
--- a/airflow/timetables/events.py
+++ b/airflow/timetables/events.py
@@ -17,12 +17,16 @@
from __future__ import annotations
import itertools
-from typing import Iterable
+from typing import TYPE_CHECKING, Iterable
import pendulum
-from pendulum import DateTime
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction,
Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
+
+if TYPE_CHECKING:
+ from pendulum import DateTime
+
+ from airflow.timetables.base import TimeRestriction
class EventsTimetable(Timetable):
diff --git a/airflow/timetables/interval.py b/airflow/timetables/interval.py
index 27e128ff52..077c4195a7 100644
--- a/airflow/timetables/interval.py
+++ b/airflow/timetables/interval.py
@@ -17,16 +17,19 @@
from __future__ import annotations
import datetime
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, Union
from dateutil.relativedelta import relativedelta
from pendulum import DateTime
from airflow.exceptions import AirflowTimetableInvalid
from airflow.timetables._cron import CronMixin
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction,
Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
from airflow.utils.timezone import convert_to_utc
+if TYPE_CHECKING:
+ from airflow.timetables.base import TimeRestriction
+
Delta = Union[datetime.timedelta, relativedelta]
diff --git a/airflow/timetables/simple.py b/airflow/timetables/simple.py
index 53ddf6a7a8..0dd73627d6 100644
--- a/airflow/timetables/simple.py
+++ b/airflow/timetables/simple.py
@@ -21,12 +21,13 @@ from typing import TYPE_CHECKING, Any, Collection
from pendulum import DateTime
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction,
Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
if TYPE_CHECKING:
from sqlalchemy import Session
from airflow.models.dataset import DatasetEvent
+ from airflow.timetables.base import TimeRestriction
from airflow.utils.types import DagRunType
diff --git a/airflow/timetables/trigger.py b/airflow/timetables/trigger.py
index e5c5b7c153..95d2923803 100644
--- a/airflow/timetables/trigger.py
+++ b/airflow/timetables/trigger.py
@@ -17,14 +17,18 @@
from __future__ import annotations
import datetime
-from typing import Any
+from typing import TYPE_CHECKING, Any
-from dateutil.relativedelta import relativedelta
from pendulum import DateTime
-from pendulum.tz.timezone import Timezone
from airflow.timetables._cron import CronMixin
-from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction,
Timetable
+from airflow.timetables.base import DagRunInfo, DataInterval, Timetable
+
+if TYPE_CHECKING:
+ from dateutil.relativedelta import relativedelta
+ from pendulum.tz.timezone import Timezone
+
+ from airflow.timetables.base import TimeRestriction
class CronTriggerTimetable(CronMixin, Timetable):
diff --git a/airflow/triggers/external_task.py
b/airflow/triggers/external_task.py
index 431d91a15e..00c7c52284 100644
--- a/airflow/triggers/external_task.py
+++ b/airflow/triggers/external_task.py
@@ -18,18 +18,23 @@ from __future__ import annotations
import asyncio
import typing
-from datetime import datetime
from asgiref.sync import sync_to_async
from sqlalchemy import func
-from sqlalchemy.orm import Session
from airflow.models import DagRun, TaskInstance
from airflow.triggers.base import BaseTrigger, TriggerEvent
from airflow.utils.session import NEW_SESSION, provide_session
-from airflow.utils.state import DagRunState, TaskInstanceState
+from airflow.utils.state import TaskInstanceState
from airflow.utils.timezone import utcnow
+if typing.TYPE_CHECKING:
+ from datetime import datetime
+
+ from sqlalchemy.orm import Session
+
+ from airflow.utils.state import DagRunState
+
class TaskStateTrigger(BaseTrigger):
"""
