This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 17037e6a2cf Lazy import PodGenerator for deserialization (#56733)
17037e6a2cf is described below
commit 17037e6a2cf7151c361bd611e20e96b68d3f6096
Author: Jarek Potiuk <[email protected]>
AuthorDate: Thu Oct 16 21:14:38 2025 +0200
Lazy import PodGenerator for deserialization (#56733)
The #56692 introduced optimization for PodGenerator imports - but there
was a problem that when deserializing Pod it failed when no k8s classes
were loaded - but it really is not optimisation but failure - nothing
actually prevents us from importing the k8s classes and we actually have
to do it in case we want to deserialize serialized Pod. # Please enter
the commit message for your changes. Lines starting
* fixup! Skip PodGenerator import for deserialization when no k8s installed
* fixup! fixup! Skip PodGenerator import for deserialization when no k8s
installed
---------
Co-authored-by: Kaxil Naik <[email protected]>
---
.../src/airflow/serialization/serialized_objects.py | 20 ++++++++++++++++----
.../unit/serialization/test_dag_serialization.py | 5 +++--
2 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py
b/airflow-core/src/airflow/serialization/serialized_objects.py
index bd8a4096ca5..94a0a3dfafd 100644
--- a/airflow-core/src/airflow/serialization/serialized_objects.py
+++ b/airflow-core/src/airflow/serialization/serialized_objects.py
@@ -925,8 +925,13 @@ class BaseSerialization:
elif type_ == DAT.DATETIME:
return from_timestamp(var)
elif type_ == DAT.POD:
- if not _has_kubernetes():
- raise RuntimeError("Cannot deserialize POD objects without
kubernetes libraries installed!")
+ # Attempt to import kubernetes for deserialization. Using
attempt_import=True allows
+ # lazy loading of kubernetes libraries only when actually needed
for POD deserialization.
+ if not _has_kubernetes(attempt_import=True):
+ raise RuntimeError(
+ "Cannot deserialize POD objects without kubernetes
libraries. "
+ "Please install the cncf.kubernetes provider."
+ )
pod = PodGenerator.deserialize_model_dict(var)
return pod
elif type_ == DAT.TIMEDELTA:
@@ -3799,13 +3804,20 @@ class SerializedAssetWatcher(AssetWatcher):
trigger: dict
-def _has_kubernetes() -> bool:
+def _has_kubernetes(attempt_import: bool = False) -> bool:
+ """
+ Check if kubernetes libraries are available.
+
+ :param attempt_import: If true, attempt to import kubernetes libraries if
not already loaded. If
+ False, only check if already in sys.modules (avoids expensive import).
+ :return: True if kubernetes libraries are available, False otherwise.
+ """
global HAS_KUBERNETES
if "HAS_KUBERNETES" in globals():
return HAS_KUBERNETES
# Check if kubernetes is already imported before triggering expensive
import
- if "kubernetes.client" not in sys.modules:
+ if "kubernetes.client" not in sys.modules and not attempt_import:
HAS_KUBERNETES = False
return False
diff --git a/airflow-core/tests/unit/serialization/test_dag_serialization.py
b/airflow-core/tests/unit/serialization/test_dag_serialization.py
index 433541e10c1..46f77575c69 100644
--- a/airflow-core/tests/unit/serialization/test_dag_serialization.py
+++ b/airflow-core/tests/unit/serialization/test_dag_serialization.py
@@ -2531,7 +2531,7 @@ class TestStringifiedDAGs:
def test_kubernetes_optional():
- """Serialisation / deserialisation continues to work without kubernetes
installed"""
+ """Test that serialization module loads without kubernetes, but
deserialization of PODs requires it"""
def mock__import__(name, globals_=None, locals_=None, fromlist=(),
level=0):
if level == 0 and name.partition(".")[0] == "kubernetes":
@@ -2558,7 +2558,8 @@ def test_kubernetes_optional():
"__var": PodGenerator.serialize_pod(executor_config_pod),
}
- with pytest.raises(RuntimeError):
+ # we should error if attempting to deserialize POD without kubernetes
installed
+ with pytest.raises(RuntimeError, match="Cannot deserialize POD objects
without kubernetes"):
module.BaseSerialization.from_dict(pod_override)
# basic serialization should succeed
