This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f22dbc02403 Add workers.celery.containerLifecycleHooks &
workers.kubernetes.containerLifecycleHooks (#61369)
f22dbc02403 is described below
commit f22dbc02403b45ea32bb4e4e19a06247570ecd0e
Author: Przemysław Mirowski <[email protected]>
AuthorDate: Thu Feb 26 22:02:50 2026 +0100
Add workers.celery.containerLifecycleHooks &
workers.kubernetes.containerLifecycleHooks (#61369)
* Add workers.celery.containerLifecycleHooks &
workers.kubernetes.containerLifecycleHooks
* Refactor some new tests & simplify them
---
chart/files/pod-template-file.kubernetes-helm-yaml | 2 +-
chart/values.schema.json | 60 +++++++++-
chart/values.yaml | 8 ++
.../airflow_aux/test_container_lifecycle.py | 28 ++++-
.../airflow_aux/test_pod_template_file.py | 110 ++++++++++---------
.../tests/helm_tests/airflow_core/test_worker.py | 75 ++++++-------
.../helm_tests/airflow_core/test_worker_sets.py | 121 ++++++++++++++++-----
.../helm_tests/security/test_security_context.py | 8 +-
8 files changed, 282 insertions(+), 130 deletions(-)
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml
b/chart/files/pod-template-file.kubernetes-helm-yaml
index 28042488f87..088cf55ec6a 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -27,7 +27,7 @@
{{- $containerSecurityContextKerberosInitContainer := include
"containerSecurityContext" (list
.Values.workers.kubernetes.kerberosInitContainer
.Values.workers.kerberosInitContainer .Values) }}
{{- $containerLifecycleHooksKerberosInitContainer := or
.Values.workers.kubernetes.kerberosInitContainer.containerLifecycleHooks
.Values.workers.kerberosInitContainer.containerLifecycleHooks
.Values.containerLifecycleHooks }}
{{- $containerSecurityContext := include "containerSecurityContext" (list
.Values.workers.kubernetes .Values.workers .Values) }}
-{{- $containerLifecycleHooks := or .Values.workers.containerLifecycleHooks
.Values.containerLifecycleHooks }}
+{{- $containerLifecycleHooks := or
.Values.workers.kubernetes.containerLifecycleHooks
.Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }}
{{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict"
(.Values.workers.safeToEvict | toString) }}
{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations)
$safeToEvict .Values.workers.podAnnotations }}
{{- $schedulerName := or .Values.workers.schedulerName .Values.schedulerName }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 9c0fa888ac6..2dffad8087d 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2452,7 +2452,7 @@
]
},
"containerLifecycleHooks": {
- "description": "Container Lifecycle Hooks definition for
Airflow Celery workers and pods created with pod-template-file. If not set, the
values from global `containerLifecycleHooks` will be used.",
+ "description": "Container Lifecycle Hooks definition for
Airflow Celery workers and pods created with pod-template-file. If not set, the
values from global `containerLifecycleHooks` will be used. Use
`workers.celery.containerLifecycleHooks` and/or
`workers.kubernetes.containerLifecycleHooks` to separate value between Celery
workers and pod-template-file.",
"type": "object",
"$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle",
"default": {},
@@ -2850,6 +2850,35 @@
}
}
},
+ "containerLifecycleHooks": {
+ "description": "Container Lifecycle Hooks
definition for Airflow Celery workers. If not set, the values from
`workers.containerLifecycleHooks` will be used.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.Lifecycle",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+ "postStart": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo postStart handler >
/usr/share/message"
+ ]
+ }
+ },
+ "preStop": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo preStop handler >
/usr/share/message"
+ ]
+ }
+ }
+ }
+ ]
+ },
"persistence": {
"description": "Persistence configuration for
Airflow Celery workers.",
"type": "object",
@@ -3065,6 +3094,35 @@
}
}
},
+ "containerLifecycleHooks": {
+ "description": "Container Lifecycle Hooks
definition for pods created with pod-template-file. If not set, the values from
`workers.containerLifecycleHooks` will be used.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.Lifecycle",
+ "default": {},
+ "x-docsSection": "Kubernetes",
+ "examples": [
+ {
+ "postStart": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo postStart handler >
/usr/share/message"
+ ]
+ }
+ },
+ "preStop": {
+ "exec": {
+ "command": [
+ "/bin/sh",
+ "-c",
+ "echo preStop handler >
/usr/share/message"
+ ]
+ }
+ }
+ }
+ ]
+ },
"kerberosInitContainer": {
"description": "Kerberos init container for pods
created with pod-template-file.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index 3b75f3aa2d3..d7d5a009324 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -699,6 +699,8 @@ workers:
# Container level Lifecycle Hooks definition for
# Airflow Celery workers and pods created with pod-template-file
+ # Use workers.celery.containerLifecycleHooks and/or
workers.kubernetes.containerLifecycleHooks
+ # to separate value between Celery workers and pod-template-file.
containerLifecycleHooks: {}
# Worker pod disruption budget
@@ -1105,6 +1107,9 @@ workers:
pod: {}
container: {}
+ # Container level Lifecycle Hooks definition for Airflow Celery workers
+ containerLifecycleHooks: {}
+
# Persistence volume configuration for Airflow Celery workers
persistence:
# Enable persistent volumes
@@ -1166,6 +1171,9 @@ workers:
pod: {}
container: {}
+ # Container level Lifecycle Hooks definition for pods created with
pod-template-file
+ containerLifecycleHooks: {}
+
# Kerberos init container configuration for pods created with
pod-template-file
# If not set, the values from `workers.kubernetesInitContainer` section
will be used.
kerberosInitContainer:
diff --git
a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
index 68bcd97e7d1..6ed68d6abe1 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py
@@ -107,8 +107,30 @@ class TestContainerLifecycleHooks:
assert
jmespath.search("spec.template.spec.containers[0].lifecycle", doc) !=
LIFECYCLE_PARSED
# <local>.containerLifecycleWebhooks > containerLifecycleWebhooks
- @pytest.mark.parametrize("hook_type", ["preStop", "postStart"])
- def test_check_main_container_setting(self, hook_type):
+ @pytest.mark.parametrize(
+ ("hook_type", "workers_values"),
+ [
+ ("preStop", {"containerLifecycleHooks": {"preStop":
LIFECYCLE_TEMPLATE}}),
+ ("preStop", {"celery": {"containerLifecycleHooks": {"preStop":
LIFECYCLE_TEMPLATE}}}),
+ (
+ "preStop",
+ {
+ "containerLifecycleHooks": {"postStart":
LIFECYCLE_TEMPLATE},
+ "celery": {"containerLifecycleHooks": {"preStop":
LIFECYCLE_TEMPLATE}},
+ },
+ ),
+ ("postStart", {"containerLifecycleHooks": {"postStart":
LIFECYCLE_TEMPLATE}}),
+ ("postStart", {"celery": {"containerLifecycleHooks": {"postStart":
LIFECYCLE_TEMPLATE}}}),
+ (
+ "postStart",
+ {
+ "containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE},
+ "celery": {"containerLifecycleHooks": {"postStart":
LIFECYCLE_TEMPLATE}},
+ },
+ ),
+ ],
+ )
+ def test_check_main_container_setting(self, hook_type, workers_values):
docs = render_chart(
name=RELEASE_NAME,
values={
@@ -116,7 +138,7 @@ class TestContainerLifecycleHooks:
"flower": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
"scheduler": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
"webserver": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
- "workers": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
+ "workers": workers_values,
"migrateDatabaseJob": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
"triggerer": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
"redis": {"containerLifecycleHooks": {hook_type:
LIFECYCLE_TEMPLATE}},
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
index 3ecbdac1665..70e1aac251c 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
@@ -1075,22 +1075,43 @@ class TestPodTemplateFile:
assert jmespath.search("spec.priorityClassName", docs[0]) ==
"test-priority"
- def test_workers_container_lifecycle_webhooks_are_configurable(self):
- docs = render_chart(
- name="test-release",
- values={
- "workers": {
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "containerLifecycleHooks": {
+ "preStop": {"exec": {"command": ["echo", "preStop", "{{
.Release.Name }}"]}}
+ }
+ },
+ {
+ "kubernetes": {
+ "containerLifecycleHooks": {
+ "preStop": {"exec": {"command": ["echo", "preStop",
"{{ .Release.Name }}"]}}
+ }
+ }
+ },
+ {
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "postStart",
"{{ .Release.Name }}"]}}
+ },
+ "kubernetes": {
"containerLifecycleHooks": {
"preStop": {"exec": {"command": ["echo", "preStop",
"{{ .Release.Name }}"]}}
}
},
},
+ ],
+ )
+ def test_workers_container_lifecycle_webhooks_are_configurable(self,
workers_values):
+ docs = render_chart(
+ name="test-release",
+ values={"workers": workers_values},
show_only=["templates/pod-template-file.yaml"],
chart_dir=self.temp_chart_dir,
)
- assert jmespath.search("spec.containers[0].lifecycle.preStop",
docs[0]) == {
- "exec": {"command": ["echo", "preStop", "test-release"]}
+ assert jmespath.search("spec.containers[0].lifecycle", docs[0]) == {
+ "preStop": {"exec": {"command": ["echo", "preStop",
"test-release"]}}
}
def test_termination_grace_period_seconds(self):
@@ -1385,7 +1406,7 @@ class TestPodTemplateFile:
{
"kerberosInitContainer": {
"enabled": True,
- "securityContexts": {"container": {"runAsUser": 1000}},
+ "securityContexts": {"container":
{"allowPrivilegeEscalation": False}},
},
"kubernetes": {
"kerberosInitContainer": {
@@ -1398,9 +1419,7 @@ class TestPodTemplateFile:
)
def test_kerberos_init_container_security_context(self, workers_values):
docs = render_chart(
- values={
- "workers": workers_values,
- },
+ values={"workers": workers_values},
show_only=["templates/pod-template-file.yaml"],
chart_dir=self.temp_chart_dir,
)
@@ -1410,51 +1429,45 @@ class TestPodTemplateFile:
) == {"runAsUser": 2000}
@pytest.mark.parametrize(
- ("workers_values", "expected"),
+ "workers_values",
[
- (
- {
+ {
+ "kerberosInitContainer": {
+ "enabled": True,
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
+ },
+ {
+ "kubernetes": {
"kerberosInitContainer": {
"enabled": True,
- "containerLifecycleHooks": {"postStart": {"exec":
{"command": ["echo", "base"]}}},
- }
- },
- {"postStart": {"exec": {"command": ["echo", "base"]}}},
- ),
- (
- {
- "kubernetes": {
- "kerberosInitContainer": {
- "enabled": True,
- "containerLifecycleHooks": {
- "postStart": {"exec": {"command": ["echo",
"kubernetes"]}}
- },
- }
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
}
+ }
+ },
+ {
+ "kerberosInitContainer": {
+ "enabled": True,
+ "containerLifecycleHooks": {"preStop": {"exec":
{"command": ["echo", "base"]}}},
},
- {"postStart": {"exec": {"command": ["echo", "kubernetes"]}}},
- ),
- (
- {
+ "kubernetes": {
"kerberosInitContainer": {
"enabled": True,
- "containerLifecycleHooks": {"preStop": {"exec":
{"command": ["echo", "base"]}}},
- },
- "kubernetes": {
- "kerberosInitContainer": {
- "enabled": True,
- "containerLifecycleHooks": {
- "postStart": {"exec": {"command": ["echo",
"kubernetes"]}}
- },
- }
- },
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
},
- {"postStart": {"exec": {"command": ["echo", "kubernetes"]}}},
- ),
+ },
],
)
- def test_kerberos_init_container_lifecycle_hooks(self, workers_values,
expected):
+ def test_kerberos_init_container_lifecycle_hooks(self, workers_values):
docs = render_chart(
+ name="test-release",
values={
"workers": workers_values,
},
@@ -1462,7 +1475,6 @@ class TestPodTemplateFile:
chart_dir=self.temp_chart_dir,
)
- assert (
- jmespath.search("spec.initContainers[?name=='kerberos-init'] |
[0].lifecycle", docs[0])
- == expected
- )
+ assert jmespath.search("spec.initContainers[?name=='kerberos-init'] |
[0].lifecycle", docs[0]) == {
+ "postStart": {"exec": {"command": ["echo", "test-release"]}}
+ }
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
index 9af9571a23e..07c93936722 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
@@ -1023,63 +1023,52 @@ class TestWorker:
}
@pytest.mark.parametrize(
- ("workers_values", "expected"),
+ "workers_values",
[
- (
- {
+ {
+ "kerberosInitContainer": {
+ "enabled": True,
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
+ },
+ {
+ "celery": {
"kerberosInitContainer": {
"enabled": True,
- "containerLifecycleHooks": {"postStart": {"exec":
{"command": ["echo", "base"]}}},
- }
- },
- {"postStart": {"exec": {"command": ["echo", "base"]}}},
- ),
- (
- {
- "celery": {
- "kerberosInitContainer": {
- "enabled": True,
- "containerLifecycleHooks": {
- "postStart": {"exec": {"command": ["echo",
"celery"]}}
- },
- }
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
}
+ }
+ },
+ {
+ "kerberosInitContainer": {
+ "enabled": True,
+ "containerLifecycleHooks": {"preStop": {"exec":
{"command": ["echo", "base"]}}},
},
- {"postStart": {"exec": {"command": ["echo", "celery"]}}},
- ),
- (
- {
+ "celery": {
"kerberosInitContainer": {
"enabled": True,
- "containerLifecycleHooks": {"postStart": {"exec":
{"command": ["echo", "base"]}}},
- },
- "celery": {
- "kerberosInitContainer": {
- "enabled": True,
- "containerLifecycleHooks": {
- "postStart": {"exec": {"command": ["echo",
"celery"]}}
- },
- }
- },
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
},
- {"postStart": {"exec": {"command": ["echo", "celery"]}}},
- ),
+ },
],
)
- def test_kerberos_init_container_lifecycle_hooks(self, workers_values,
expected):
+ def test_kerberos_init_container_lifecycle_hooks(self, workers_values):
docs = render_chart(
- values={
- "workers": workers_values,
- },
+ name="test-release",
+ values={"workers": workers_values},
show_only=["templates/workers/worker-deployment.yaml"],
)
- assert (
- jmespath.search(
- "spec.template.spec.initContainers[?name=='kerberos-init'] |
[0].lifecycle", docs[0]
- )
- == expected
- )
+ assert jmespath.search(
+ "spec.template.spec.initContainers[?name=='kerberos-init'] |
[0].lifecycle", docs[0]
+ ) == {"postStart": {"exec": {"command": ["echo", "test-release"]}}}
@pytest.mark.parametrize(
("airflow_version", "expected_arg"),
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
index 1434094172a..72bd7c61e77 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
@@ -842,7 +842,7 @@ class TestWorkerSets:
}
@pytest.mark.parametrize(
- "values",
+ "workers_values",
[
{
"celery": {
@@ -852,9 +852,7 @@ class TestWorkerSets:
"name": "test",
"kerberosInitContainer": {
"enabled": True,
- "securityContexts": {
- "container": {"runAsUser": 10},
- },
+ "securityContexts": {"container":
{"runAsUser": 10}},
},
}
],
@@ -873,9 +871,26 @@ class TestWorkerSets:
"name": "test",
"kerberosInitContainer": {
"enabled": True,
- "securityContexts": {
- "container": {"runAsUser": 10},
- },
+ "securityContexts": {"container":
{"runAsUser": 10}},
+ },
+ }
+ ],
+ },
+ },
+ {
+ "celery": {
+ "kerberosInitContainer": {
+ "securityContexts": {
+ "container": {"allowPrivilegeEscalation": False},
+ }
+ },
+ "enableDefault": False,
+ "sets": [
+ {
+ "name": "test",
+ "kerberosInitContainer": {
+ "enabled": True,
+ "securityContexts": {"container":
{"runAsUser": 10}},
},
}
],
@@ -883,9 +898,9 @@ class TestWorkerSets:
},
],
)
- def test_overwrite_kerberos_init_container_security_context(self, values):
+ def test_overwrite_kerberos_init_container_security_context(self,
workers_values):
docs = render_chart(
- values={"workers": values},
+ values={"workers": workers_values},
show_only=["templates/workers/worker-deployment.yaml"],
)
@@ -894,7 +909,7 @@ class TestWorkerSets:
) == {"runAsUser": 10}
@pytest.mark.parametrize(
- "values",
+ "workers_values",
[
{
"celery": {
@@ -931,11 +946,30 @@ class TestWorkerSets:
],
},
},
+ {
+ "celery": {
+ "kerberosInitContainer": {
+ "containerLifecycleHooks": {"preStop": {"exec":
{"command": ["echo", "test"]}}}
+ },
+ "enableDefault": False,
+ "sets": [
+ {
+ "name": "test",
+ "kerberosInitContainer": {
+ "enabled": True,
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo",
"{{ .Release.Name }}"]}},
+ },
+ },
+ }
+ ],
+ },
+ },
],
)
- def test_overwrite_kerberos_init_container_lifecycle_hooks(self, values):
+ def test_overwrite_kerberos_init_container_lifecycle_hooks(self,
workers_values):
docs = render_chart(
- values={"workers": values},
+ values={"workers": workers_values},
show_only=["templates/workers/worker-deployment.yaml"],
)
@@ -943,24 +977,55 @@ class TestWorkerSets:
"spec.template.spec.initContainers[?name=='kerberos-init'] |
[0].lifecycle", docs[0]
) == {"postStart": {"exec": {"command": ["echo", "release-name"]}}}
- def test_overwrite_container_lifecycle_hooks(self):
- docs = render_chart(
- values={
- "workers": {
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "celery": {
+ "enableDefault": False,
+ "sets": [
+ {
+ "name": "test",
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
+ ],
+ },
+ },
+ {
+ "containerLifecycleHooks": {"preStop": {"exec": {"command":
["echo", "test"]}}},
+ "celery": {
+ "enableDefault": False,
+ "sets": [
+ {
+ "name": "test",
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
+ ],
+ },
+ },
+ {
+ "celery": {
"containerLifecycleHooks": {"preStop": {"exec":
{"command": ["echo", "test"]}}},
- "celery": {
- "enableDefault": False,
- "sets": [
- {
- "name": "test",
- "containerLifecycleHooks": {
- "postStart": {"exec": {"command": ["echo",
"{{ .Release.Name }}"]}}
- },
- }
- ],
- },
- }
+ "enableDefault": False,
+ "sets": [
+ {
+ "name": "test",
+ "containerLifecycleHooks": {
+ "postStart": {"exec": {"command": ["echo", "{{
.Release.Name }}"]}}
+ },
+ }
+ ],
+ },
},
+ ],
+ )
+ def test_overwrite_container_lifecycle_hooks(self, workers_values):
+ docs = render_chart(
+ values={"workers": workers_values},
show_only=["templates/workers/worker-deployment.yaml"],
)
diff --git a/helm-tests/tests/helm_tests/security/test_security_context.py
b/helm-tests/tests/helm_tests/security/test_security_context.py
index 65534dcf3ca..fd769e0faac 100644
--- a/helm-tests/tests/helm_tests/security/test_security_context.py
+++ b/helm-tests/tests/helm_tests/security/test_security_context.py
@@ -624,7 +624,7 @@ class TestSecurityContext:
{
"kerberosInitContainer": {
"enabled": True,
- "securityContexts": {"container": {"runAsUser": 1000}},
+ "securityContexts": {"container":
{"allowPrivilegeEscalation": False}},
},
"celery": {
"kerberosInitContainer": {
@@ -635,11 +635,9 @@ class TestSecurityContext:
},
],
)
- def test_worker_kerberos_init_container_security_context(self,
workers_values):
+ def test_worker_kerberos_init_container_security_contexts(self,
workers_values):
docs = render_chart(
- values={
- "workers": workers_values,
- },
+ values={"workers": workers_values},
show_only=["templates/workers/worker-deployment.yaml"],
)
