This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 977c119a050 Add workers.celery.labels & workers.kubernetes.labels
(#65030)
977c119a050 is described below
commit 977c119a05041e460085aae0589186b35a278f4c
Author: Przemysław Mirowski <[email protected]>
AuthorDate: Sun Apr 12 20:58:31 2026 +0200
Add workers.celery.labels & workers.kubernetes.labels (#65030)
* Add workers.celery.labels & workers.kubernetes.labels
* Add newsfragment
---
chart/docs/customizing-labels.rst | 7 +-
chart/files/pod-template-file.kubernetes-helm-yaml | 4 +-
chart/newsfragments/65030.significant.rst | 1 +
chart/templates/NOTES.txt | 8 +
.../workers/worker-kubernetes-serviceaccount.yaml | 4 +-
chart/values.schema.json | 18 +-
chart/values.yaml | 7 +
.../airflow_aux/test_pod_template_file.py | 30 ++-
.../helm_tests/airflow_core/test_pdb_worker.py | 31 ++-
.../tests/helm_tests/airflow_core/test_worker.py | 215 ++++++++++++++++-----
.../helm_tests/airflow_core/test_worker_sets.py | 7 +
11 files changed, 260 insertions(+), 72 deletions(-)
diff --git a/chart/docs/customizing-labels.rst
b/chart/docs/customizing-labels.rst
index 978dbec2152..80b56cee809 100644
--- a/chart/docs/customizing-labels.rst
+++ b/chart/docs/customizing-labels.rst
@@ -42,7 +42,7 @@ You can also set specific labels for individual Airflow
components, which will b
If the same label key exists in both global and component-specific labels, the
component-specific value takes precedence (overrides the global value).
This allows you to customize labels for specific components while still
maintaining common global labels across all resources.
-For example, to add specific labels to different components:
+For example, to add specific labels to different components like scheduler or
api-server:
.. code-block:: yaml
:caption: values.yaml
@@ -56,11 +56,6 @@ For example, to add specific labels to different components:
labels:
role: scheduler
- # Worker specific labels
- workers:
- labels:
- role: worker
-
# API Server specific labels
apiServer:
labels:
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml
b/chart/files/pod-template-file.kubernetes-helm-yaml
index 305e7b5e723..6449fe329ad 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -39,8 +39,8 @@ metadata:
tier: airflow
component: worker
release: {{ .Release.Name }}
- {{- if or (.Values.labels) (.Values.workers.labels) }}
- {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4
}}
+ {{- if or .Values.labels .Values.workers.labels
.Values.workers.kubernetes.labels }}
+ {{- mustMerge (.Values.workers.kubernetes.labels | default
.Values.workers.labels) .Values.labels | toYaml | nindent 4 }}
{{- end }}
annotations:
{{- tpl (toYaml $podAnnotations) . | nindent 4 }}
diff --git a/chart/newsfragments/65030.significant.rst
b/chart/newsfragments/65030.significant.rst
new file mode 100644
index 00000000000..055bfa76519
--- /dev/null
+++ b/chart/newsfragments/65030.significant.rst
@@ -0,0 +1 @@
+``workers.labels`` field is now deprecated in favor of
``workers.celery.labels`` and ``workers.kubernetes.labels``. Please update your
configuration accordingly.
diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt
index 205fd39ec19..b5038eaa55f 100644
--- a/chart/templates/NOTES.txt
+++ b/chart/templates/NOTES.txt
@@ -813,6 +813,14 @@ DEPRECATION WARNING:
{{- end }}
+{{- if not (empty .Values.workers.labels) }}
+
+ DEPRECATION WARNING:
+ `workers.labels` has been renamed to
`workers.celery.labels`/`workers.kubernetes.labels`.
+ Please change your values as support for the old name will be dropped in a
future release.
+
+{{- end }}
+
{{- if not (empty .Values.workers.volumeClaimTemplates) }}
DEPRECATION WARNING:
diff --git a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml
b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml
index 7530f0732d3..b74474b42f2 100644
--- a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml
+++ b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml
@@ -32,8 +32,8 @@ metadata:
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
- {{- if or .Values.labels .Values.workers.labels }}
- {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4
}}
+ {{- if or .Values.labels .Values.workers.labels
.Values.workers.kubernetes.labels }}
+ {{- mustMerge (.Values.workers.kubernetes.labels | default
.Values.workers.labels) .Values.labels | toYaml | nindent 4 }}
{{- end }}
{{- with (.Values.workers.kubernetes.serviceAccount.annotations | default
.Values.workers.serviceAccount.annotations) }}
annotations: {{- toYaml . | nindent 4 }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 984149f26b2..a9560366ffe 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2428,7 +2428,7 @@
"x-docsSection": "Common"
},
"labels": {
- "description": "Labels to add to the Airflow Celery
workers objects and pods created with pod-template-file.",
+ "description": "Labels to add to the Airflow Celery
workers objects and pods created with pod-template-file (deprecated, use
``workers.celery.labels`` and/or ``workers.kubernetes.labels`` instead).",
"type": "object",
"default": {},
"additionalProperties": {
@@ -3648,6 +3648,14 @@
"type": "string"
}
},
+ "labels": {
+ "description": "Labels to add to the Airflow
Celery workers objects.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
"logGroomerSidecar": {
"description": "Configuration for Airflow Celery
worker log groomer sidecar.",
"type": "object",
@@ -4349,6 +4357,14 @@
"type": "string"
}
},
+ "labels": {
+ "description": "Labels to add to the pods created
with pod-template-file.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
"schedulerName": {
"description": "Specify kube scheduler name for
pods created with pod-template-file.",
"type": [
diff --git a/chart/values.yaml b/chart/values.yaml
index fa56e57bebc..75e0c979656 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -1139,6 +1139,7 @@ workers:
podAnnotations: {}
# Labels specific to Airflow Celery workers objects and pods created with
pod-template-file
+ # (deprecated, use `workers.celery.labels` and/or
`workers.kubernetes.labels` instead)
labels: {}
# Log groomer configuration for Airflow Celery workers
@@ -1550,6 +1551,9 @@ workers:
# Pod annotations for the Airflow Celery workers (templated)
podAnnotations: {}
+ # Labels specific to Airflow Celery workers objects
+ labels: {}
+
# Log groomer configuration for Airflow Celery workers
logGroomerSidecar:
# Whether to deploy the Airflow Celery worker log groomer sidecar
@@ -1778,6 +1782,9 @@ workers:
# Pod annotations for the pods created with pod-template-file (templated)
podAnnotations: {}
+ # Labels specific to pods created with pod-template-file
+ labels: {}
+
schedulerName: ~
# Airflow scheduler settings
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
index ed57a426ce5..55a1fb94d4c 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
@@ -1372,20 +1372,38 @@ class TestPodTemplateFile:
"valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key":
"my-key"}},
} in jmespath.search("spec.containers[0].env", docs[0])
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "kubernetes": {
+ "labels": {"test_label": "test_label_value"},
+ }
+ },
+ {
+ "labels": {"key": "value"},
+ "kubernetes": {
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "KubernetesExecutor",
- "workers": {
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/pod-template-file.yaml"],
chart_dir=self.temp_chart_dir,
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
@pytest.mark.parametrize(
"workers_values",
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py
b/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py
index b3ba1467910..2e18abad536 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py
@@ -57,22 +57,35 @@ class TestWorkerPdb:
@pytest.mark.parametrize(
"workers_values",
[
- {"podDisruptionBudget": {"enabled": True}},
- {"celery": {"podDisruptionBudget": {"enabled": True}}},
+ {"podDisruptionBudget": {"enabled": True}, "labels":
{"test_label": "test_label_value"}},
+ {
+ "celery": {"podDisruptionBudget": {"enabled": True}},
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "celery": {
+ "podDisruptionBudget": {"enabled": True},
+ "labels": {"test_label": "test_label_value"},
+ }
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {
+ "podDisruptionBudget": {"enabled": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
],
)
def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
- values={
- "workers": {
- **workers_values,
- "labels": {"test_label": "test_label_value"},
- },
- },
+ values={"workers": workers_values},
show_only=["templates/workers/worker-poddisruptionbudget.yaml"],
)
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
@pytest.mark.parametrize(
"workers_values",
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
index 376a2d30c08..10124a25008 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
@@ -471,19 +471,37 @@ class TestWorker:
"spec.template.spec.initContainers[0].env", docs[0]
)
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "celery": {
+ "labels": {"test_label": "test_label_value"},
+ }
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "CeleryExecutor",
- "workers": {
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-deployment.yaml"],
)
- assert "test_label" in
jmespath.search("spec.template.metadata.labels", docs[0])
- assert jmespath.search("spec.template.metadata.labels",
docs[0])["test_label"] == "test_label_value"
+ labels = jmespath.search("spec.template.metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
@pytest.mark.parametrize(
"workers_values",
@@ -2300,20 +2318,40 @@ class TestWorkerCeleryLogGroomer(LogGroomerTestBase):
class TestWorkerKedaAutoScaler:
"""Tests worker keda auto scaler."""
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "celery": {"keda": {"enabled": True}},
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "celery": {
+ "keda": {"enabled": True},
+ "labels": {"test_label": "test_label_value"},
+ }
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {
+ "keda": {"enabled": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "CeleryExecutor",
- "workers": {
- "celery": {"keda": {"enabled": True}},
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-kedaautoscaler.yaml"],
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
def test_should_remove_replicas_field(self):
docs = render_chart(
@@ -2469,20 +2507,37 @@ class TestWorkerHPAAutoScaler:
assert len(docs) == 1
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {"celery": {"hpa": {"enabled": True}}, "labels": {"test_label":
"test_label_value"}},
+ {
+ "celery": {
+ "hpa": {"enabled": True},
+ "labels": {"test_label": "test_label_value"},
+ }
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {
+ "hpa": {"enabled": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "CeleryExecutor",
- "workers": {
- "celery": {"hpa": {"enabled": True}},
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-hpa.yaml"],
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
def test_should_remove_replicas_field(self):
docs = render_chart(
@@ -2607,38 +2662,66 @@ class TestWorkerHPAAutoScaler:
class TestWorkerNetworkPolicy:
"""Tests worker network policy."""
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "celery": {"labels": {"test_label": "test_label_value"}},
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {"labels": {"test_label": "test_label_value"}},
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"networkPolicies": {"enabled": True},
"executor": "CeleryExecutor",
- "workers": {
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-networkpolicy.yaml"],
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
class TestWorkerService:
"""Tests worker service."""
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "celery": {"labels": {"test_label": "test_label_value"}},
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {"labels": {"test_label": "test_label_value"}},
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "CeleryExecutor",
- "workers": {
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-service.yaml"],
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
class TestWorkerCeleryServiceAccount:
@@ -2770,20 +2853,40 @@ class TestWorkerCeleryServiceAccount:
assert jmespath.search("metadata.name", docs[0]) == "test"
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "celery": {"serviceAccount": {"create": True}},
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "celery": {
+ "serviceAccount": {"create": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ {
+ "labels": {"key": "value"},
+ "celery": {
+ "serviceAccount": {"create": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "CeleryExecutor",
- "workers": {
- "celery": {"serviceAccount": {"create": True}},
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-serviceaccount.yaml"],
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
class TestWorkerKubernetesServiceAccount:
@@ -2924,17 +3027,37 @@ class TestWorkerKubernetesServiceAccount:
assert jmespath.search("metadata.name", docs[0]) == "test"
- def test_should_add_component_specific_labels(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {
+ "kubernetes": {"serviceAccount": {"create": True}},
+ "labels": {"test_label": "test_label_value"},
+ },
+ {
+ "kubernetes": {
+ "serviceAccount": {"create": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ {
+ "labels": {"key": "value"},
+ "kubernetes": {
+ "serviceAccount": {"create": True},
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+ ],
+ )
+ def test_should_add_component_specific_labels(self, workers_values):
docs = render_chart(
values={
"executor": "KubernetesExecutor",
- "workers": {
- "kubernetes": {"serviceAccount": {"create": True}},
- "labels": {"test_label": "test_label_value"},
- },
+ "workers": workers_values,
},
show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"],
)
- assert "test_label" in jmespath.search("metadata.labels", docs[0])
- assert jmespath.search("metadata.labels", docs[0])["test_label"] ==
"test_label_value"
+ labels = jmespath.search("metadata.labels", docs[0])
+ assert labels["test_label"] == "test_label_value"
+ assert "key" not in labels
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
index ed2d677fce9..74bb2f63342 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
@@ -3167,6 +3167,13 @@ class TestWorkerSets:
"labels": {"echo": "test"},
"celery": {"enableDefault": False, "sets": [{"name": "set1",
"labels": {"test": "echo"}}]},
},
+ {
+ "celery": {
+ "enableDefault": False,
+ "labels": {"echo": "test"},
+ "sets": [{"name": "set1", "labels": {"test": "echo"}}],
+ },
+ },
],
)
def test_overwrite_labels(self, workers_values):
