This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6cbf4c8bc41 Add workers.celery.extraVolumes &
workers.kubernetes.extraVolumes (#64746)
6cbf4c8bc41 is described below
commit 6cbf4c8bc41a06e4f3a1ab28b05e89083cdb8b17
Author: Przemysław Mirowski <[email protected]>
AuthorDate: Fri Apr 10 22:38:31 2026 +0200
Add workers.celery.extraVolumes & workers.kubernetes.extraVolumes (#64746)
* Seperate extraVolumes & extraVolumeMounts in values.yaml
* Refactor workers extraVolumes-related tests
* Add workers.celery.extraVolumes & workers.kubernetes.extraVolumes
* Add newsfragment
---
chart/files/pod-template-file.kubernetes-helm-yaml | 4 +--
chart/newsfragments/64746.significant.rst | 1 +
chart/values.schema.json | 18 +++++++++++-
chart/values.yaml | 32 +++++++++++++++++++---
.../airflow_aux/test_pod_template_file.py | 28 +++++++++++++++----
.../tests/helm_tests/airflow_core/test_worker.py | 27 ++++++++++++++++--
.../helm_tests/airflow_core/test_worker_sets.py | 12 ++++++++
7 files changed, 106 insertions(+), 16 deletions(-)
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml
b/chart/files/pod-template-file.kubernetes-helm-yaml
index d0279bd3320..23fe2fc1421 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -287,6 +287,6 @@ spec:
- name: kerberos-ccache
emptyDir: {}
{{- end }}
- {{- if .Values.workers.extraVolumes }}
- {{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 2 }}
+ {{- if or .Values.workers.kubernetes.extraVolumes
.Values.workers.extraVolumes }}
+ {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumes | default
.Values.workers.extraVolumes)) . | nindent 2 }}
{{- end }}
diff --git a/chart/newsfragments/64746.significant.rst
b/chart/newsfragments/64746.significant.rst
new file mode 100644
index 00000000000..9aafcef7696
--- /dev/null
+++ b/chart/newsfragments/64746.significant.rst
@@ -0,0 +1 @@
+``workers.extraVolumes`` field is now deprecated in favor of
``workers.celery.extraVolumes`` and ``workers.kubernetes.extraVolumes``. Please
update your configuration accordingly.
diff --git a/chart/values.schema.json b/chart/values.schema.json
index eda22559b23..475fd97eb8d 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2317,7 +2317,7 @@
}
},
"extraVolumes": {
- "description": "Additional volumes attached to the Airflow
Celery workers and pods created with pod-template-file.",
+ "description": "Additional volumes attached to the Airflow
Celery workers and pods created with pod-template-file (deprecated, use
``workers.celery.extraVolumes`` and/or ``workers.kubernetes.extraVolumes``
instead).",
"type": "array",
"default": [],
"items": {
@@ -3396,6 +3396,14 @@
"$ref":
"#/definitions/io.k8s.api.core.v1.Container"
}
},
+ "extraVolumes": {
+ "description": "Additional volumes attached to the
Airflow Celery workers.",
+ "type": "array",
+ "default": [],
+ "items": {
+ "$ref":
"#/definitions/io.k8s.api.core.v1.Volume"
+ }
+ },
"extraPorts": {
"description": "Expose additional ports of Airflow
Celery worker container.",
"type": "array",
@@ -3902,6 +3910,14 @@
"$ref":
"#/definitions/io.k8s.api.core.v1.Container"
}
},
+ "extraVolumes": {
+ "description": "Additional volumes attached to the
pods created with pod-template-file.",
+ "type": "array",
+ "default": [],
+ "items": {
+ "$ref":
"#/definitions/io.k8s.api.core.v1.Volume"
+ }
+ },
"nodeSelector": {
"description": "Select certain nodes for pods
created with pod-template-file.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index 309b5490951..9db202b4a65 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -1058,10 +1058,10 @@ workers:
# instead)
extraInitContainers: []
- # Additional volumes and volume mounts attached to the
- # Airflow Celery workers and pods created with pod-template-file
+ # Additional volumes attached to the Airflow Celery workers
+ # and pods created with pod-template-file
+ # (deprecated, use `workers.celery.extraVolumes` and/or
`workers.kubernetes.extraVolumes` instead)
extraVolumes: []
- extraVolumeMounts: []
# Mount additional volumes into workers pods. It can be templated like in
the following example:
# extraVolumes:
# - name: my-templated-extra-volume
@@ -1069,7 +1069,11 @@ workers:
# secretName: '{{ include "my_secret_template" . }}'
# defaultMode: 0640
# optional: true
- #
+
+ # Additional volume mounts attached to the Airflow Celery workers
+ # and pods created with pod-template-file
+ extraVolumeMounts: []
+ # Mount additional volumes into workers pods. It can be templated like in
the following example:
# extraVolumeMounts:
# - name: my-templated-extra-volume
# mountPath: "{{ .Values.my_custom_path }}"
@@ -1457,6 +1461,16 @@ workers:
# Add additional init containers into Airflow Celery workers (templated)
extraInitContainers: []
+ # Additional volumes attached to the Airflow Celery workers
+ extraVolumes: []
+ # Mount additional volumes into workers pods. It can be templated like in
the following example:
+ # extraVolumes:
+ # - name: my-templated-extra-volume
+ # secret:
+ # secretName: '{{ include "my_secret_template" . }}'
+ # defaultMode: 0640
+ # optional: true
+
# Expose additional ports of Airflow Celery workers. These can be used for
additional metric collection.
extraPorts: []
@@ -1627,6 +1641,16 @@ workers:
# Add additional init containers into pods created with pod-template-file
(templated)
extraInitContainers: []
+ # Additional volumes attached to the pods created with pod-template-file
+ extraVolumes: []
+ # Mount additional volumes into workers pods. It can be templated like in
the following example:
+ # extraVolumes:
+ # - name: my-templated-extra-volume
+ # secret:
+ # secretName: '{{ include "my_secret_template" . }}'
+ # defaultMode: 0640
+ # optional: true
+
# Select certain nodes for pods created with pod-template-file
nodeSelector: {}
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
index 82d9bf2fc2c..f903c56b04a 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py
@@ -924,11 +924,31 @@ class TestPodTemplateFile:
assert jmespath.search("spec.securityContext.runAsUser", docs[0]) == 1
- def test_should_create_valid_volume_mount_and_volume(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}",
"emptyDir": {}}]},
+ {"kubernetes": {"extraVolumes": [{"name": "test-volume-{{
.Chart.Name }}", "emptyDir": {}}]}},
+ {
+ "extraVolumes": [{"name": "test", "emptyDir": {}}],
+ "kubernetes": {"extraVolumes": [{"name": "test-volume-{{
.Chart.Name }}", "emptyDir": {}}]},
+ },
+ ],
+ )
+ def test_should_create_valid_volume(self, workers_values):
+ docs = render_chart(
+ values={"workers": workers_values},
+ show_only=["templates/pod-template-file.yaml"],
+ chart_dir=self.temp_chart_dir,
+ )
+
+ # [2:] -> Skipping logs and config volumes
+ assert jmespath.search("spec.volumes[2:].name", docs[0]) ==
["test-volume-airflow"]
+
+ def test_should_create_valid_volume_mount(self):
docs = render_chart(
values={
"workers": {
- "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}",
"emptyDir": {}}],
"extraVolumeMounts": [
{"name": "test-volume-{{ .Chart.Name }}", "mountPath":
"/opt/test"}
],
@@ -938,10 +958,6 @@ class TestPodTemplateFile:
chart_dir=self.temp_chart_dir,
)
- assert "test-volume-airflow" in jmespath.search(
- "spec.volumes[*].name",
- docs[0],
- )
assert "test-volume-airflow" in jmespath.search(
"spec.containers[0].volumeMounts[*].name",
docs[0],
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
index 9c103af2040..1db7e103358 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py
@@ -340,12 +340,34 @@ class TestWorker:
{"name": "release-name-test-init-container"}
]
- def test_should_add_extra_volume_and_extra_volume_mount(self):
+ @pytest.mark.parametrize(
+ "workers_values",
+ [
+ {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}",
"emptyDir": {}}]},
+ {"celery": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name
}}", "emptyDir": {}}]}},
+ {
+ "extraVolumes": [{"name": "test", "emptyDir": {}}],
+ "celery": {"extraVolumes": [{"name": "test-volume-{{
.Chart.Name }}", "emptyDir": {}}]},
+ },
+ ],
+ )
+ def test_should_add_extra_volume(self, workers_values):
+ docs = render_chart(
+ values={
+ "executor": "CeleryExecutor",
+ "workers": workers_values,
+ },
+ show_only=["templates/workers/worker-deployment.yaml"],
+ )
+
+ # [:-1] -> Skipping config volume
+ assert jmespath.search("spec.template.spec.volumes[:-1].name",
docs[0]) == ["test-volume-airflow"]
+
+ def test_should_add_extra_volume_mount(self):
docs = render_chart(
values={
"executor": "CeleryExecutor",
"workers": {
- "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}",
"emptyDir": {}}],
"extraVolumeMounts": [
{"name": "test-volume-{{ .Chart.Name }}", "mountPath":
"/opt/test"}
],
@@ -354,7 +376,6 @@ class TestWorker:
show_only=["templates/workers/worker-deployment.yaml"],
)
- assert jmespath.search("spec.template.spec.volumes[0].name", docs[0])
== "test-volume-airflow"
assert (
jmespath.search("spec.template.spec.containers[0].volumeMounts[0].name",
docs[0])
== "test-volume-airflow"
diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
index 898f630ff44..1d2149f3966 100644
--- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
+++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py
@@ -2564,6 +2564,18 @@ class TestWorkerSets:
],
},
},
+ {
+ "celery": {
+ "enableDefault": False,
+ "extraVolumes": [{"name": "test", "emptyDir": {}}],
+ "sets": [
+ {
+ "name": "set1",
+ "extraVolumes": [{"name": "test-volume-{{
.Chart.Name }}", "emptyDir": {}}],
+ }
+ ],
+ },
+ },
],
)
def test_overwrite_extra_volumes(self, workers_values):
