This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 03a305f660 Add support for custom emptyDir config (#34837)
03a305f660 is described below
commit 03a305f6600b822d4f4ab74ed5ae8c0f8905fbc2
Author: Pedro Miranda <[email protected]>
AuthorDate: Wed Oct 11 10:24:43 2023 -0500
Add support for custom emptyDir config (#34837)
---
chart/files/pod-template-file.kubernetes-helm-yaml | 4 +--
.../dag-processor/dag-processor-deployment.yaml | 4 +--
chart/templates/redis/redis-statefulset.yaml | 2 +-
.../templates/scheduler/scheduler-deployment.yaml | 4 +--
.../templates/triggerer/triggerer-deployment.yaml | 4 +--
.../templates/webserver/webserver-deployment.yaml | 2 +-
chart/templates/workers/worker-deployment.yaml | 4 +--
chart/values.schema.json | 18 ++++++++++++
chart/values.yaml | 15 ++++++++++
helm_tests/airflow_aux/test_pod_template_file.py | 33 +++++++++++++++-------
helm_tests/airflow_core/test_dag_processor.py | 19 +++++++++----
helm_tests/airflow_core/test_scheduler.py | 19 +++++++++----
helm_tests/airflow_core/test_triggerer.py | 19 +++++++++----
helm_tests/airflow_core/test_worker.py | 19 +++++++++----
helm_tests/other/test_redis.py | 17 +++++++++++
15 files changed, 137 insertions(+), 46 deletions(-)
diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml
b/chart/files/pod-template-file.kubernetes-helm-yaml
index 48e2c476b9..0c35d4ae2f 100644
--- a/chart/files/pod-template-file.kubernetes-helm-yaml
+++ b/chart/files/pod-template-file.kubernetes-helm-yaml
@@ -117,14 +117,14 @@ spec:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict) .Values.dags.gitSync.emptyDirConfig)
| nindent 6 }}
{{- end }}
{{- if .Values.logs.persistence.enabled }}
- name: logs
persistentVolumeClaim:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else }}
- - emptyDir: {}
+ - emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) |
nindent 6 }}
name: logs
{{- end }}
{{- if and .Values.dags.gitSync.enabled .Values.dags.gitSync.sshKeySecret
}}
diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml
b/chart/templates/dag-processor/dag-processor-deployment.yaml
index 4e87bae3c0..61cd3f56bc 100644
--- a/chart/templates/dag-processor/dag-processor-deployment.yaml
+++ b/chart/templates/dag-processor/dag-processor-deployment.yaml
@@ -238,7 +238,7 @@ spec:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict)
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- end }}
{{- if and .Values.dags.gitSync.enabled
.Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
@@ -255,7 +255,7 @@ spec:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else }}
- name: logs
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) |
nindent 12 }}
{{- end }}
{{- end }}
{{- end }}
diff --git a/chart/templates/redis/redis-statefulset.yaml
b/chart/templates/redis/redis-statefulset.yaml
index 224f38f8f5..62220a7ca5 100644
--- a/chart/templates/redis/redis-statefulset.yaml
+++ b/chart/templates/redis/redis-statefulset.yaml
@@ -106,7 +106,7 @@ spec:
{{- if not .Values.redis.persistence.enabled }}
volumes:
- name: redis-db
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict) .Values.redis.emptyDirConfig) |
nindent 12 }}
{{- else }}
volumeClaimTemplates:
- metadata:
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml
b/chart/templates/scheduler/scheduler-deployment.yaml
index 2ea30e3e2e..8e599b4a28 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -298,7 +298,7 @@ spec:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict)
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
@@ -316,7 +316,7 @@ spec:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $stateful }}
- name: logs
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) |
nindent 12 }}
{{- else }}
volumeClaimTemplates:
- metadata:
diff --git a/chart/templates/triggerer/triggerer-deployment.yaml
b/chart/templates/triggerer/triggerer-deployment.yaml
index bef188f68a..ef6f66db85 100644
--- a/chart/templates/triggerer/triggerer-deployment.yaml
+++ b/chart/templates/triggerer/triggerer-deployment.yaml
@@ -270,7 +270,7 @@ spec:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict)
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | nindent 8 }}
{{- end }}
@@ -287,7 +287,7 @@ spec:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $persistence }}
- name: logs
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) |
nindent 12 }}
{{- else }}
volumeClaimTemplates:
- metadata:
diff --git a/chart/templates/webserver/webserver-deployment.yaml
b/chart/templates/webserver/webserver-deployment.yaml
index 26fe76d9ea..1fa06b0724 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -274,7 +274,7 @@ spec:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict)
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
diff --git a/chart/templates/workers/worker-deployment.yaml
b/chart/templates/workers/worker-deployment.yaml
index 213fadbb03..21f7c8fd2d 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -373,7 +373,7 @@ spec:
claimName: {{ template "airflow_dags_volume_claim" . }}
{{- else if .Values.dags.gitSync.enabled }}
- name: dags
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict)
.Values.dags.gitSync.emptyDirConfig) | nindent 12 }}
{{- if .Values.dags.gitSync.sshKeySecret }}
{{- include "git_sync_ssh_key_volume" . | indent 8 }}
{{- end }}
@@ -384,7 +384,7 @@ spec:
claimName: {{ template "airflow_logs_volume_claim" . }}
{{- else if not $persistence }}
- name: logs
- emptyDir: {}
+ emptyDir: {{- toYaml (default (dict) .Values.logs.emptyDirConfig) |
nindent 12 }}
{{- else }}
volumeClaimTemplates:
- metadata:
diff --git a/chart/values.schema.json b/chart/values.schema.json
index dce38a205f..8a11d6e672 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -6164,6 +6164,12 @@
}
}
},
+ "emptyDirConfig": {
+ "description": "Configuration for redis empty dir volume.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+ "default": null
+ },
"resources": {
"description": "Resources for the Redis pods",
"type": "object",
@@ -7046,6 +7052,12 @@
}
]
},
+ "emptyDirConfig": {
+ "description": "Configuration for dags empty dir
volume.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+ "default": null
+ },
"containerLifecycleHooks": {
"description": "Container Lifecycle Hooks
definition for the git sync sidecar. If not set, the values from global
`containerLifecycleHooks` will be used.",
"type": "object",
@@ -7235,6 +7247,12 @@
"default": null
}
}
+ },
+ "emptyDirConfig": {
+ "description": "Configuration for logs empty dir volume.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource",
+ "default": null
}
}
}
diff --git a/chart/values.yaml b/chart/values.yaml
index 7b238523ab..bc7d19355d 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -2066,6 +2066,11 @@ redis:
# Annotations to add to redis volumes
annotations: {}
+ # Configuration for empty dir volume (if redis.persistence.enabled == false)
+ # emptyDirConfig:
+ # sizeLimit: 1Gi
+ # medium: Memory
+
resources: {}
# limits:
# cpu: 100m
@@ -2440,6 +2445,11 @@ dags:
# - name: ""
# value: ""
+ # Configuration for empty dir volume
+ # emptyDirConfig:
+ # sizeLimit: 1Gi
+ # medium: Memory
+
resources: {}
# limits:
# cpu: 100m
@@ -2449,6 +2459,11 @@ dags:
# memory: 128Mi
logs:
+ # Configuration for empty dir volume (if logs.persistence.enabled == false)
+ # emptyDirConfig:
+ # sizeLimit: 1Gi
+ # medium: Memory
+
persistence:
# Enable persistent volume for storing logs
enabled: false
diff --git a/helm_tests/airflow_aux/test_pod_template_file.py
b/helm_tests/airflow_aux/test_pod_template_file.py
index e56d77c55a..c1a831e6b1 100644
--- a/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm_tests/airflow_aux/test_pod_template_file.py
@@ -267,29 +267,42 @@ class TestPodTemplateFile:
"spec.volumes", docs[0]
)
- def test_should_use_empty_dir_for_gitsync_without_persistence(self):
+ @pytest.mark.parametrize(
+ "dags_gitsync_values, expected",
+ [
+ ({"enabled": True}, {"emptyDir": {}}),
+ ({"enabled": True, "emptyDirConfig": {"sizeLimit": "10Gi"}},
{"emptyDir": {"sizeLimit": "10Gi"}}),
+ ],
+ )
+ def test_should_use_empty_dir_for_gitsync_without_persistence(self,
dags_gitsync_values, expected):
docs = render_chart(
- values={"dags": {"gitSync": {"enabled": True}}},
+ values={"dags": {"gitSync": dags_gitsync_values}},
show_only=["templates/pod-template-file.yaml"],
chart_dir=self.temp_chart_dir,
)
-
- assert {"name": "dags", "emptyDir": {}} in
jmespath.search("spec.volumes", docs[0])
+ assert {"name": "dags", **expected} in jmespath.search("spec.volumes",
docs[0])
@pytest.mark.parametrize(
- "log_persistence_values, expected",
+ "log_values, expected",
[
- ({"enabled": False}, {"emptyDir": {}}),
- ({"enabled": True}, {"persistentVolumeClaim": {"claimName":
"release-name-logs"}}),
+ ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
+ (
+ {"persistence": {"enabled": False}, "emptyDirConfig":
{"sizeLimit": "10Gi"}},
+ {"emptyDir": {"sizeLimit": "10Gi"}},
+ ),
+ (
+ {"persistence": {"enabled": True}},
+ {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+ ),
(
- {"enabled": True, "existingClaim": "test-claim"},
+ {"persistence": {"enabled": True, "existingClaim":
"test-claim"}},
{"persistentVolumeClaim": {"claimName": "test-claim"}},
),
],
)
- def test_logs_persistence_changes_volume(self, log_persistence_values,
expected):
+ def test_logs_persistence_changes_volume(self, log_values, expected):
docs = render_chart(
- values={"logs": {"persistence": log_persistence_values}},
+ values={"logs": log_values},
show_only=["templates/pod-template-file.yaml"],
chart_dir=self.temp_chart_dir,
)
diff --git a/helm_tests/airflow_core/test_dag_processor.py
b/helm_tests/airflow_core/test_dag_processor.py
index 2ab941f8e2..b9843c1594 100644
--- a/helm_tests/airflow_core/test_dag_processor.py
+++ b/helm_tests/airflow_core/test_dag_processor.py
@@ -368,20 +368,27 @@ class TestDagProcessor:
)
@pytest.mark.parametrize(
- "log_persistence_values, expected_volume",
+ "log_values, expected_volume",
[
- ({"enabled": False}, {"emptyDir": {}}),
- ({"enabled": True}, {"persistentVolumeClaim": {"claimName":
"release-name-logs"}}),
+ ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
(
- {"enabled": True, "existingClaim": "test-claim"},
+ {"persistence": {"enabled": False}, "emptyDirConfig":
{"sizeLimit": "10Gi"}},
+ {"emptyDir": {"sizeLimit": "10Gi"}},
+ ),
+ (
+ {"persistence": {"enabled": True}},
+ {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+ ),
+ (
+ {"persistence": {"enabled": True, "existingClaim":
"test-claim"}},
{"persistentVolumeClaim": {"claimName": "test-claim"}},
),
],
)
- def test_logs_persistence_changes_volume(self, log_persistence_values,
expected_volume):
+ def test_logs_persistence_changes_volume(self, log_values,
expected_volume):
docs = render_chart(
values={
- "logs": {"persistence": log_persistence_values},
+ "logs": log_values,
"dagProcessor": {"enabled": True},
},
show_only=["templates/dag-processor/dag-processor-deployment.yaml"],
diff --git a/helm_tests/airflow_core/test_scheduler.py
b/helm_tests/airflow_core/test_scheduler.py
index d1007d8700..c7336bbc98 100644
--- a/helm_tests/airflow_core/test_scheduler.py
+++ b/helm_tests/airflow_core/test_scheduler.py
@@ -426,19 +426,26 @@ class TestScheduler:
)
@pytest.mark.parametrize(
- "log_persistence_values, expected_volume",
+ "log_values, expected_volume",
[
- ({"enabled": False}, {"emptyDir": {}}),
- ({"enabled": True}, {"persistentVolumeClaim": {"claimName":
"release-name-logs"}}),
+ ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
(
- {"enabled": True, "existingClaim": "test-claim"},
+ {"persistence": {"enabled": False}, "emptyDirConfig":
{"sizeLimit": "10Gi"}},
+ {"emptyDir": {"sizeLimit": "10Gi"}},
+ ),
+ (
+ {"persistence": {"enabled": True}},
+ {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+ ),
+ (
+ {"persistence": {"enabled": True, "existingClaim":
"test-claim"}},
{"persistentVolumeClaim": {"claimName": "test-claim"}},
),
],
)
- def test_logs_persistence_changes_volume(self, log_persistence_values,
expected_volume):
+ def test_logs_persistence_changes_volume(self, log_values,
expected_volume):
docs = render_chart(
- values={"logs": {"persistence": log_persistence_values}},
+ values={"logs": log_values},
show_only=["templates/scheduler/scheduler-deployment.yaml"],
)
diff --git a/helm_tests/airflow_core/test_triggerer.py
b/helm_tests/airflow_core/test_triggerer.py
index 6375071849..3c227afdf5 100644
--- a/helm_tests/airflow_core/test_triggerer.py
+++ b/helm_tests/airflow_core/test_triggerer.py
@@ -389,21 +389,28 @@ class TestTriggerer:
)
@pytest.mark.parametrize(
- "log_persistence_values, expected_volume",
+ "log_values, expected_volume",
[
- ({"enabled": False}, {"emptyDir": {}}),
- ({"enabled": True}, {"persistentVolumeClaim": {"claimName":
"release-name-logs"}}),
+ ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
(
- {"enabled": True, "existingClaim": "test-claim"},
+ {"persistence": {"enabled": False}, "emptyDirConfig":
{"sizeLimit": "10Gi"}},
+ {"emptyDir": {"sizeLimit": "10Gi"}},
+ ),
+ (
+ {"persistence": {"enabled": True}},
+ {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+ ),
+ (
+ {"persistence": {"enabled": True, "existingClaim":
"test-claim"}},
{"persistentVolumeClaim": {"claimName": "test-claim"}},
),
],
)
- def test_logs_persistence_changes_volume(self, log_persistence_values,
expected_volume):
+ def test_logs_persistence_changes_volume(self, log_values,
expected_volume):
docs = render_chart(
values={
"triggerer": {"persistence": {"enabled": False}},
- "logs": {"persistence": log_persistence_values},
+ "logs": log_values,
},
show_only=["templates/triggerer/triggerer-deployment.yaml"],
)
diff --git a/helm_tests/airflow_core/test_worker.py
b/helm_tests/airflow_core/test_worker.py
index 80682f9e17..6dfe99d7d4 100644
--- a/helm_tests/airflow_core/test_worker.py
+++ b/helm_tests/airflow_core/test_worker.py
@@ -456,22 +456,29 @@ class TestWorker:
assert livenessprobe is None
@pytest.mark.parametrize(
- "log_persistence_values, expected_volume",
+ "log_values, expected_volume",
[
- ({"enabled": False}, {"emptyDir": {}}),
- ({"enabled": True}, {"persistentVolumeClaim": {"claimName":
"release-name-logs"}}),
+ ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
(
- {"enabled": True, "existingClaim": "test-claim"},
+ {"persistence": {"enabled": False}, "emptyDirConfig":
{"sizeLimit": "10Gi"}},
+ {"emptyDir": {"sizeLimit": "10Gi"}},
+ ),
+ (
+ {"persistence": {"enabled": True}},
+ {"persistentVolumeClaim": {"claimName": "release-name-logs"}},
+ ),
+ (
+ {"persistence": {"enabled": True, "existingClaim":
"test-claim"}},
{"persistentVolumeClaim": {"claimName": "test-claim"}},
),
],
)
- def test_logs_persistence_changes_volume(self, log_persistence_values,
expected_volume):
+ def test_logs_persistence_changes_volume(self, log_values,
expected_volume):
docs = render_chart(
values={
"executor": "CeleryExecutor",
"workers": {"persistence": {"enabled": False}},
- "logs": {"persistence": log_persistence_values},
+ "logs": log_values,
},
show_only=["templates/workers/worker-deployment.yaml"],
)
diff --git a/helm_tests/other/test_redis.py b/helm_tests/other/test_redis.py
index 466e7b6bf1..82266e3f47 100644
--- a/helm_tests/other/test_redis.py
+++ b/helm_tests/other/test_redis.py
@@ -340,6 +340,23 @@ class TestRedis:
)
assert {"foo": "bar"} ==
jmespath.search("spec.volumeClaimTemplates[0].metadata.annotations", docs[0])
+ @pytest.mark.parametrize(
+ "redis_values, expected",
+ [
+ ({"persistence": {"enabled": False}}, {"emptyDir": {}}),
+ (
+ {"persistence": {"enabled": False}, "emptyDirConfig":
{"sizeLimit": "10Gi"}},
+ {"emptyDir": {"sizeLimit": "10Gi"}},
+ ),
+ ],
+ )
+ def test_should_use_empty_dir_on_persistence_disabled(self, redis_values,
expected):
+ docs = render_chart(
+ values={"redis": redis_values},
+ show_only=["templates/redis/redis-statefulset.yaml"],
+ )
+ assert {"name": "redis-db", **expected} in
jmespath.search("spec.template.spec.volumes", docs[0])
+
class TestRedisServiceAccount:
"""Tests redis service account."""