This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 06e53f2 Chart: configurable number of retention days for log groomers
(#17764)
06e53f2 is described below
commit 06e53f26e5cb2f1ad4aabe05fa12d2db9c66e282
Author: cccs-seb <64980897+cccs-...@users.noreply.github.com>
AuthorDate: Tue Aug 24 19:34:06 2021 -0400
Chart: configurable number of retention days for log groomers (#17764)
---
.../templates/scheduler/scheduler-deployment.yaml | 5 ++++
chart/templates/workers/worker-deployment.yaml | 5 ++++
chart/tests/test_scheduler.py | 30 ++++++++++++++++++++++
chart/tests/test_worker.py | 30 ++++++++++++++++++++++
chart/values.schema.json | 10 ++++++++
chart/values.yaml | 4 +++
6 files changed, 84 insertions(+)
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml
b/chart/templates/scheduler/scheduler-deployment.yaml
index 8c51fd7..25d2554 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -211,6 +211,11 @@ spec:
{{- if .Values.scheduler.logGroomerSidecar.args }}
args: {{ tpl (toYaml .Values.scheduler.logGroomerSidecar.args) . |
nindent 12 }}
{{- end }}
+ {{ if .Values.scheduler.logGroomerSidecar.retentionDays }}
+ env:
+ - name: AIRFLOW__LOG_RETENTION_DAYS
+ value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
+ {{- end }}
volumeMounts:
- name: logs
mountPath: {{ template "airflow_logs" . }}
diff --git a/chart/templates/workers/worker-deployment.yaml
b/chart/templates/workers/worker-deployment.yaml
index 787d387..64abb1a 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -200,6 +200,11 @@ spec:
{{- if .Values.workers.logGroomerSidecar.args }}
args: {{ tpl (toYaml .Values.workers.logGroomerSidecar.args) . |
nindent 12 }}
{{- end }}
+ {{ if .Values.workers.logGroomerSidecar.retentionDays }}
+ env:
+ - name: AIRFLOW__LOG_RETENTION_DAYS
+ value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
+ {{- end }}
resources:
{{ toYaml .Values.workers.logGroomerSidecar.resources | indent 12 }}
volumeMounts:
diff --git a/chart/tests/test_scheduler.py b/chart/tests/test_scheduler.py
index 0c06163..1225790 100644
--- a/chart/tests/test_scheduler.py
+++ b/chart/tests/test_scheduler.py
@@ -354,6 +354,14 @@ class SchedulerTest(unittest.TestCase):
assert jmespath.search("spec.template.spec.containers[1].command",
docs[0]) is None
assert ["bash", "/clean-logs"] ==
jmespath.search("spec.template.spec.containers[1].args", docs[0])
+ def test_log_groomer_collector_default_retention_days(self):
+ docs =
render_chart(show_only=["templates/scheduler/scheduler-deployment.yaml"])
+
+ assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
+ "spec.template.spec.containers[1].env[0].name", docs[0]
+ )
+ assert "15" ==
jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0])
+
@parameterized.expand(
[
(None, None),
@@ -389,6 +397,28 @@ class SchedulerTest(unittest.TestCase):
@parameterized.expand(
[
+ (None, None),
+ (30, "30"),
+ ]
+ )
+ def test_log_groomer_retention_days_overrides(self, retention_days,
retention_result):
+ docs = render_chart(
+ values={"scheduler": {"logGroomerSidecar": {"retentionDays":
retention_days}}},
+ show_only=["templates/scheduler/scheduler-deployment.yaml"],
+ )
+
+ if retention_result:
+ assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
+ "spec.template.spec.containers[1].env[0].name", docs[0]
+ )
+ assert retention_result == jmespath.search(
+ "spec.template.spec.containers[1].env[0].value", docs[0]
+ )
+ else:
+ assert jmespath.search("spec.template.spec.containers[1].env",
docs[0]) is None
+
+ @parameterized.expand(
+ [
({"gitSync": {"enabled": True}},),
({"gitSync": {"enabled": True}, "persistence": {"enabled":
True}},),
]
diff --git a/chart/tests/test_worker.py b/chart/tests/test_worker.py
index 8f4685d..7e79389 100644
--- a/chart/tests/test_worker.py
+++ b/chart/tests/test_worker.py
@@ -423,6 +423,14 @@ class WorkerTest(unittest.TestCase):
assert jmespath.search("spec.template.spec.containers[1].command",
docs[0]) is None
assert ["bash", "/clean-logs"] ==
jmespath.search("spec.template.spec.containers[1].args", docs[0])
+ def test_log_groomer_collector_default_retention_days(self):
+ docs =
render_chart(show_only=["templates/workers/worker-deployment.yaml"])
+
+ assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
+ "spec.template.spec.containers[1].env[0].name", docs[0]
+ )
+ assert "15" ==
jmespath.search("spec.template.spec.containers[1].env[0].value", docs[0])
+
@parameterized.expand(
[
(None, None),
@@ -456,6 +464,28 @@ class WorkerTest(unittest.TestCase):
assert ["RELEASE-NAME"] ==
jmespath.search("spec.template.spec.containers[1].command", docs[0])
assert ["Helm"] ==
jmespath.search("spec.template.spec.containers[1].args", docs[0])
+ @parameterized.expand(
+ [
+ (None, None),
+ (30, "30"),
+ ]
+ )
+ def test_log_groomer_retention_days_overrides(self, retention_days,
retention_result):
+ docs = render_chart(
+ values={"workers": {"logGroomerSidecar": {"retentionDays":
retention_days}}},
+ show_only=["templates/workers/worker-deployment.yaml"],
+ )
+
+ if retention_result:
+ assert "AIRFLOW__LOG_RETENTION_DAYS" == jmespath.search(
+ "spec.template.spec.containers[1].env[0].name", docs[0]
+ )
+ assert retention_result == jmespath.search(
+ "spec.template.spec.containers[1].env[0].value", docs[0]
+ )
+ else:
+ assert jmespath.search("spec.template.spec.containers[1].env",
docs[0]) is None
+
def test_dags_gitsync_sidecar_and_init_container(self):
docs = render_chart(
values={"dags": {"gitSync": {"enabled": True}}},
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 3c21bf2..491404b 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -1132,6 +1132,11 @@
"/clean-logs"
]
},
+ "retentionDays": {
+ "description": "Number of days to retain the logs
when running the Airflow workers log groomer sidecar.",
+ "type": "integer",
+ "default": 15
+ },
"resources": {
"description": "Resources for Airflow workers log
groomer sidecar.",
"type": "object",
@@ -1375,6 +1380,11 @@
"/clean-logs"
]
},
+ "retentionDays": {
+ "description": "Number of days to retain the logs
when running the Airflow scheduler log groomer sidecar.",
+ "type": "integer",
+ "default": 15
+ },
"resources": {
"description": "Resources for log groomer
sidecar.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index a634d1b..a7cbd19d 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -453,6 +453,8 @@ workers:
command: ~
# Args to use when running the Airflow worker log groomer sidecar
(templated).
args: ["bash", "/clean-logs"]
+ # Number of days to retain logs
+ retentionDays: 15
resources: {}
# limits:
# cpu: 100m
@@ -546,6 +548,8 @@ scheduler:
command: ~
# Args to use when running the Airflow scheduler log groomer sidecar
(templated).
args: ["bash", "/clean-logs"]
+ # Number of days to retain logs
+ retentionDays: 15
resources: {}
# limits:
# cpu: 100m
