jedcunningham commented on code in PR #58155:
URL: https://github.com/apache/airflow/pull/58155#discussion_r2551911526
##########
chart/templates/database-cleanup/database-cleanup-cronjob.yaml:
##########
@@ -0,0 +1,121 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup CronJob
+#################################
+{{- if and .Values.databaseCleanup.enabled (or .Values.databaseCleanup.command
.Values.databaseCleanup.args) }}
+{{- $nodeSelector := or .Values.databaseCleanup.nodeSelector
.Values.nodeSelector }}
+{{- $affinity := or .Values.databaseCleanup.affinity .Values.affinity }}
+{{- $tolerations := or .Values.databaseCleanup.tolerations .Values.tolerations
}}
+{{- $topologySpreadConstraints := or
.Values.databaseCleanup.topologySpreadConstraints
.Values.topologySpreadConstraints }}
+{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values.databaseCleanup) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values.databaseCleanup) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: {{ include "airflow.fullname" . }}-database-cleanup
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- with .Values.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.databaseCleanup.jobAnnotations }}
+ annotations: {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ schedule: "{{ tpl .Values.databaseCleanup.schedule . }}"
+ # The cron job does not allow concurrent runs; if it is time for a new job
run and the previous job run hasn't finished yet, the cron job skips the new
job run
+ concurrencyPolicy: Forbid
+ {{- if not ( eq .Values.databaseCleanup.failedJobsHistoryLimit nil) }}
+ failedJobsHistoryLimit: {{ .Values.databaseCleanup.failedJobsHistoryLimit }}
+ {{- end }}
+ {{- if not (eq .Values.databaseCleanup.successfulJobsHistoryLimit nil) }}
+ successfulJobsHistoryLimit: {{
.Values.databaseCleanup.successfulJobsHistoryLimit }}
+ {{- end }}
+ jobTemplate:
+ spec:
+ backoffLimit: 1
+ template:
+ metadata:
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
Review Comment:
```suggestion
component: database-cleanup
```
##########
chart/templates/database-cleanup/database-cleanup-cronjob.yaml:
##########
@@ -0,0 +1,121 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup CronJob
+#################################
+{{- if and .Values.databaseCleanup.enabled (or .Values.databaseCleanup.command
.Values.databaseCleanup.args) }}
+{{- $nodeSelector := or .Values.databaseCleanup.nodeSelector
.Values.nodeSelector }}
+{{- $affinity := or .Values.databaseCleanup.affinity .Values.affinity }}
+{{- $tolerations := or .Values.databaseCleanup.tolerations .Values.tolerations
}}
+{{- $topologySpreadConstraints := or
.Values.databaseCleanup.topologySpreadConstraints
.Values.topologySpreadConstraints }}
+{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values.databaseCleanup) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values.databaseCleanup) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: {{ include "airflow.fullname" . }}-database-cleanup
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- with .Values.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.databaseCleanup.jobAnnotations }}
+ annotations: {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ schedule: "{{ tpl .Values.databaseCleanup.schedule . }}"
+ # The cron job does not allow concurrent runs; if it is time for a new job
run and the previous job run hasn't finished yet, the cron job skips the new
job run
+ concurrencyPolicy: Forbid
+ {{- if not ( eq .Values.databaseCleanup.failedJobsHistoryLimit nil) }}
+ failedJobsHistoryLimit: {{ .Values.databaseCleanup.failedJobsHistoryLimit }}
+ {{- end }}
+ {{- if not (eq .Values.databaseCleanup.successfulJobsHistoryLimit nil) }}
+ successfulJobsHistoryLimit: {{
.Values.databaseCleanup.successfulJobsHistoryLimit }}
+ {{- end }}
+ jobTemplate:
+ spec:
+ backoffLimit: 1
+ template:
+ metadata:
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
+ release: {{ .Release.Name }}
+ {{- if or (.Values.labels) (.Values.databaseCleanup.labels) }}
+ {{- mustMerge .Values.databaseCleanup.labels .Values.labels |
toYaml | nindent 12 }}
+ {{- end }}
+ annotations:
+ sidecar.istio.io/inject: "false"
Review Comment:
```suggestion
```
Not sure we need this? Probably should drop it from the pod cleaner too.
##########
helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup_pods.py:
##########
@@ -0,0 +1,464 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+import pytest
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestDatabaseCleanupDeployment:
+ """Tests database cleanup pods deployments."""
+
+ def test_should_have_a_schedule_with_defaults(self):
+ doc = render_chart(
+ values={
+ "databaseCleanup": {"enabled": True},
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )[0]
+
+ assert doc["spec"]["schedule"] == "0 0 * * 0"
+
+ cron_tests = [
+ ("release-name", "*/5 * * * *", "*/5 * * * *"),
+ ("something-else", "@hourly", "@hourly"),
+ (
+ "custom-name",
+ '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 *
* * *',
+ "7-59/15 * * * *",
+ ),
+ (
+ "airflow-rules",
+ '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 *
* * *',
+ "10-59/15 * * * *",
+ ),
+ ]
+
+ @pytest.mark.parametrize(
+ ("release_name", "schedule_value", "schedule_result"),
+ cron_tests,
+ ids=[x[0] for x in cron_tests],
+ )
+ def test_should_work_with_custom_schedule_string(self, release_name,
schedule_value, schedule_result):
+ doc = render_chart(
+ name=release_name,
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "schedule": schedule_value,
+ },
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )[0]
+
+ assert doc["spec"]["schedule"] == schedule_result
+
+
+class TestDatabaseCleanupPods:
+ """Tests cleanup of pods."""
+
+ def test_should_create_cronjob_for_enabled_cleanup(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {"enabled": True},
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert (
+
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].name",
docs[0])
+ == "airflow-database-cleanup"
Review Comment:
```suggestion
== "database-cleanup"
```
##########
chart/templates/database-cleanup/database-cleanup-serviceaccount.yaml:
##########
@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup ServiceAccount
+#################################
+{{- if and .Values.databaseCleanup.serviceAccount.create
.Values.databaseCleanup.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{
.Values.databaseCleanup.serviceAccount.automountServiceAccountToken }}
+metadata:
+ name: {{ include "databaseCleanup.serviceAccountName" . }}
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup
Review Comment:
```suggestion
component: database-cleanup
```
##########
chart/templates/database-cleanup/database-cleanup-cronjob.yaml:
##########
@@ -0,0 +1,121 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup CronJob
+#################################
+{{- if and .Values.databaseCleanup.enabled (or .Values.databaseCleanup.command
.Values.databaseCleanup.args) }}
+{{- $nodeSelector := or .Values.databaseCleanup.nodeSelector
.Values.nodeSelector }}
+{{- $affinity := or .Values.databaseCleanup.affinity .Values.affinity }}
+{{- $tolerations := or .Values.databaseCleanup.tolerations .Values.tolerations
}}
+{{- $topologySpreadConstraints := or
.Values.databaseCleanup.topologySpreadConstraints
.Values.topologySpreadConstraints }}
+{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values.databaseCleanup) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values.databaseCleanup) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: {{ include "airflow.fullname" . }}-database-cleanup
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- with .Values.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+ {{- with .Values.databaseCleanup.jobAnnotations }}
+ annotations: {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ schedule: "{{ tpl .Values.databaseCleanup.schedule . }}"
+ # The cron job does not allow concurrent runs; if it is time for a new job
run and the previous job run hasn't finished yet, the cron job skips the new
job run
+ concurrencyPolicy: Forbid
+ {{- if not ( eq .Values.databaseCleanup.failedJobsHistoryLimit nil) }}
+ failedJobsHistoryLimit: {{ .Values.databaseCleanup.failedJobsHistoryLimit }}
+ {{- end }}
+ {{- if not (eq .Values.databaseCleanup.successfulJobsHistoryLimit nil) }}
+ successfulJobsHistoryLimit: {{
.Values.databaseCleanup.successfulJobsHistoryLimit }}
+ {{- end }}
+ jobTemplate:
+ spec:
+ backoffLimit: 1
+ template:
+ metadata:
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
+ release: {{ .Release.Name }}
+ {{- if or (.Values.labels) (.Values.databaseCleanup.labels) }}
+ {{- mustMerge .Values.databaseCleanup.labels .Values.labels |
toYaml | nindent 12 }}
+ {{- end }}
+ annotations:
+ sidecar.istio.io/inject: "false"
+ {{- if .Values.airflowPodAnnotations }}
+ {{- toYaml .Values.airflowPodAnnotations | nindent 12 }}
+ {{- end }}
+ {{- if .Values.databaseCleanup.podAnnotations }}
+ {{- toYaml .Values.databaseCleanup.podAnnotations | nindent 12 }}
+ {{- end }}
+ spec:
+ restartPolicy: Never
+ {{- if .Values.databaseCleanup.priorityClassName }}
+ priorityClassName: {{ .Values.databaseCleanup.priorityClassName }}
+ {{- end }}
+ nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
+ affinity: {{- toYaml $affinity | nindent 12 }}
+ {{- if .Values.schedulerName }}
+ schedulerName: {{ .Values.schedulerName }}
+ {{- end }}
+ tolerations: {{- toYaml $tolerations | nindent 12 }}
+ topologySpreadConstraints: {{- toYaml $topologySpreadConstraints |
nindent 12 }}
+ serviceAccountName: {{ include "databaseCleanup.serviceAccountName"
. }}
+ {{- if or .Values.registry.secretName .Values.registry.connection }}
+ imagePullSecrets:
+ - name: {{ template "registry_secret" . }}
+ {{- end }}
+ securityContext: {{ $securityContext | nindent 12 }}
+ containers:
+ - name: airflow-database-cleanup
Review Comment:
```suggestion
- name: database-cleanup
```
##########
helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup_pods.py:
##########
@@ -0,0 +1,464 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+import pytest
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestDatabaseCleanupDeployment:
+ """Tests database cleanup pods deployments."""
+
+ def test_should_have_a_schedule_with_defaults(self):
+ doc = render_chart(
+ values={
+ "databaseCleanup": {"enabled": True},
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )[0]
+
+ assert doc["spec"]["schedule"] == "0 0 * * 0"
+
+ cron_tests = [
+ ("release-name", "*/5 * * * *", "*/5 * * * *"),
+ ("something-else", "@hourly", "@hourly"),
+ (
+ "custom-name",
+ '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 *
* * *',
+ "7-59/15 * * * *",
+ ),
+ (
+ "airflow-rules",
+ '{{- add 3 (regexFind ".$" (adler32sum .Release.Name)) -}}-59/15 *
* * *',
+ "10-59/15 * * * *",
+ ),
+ ]
+
+ @pytest.mark.parametrize(
+ ("release_name", "schedule_value", "schedule_result"),
+ cron_tests,
+ ids=[x[0] for x in cron_tests],
+ )
+ def test_should_work_with_custom_schedule_string(self, release_name,
schedule_value, schedule_result):
+ doc = render_chart(
+ name=release_name,
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "schedule": schedule_value,
+ },
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )[0]
+
+ assert doc["spec"]["schedule"] == schedule_result
+
+
+class TestDatabaseCleanupPods:
+ """Tests cleanup of pods."""
+
+ def test_should_create_cronjob_for_enabled_cleanup(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {"enabled": True},
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert (
+
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].name",
docs[0])
+ == "airflow-database-cleanup"
+ )
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].image",
docs[0]).startswith(
+ "apache/airflow"
+ )
+ assert {"name": "config", "configMap": {"name":
"release-name-config"}} in jmespath.search(
+ "spec.jobTemplate.spec.template.spec.volumes", docs[0]
+ )
+ assert {
+ "name": "config",
+ "mountPath": "/opt/airflow/airflow.cfg",
+ "subPath": "airflow.cfg",
+ "readOnly": True,
+ } in
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].volumeMounts",
docs[0])
+ assert "successfulJobsHistoryLimit" in docs[0]["spec"]
+ assert "failedJobsHistoryLimit" in docs[0]["spec"]
+
+ def test_should_pass_validation_with_v1beta1_api(self):
+ render_chart(
+ values={"databaseCleanup": {"enabled": True}},
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ kubernetes_version="1.16.0",
+ ) # checks that no validation exception is raised
+
+ def test_should_change_image_when_set_airflow_image(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {"enabled": True},
+ "images": {"airflow": {"repository": "airflow", "tag":
"test"}},
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert (
+
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].image",
docs[0])
+ == "airflow:test"
+ )
+
+ def test_should_create_valid_affinity_tolerations_and_node_selector(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In",
"values": ["true"]},
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "tolerations": [
+ {"key": "dynamic-pods", "operator": "Equal", "value":
"true", "effect": "NoSchedule"}
+ ],
+ "nodeSelector": {"diskType": "ssd"},
+ }
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert jmespath.search("kind", docs[0]) == "CronJob"
+ assert (
+ jmespath.search(
+ "spec.jobTemplate.spec.template.spec.affinity.nodeAffinity."
+ "requiredDuringSchedulingIgnoredDuringExecution."
+ "nodeSelectorTerms[0]."
+ "matchExpressions[0]."
+ "key",
+ docs[0],
+ )
+ == "foo"
+ )
+ assert (
+ jmespath.search(
+ "spec.jobTemplate.spec.template.spec.nodeSelector.diskType",
+ docs[0],
+ )
+ == "ssd"
+ )
+ assert (
+ jmespath.search(
+ "spec.jobTemplate.spec.template.spec.tolerations[0].key",
+ docs[0],
+ )
+ == "dynamic-pods"
+ )
+
+ def test_scheduler_name(self):
+ docs = render_chart(
+ values={"databaseCleanup": {"enabled": True}, "schedulerName":
"airflow-scheduler"},
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert (
+ jmespath.search(
+ "spec.jobTemplate.spec.template.spec.schedulerName",
+ docs[0],
+ )
+ == "airflow-scheduler"
+ )
+
+ def test_retention_days_changed(self):
+ docs = render_chart(
+ values={"databaseCleanup": {"enabled": True, "retentionDays": 10}},
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].args",
docs[0]) == [
+ "-c",
+ 'CLEAN_TS=$(date -d "-10 days" +"%Y-%m-%dT%H:%M:%S"); echo
"Cleaning up metadata DB entries older than ${CLEAN_TS}"; exec airflow db clean
--clean-before-timestamp "${CLEAN_TS}" --yes --verbose',
+ ]
+
+ def test_default_command_and_args(self):
+ docs = render_chart(
+ values={"databaseCleanup": {"enabled": True}},
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].command",
docs[0]) == [
+ "bash"
+ ]
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].args",
docs[0]) == [
+ "-c",
+ 'CLEAN_TS=$(date -d "-90 days" +"%Y-%m-%dT%H:%M:%S"); echo
"Cleaning up metadata DB entries older than ${CLEAN_TS}"; exec airflow db clean
--clean-before-timestamp "${CLEAN_TS}" --yes --verbose',
+ ]
+
+ def test_should_add_extraEnvs(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
+ },
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert {"name": "TEST_ENV_1", "value": "test_env_1"} in
jmespath.search(
+ "spec.jobTemplate.spec.template.spec.containers[0].env", docs[0]
+ )
+
+ @pytest.mark.parametrize("command", [None, ["custom", "command"]])
+ @pytest.mark.parametrize("args", [None, ["custom", "args"]])
+ def test_command_and_args_overrides(self, command, args):
+ docs = render_chart(
+ values={"databaseCleanup": {"enabled": True, "command": command,
"args": args}},
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ if not command and not args:
+ assert not docs, (
+ "The CronJob should not be created if command and args are
null even if enabled is true"
+ )
+ else:
+ assert command == jmespath.search(
+ "spec.jobTemplate.spec.template.spec.containers[0].command",
docs[0]
+ )
+ assert args ==
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].args",
docs[0])
+
+ def test_command_and_args_overrides_are_templated(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "command": ["{{ .Release.Name }}"],
+ "args": ["{{ .Release.Service }}"],
+ }
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].command",
docs[0]) == [
+ "release-name"
+ ]
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].args",
docs[0]) == ["Helm"]
+
+ def test_should_set_labels_to_jobs_from_cronjob(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {"enabled": True},
+ "labels": {"project": "airflow"},
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert
jmespath.search("spec.jobTemplate.spec.template.metadata.labels", docs[0]) == {
+ "tier": "airflow",
+ "component": "airflow-database-cleanup-pods",
+ "release": "release-name",
+ "project": "airflow",
+ }
+
+ def test_should_add_component_specific_labels(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "labels": {"test_label": "test_label_value"},
+ },
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert "test_label" in
jmespath.search("spec.jobTemplate.spec.template.metadata.labels", docs[0])
+ assert (
+ jmespath.search("spec.jobTemplate.spec.template.metadata.labels",
docs[0])["test_label"]
+ == "test_label_value"
+ )
+
+ def test_should_add_component_specific_annotations(self):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "jobAnnotations": {"test_cronjob_annotation":
"test_cronjob_annotation_value"},
+ "podAnnotations": {"test_pod_annotation":
"test_pod_annotation_value"},
+ },
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert "test_cronjob_annotation" in
jmespath.search("metadata.annotations", docs[0])
+ assert (
+ jmespath.search("metadata.annotations",
docs[0])["test_cronjob_annotation"]
+ == "test_cronjob_annotation_value"
+ )
+ assert "test_pod_annotation" in jmespath.search(
+ "spec.jobTemplate.spec.template.metadata.annotations", docs[0]
+ )
+ assert (
+
jmespath.search("spec.jobTemplate.spec.template.metadata.annotations", docs[0])[
+ "test_pod_annotation"
+ ]
+ == "test_pod_annotation_value"
+ )
+
+ def test_cleanup_resources_are_configurable(self):
+ resources = {
+ "requests": {
+ "cpu": "128m",
+ "memory": "256Mi",
+ },
+ "limits": {
+ "cpu": "256m",
+ "memory": "512Mi",
+ },
+ }
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "resources": resources,
+ },
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert resources == jmespath.search(
+ "spec.jobTemplate.spec.template.spec.containers[0].resources",
docs[0]
+ )
Review Comment:
```suggestion
assert jmespath.search(
"spec.jobTemplate.spec.template.spec.containers[0].resources",
docs[0]
) == resources
```
right is right :)
##########
chart/templates/database-cleanup/database-cleanup-cronjob.yaml:
##########
@@ -0,0 +1,121 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+################################
+## Airflow Database Cleanup CronJob
+#################################
+{{- if and .Values.databaseCleanup.enabled (or .Values.databaseCleanup.command
.Values.databaseCleanup.args) }}
+{{- $nodeSelector := or .Values.databaseCleanup.nodeSelector
.Values.nodeSelector }}
+{{- $affinity := or .Values.databaseCleanup.affinity .Values.affinity }}
+{{- $tolerations := or .Values.databaseCleanup.tolerations .Values.tolerations
}}
+{{- $topologySpreadConstraints := or
.Values.databaseCleanup.topologySpreadConstraints
.Values.topologySpreadConstraints }}
+{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values.databaseCleanup) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values.databaseCleanup) }}
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+ name: {{ include "airflow.fullname" . }}-database-cleanup
+ labels:
+ tier: airflow
+ component: airflow-database-cleanup-pods
Review Comment:
```suggestion
component: database-cleanup
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
