This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b6115687ad0 Extra tests for db cleanup and fix cronjob image secrets
(#58626)
b6115687ad0 is described below
commit b6115687ad0d4a0a59e7674601987bf9cb721184
Author: Ronaldo Campos <[email protected]>
AuthorDate: Tue Nov 25 20:00:28 2025 +0000
Extra tests for db cleanup and fix cronjob image secrets (#58626)
---
.../database-cleanup/database-cleanup-cronjob.yaml | 5 +---
.../helm_tests/airflow_aux/test_airflow_common.py | 20 +++++++++-----
.../helm_tests/airflow_aux/test_annotations.py | 30 +++++++++++++++++++++
.../airflow_aux/test_basic_helm_chart.py | 6 +++++
helm-tests/tests/helm_tests/security/test_rbac.py | 31 ++++++++++++++++++++++
5 files changed, 82 insertions(+), 10 deletions(-)
diff --git a/chart/templates/database-cleanup/database-cleanup-cronjob.yaml
b/chart/templates/database-cleanup/database-cleanup-cronjob.yaml
index 12e563284b8..78e4e61abd7 100644
--- a/chart/templates/database-cleanup/database-cleanup-cronjob.yaml
+++ b/chart/templates/database-cleanup/database-cleanup-cronjob.yaml
@@ -85,10 +85,7 @@ spec:
tolerations: {{- toYaml $tolerations | nindent 12 }}
topologySpreadConstraints: {{- toYaml $topologySpreadConstraints |
nindent 12 }}
serviceAccountName: {{ include "databaseCleanup.serviceAccountName"
. }}
- {{- if or .Values.registry.secretName .Values.registry.connection }}
- imagePullSecrets:
- - name: {{ template "registry_secret" . }}
- {{- end }}
+ imagePullSecrets: {{- include "image_pull_secrets" . | nindent 12 }}
securityContext: {{ $securityContext | nindent 12 }}
containers:
- name: database-cleanup
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
b/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
index dac2f458242..5c0f4c8b8f1 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
@@ -194,6 +194,7 @@ class TestAirflowCommon:
values={
"airflowPodAnnotations": {"test-annotation/safe-to-evict":
"true"},
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"dagProcessor": {"enabled": True},
},
@@ -206,11 +207,12 @@ class TestAirflowCommon:
"templates/triggerer/triggerer-deployment.yaml",
"templates/dag-processor/dag-processor-deployment.yaml",
"templates/cleanup/cleanup-cronjob.yaml",
+ "templates/database-cleanup/database-cleanup-cronjob.yaml",
],
)
- # Objects in show_only are 8 but only one of Webserver or API server
is created so we have 7 objects
- assert len(k8s_objects) == 7
+ # Objects in show_only are 9 but only one of Webserver or API server
is created so we have 8 objects
+ assert len(k8s_objects) == 8
for k8s_object in k8s_objects:
if k8s_object["kind"] == "CronJob":
@@ -226,6 +228,7 @@ class TestAirflowCommon:
k8s_objects = render_chart(
values={
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"dagProcessor": {"enabled": True},
@@ -257,6 +260,7 @@ class TestAirflowCommon:
},
show_only=[
"templates/cleanup/cleanup-cronjob.yaml",
+ "templates/database-cleanup/database-cleanup-cronjob.yaml",
"templates/flower/flower-deployment.yaml",
"templates/jobs/create-user-job.yaml",
"templates/jobs/migrate-database-job.yaml",
@@ -272,8 +276,8 @@ class TestAirflowCommon:
],
)
- # Objects in show_only are 13 but only one of Webserver or API server
is created so we have 12 objects
- assert len(k8s_objects) == 12
+ # Objects in show_only are 14 but only one of Webserver or API server
is created so we have 13 objects
+ assert len(k8s_objects) == 13
for k8s_object in k8s_objects:
if k8s_object["kind"] == "CronJob":
@@ -464,6 +468,7 @@ class TestAirflowCommon:
"webserver": {"priorityClassName": "low-priority-webserver"},
"workers": {"priorityClassName": "low-priority-worker"},
"cleanup": {"enabled": True, "priorityClassName":
"low-priority-airflow-cleanup-pods"},
+ "databaseCleanup": {"enabled": True, "priorityClassName":
"low-priority-database-cleanup"},
"migrateDatabaseJob": {"priorityClassName":
"low-priority-run-airflow-migrations"},
"createUserJob": {"priorityClassName":
"low-priority-create-user-job"},
},
@@ -477,15 +482,16 @@ class TestAirflowCommon:
"templates/webserver/webserver-deployment.yaml",
"templates/workers/worker-deployment.yaml",
"templates/cleanup/cleanup-cronjob.yaml",
+ "templates/database-cleanup/database-cleanup-cronjob.yaml",
"templates/jobs/migrate-database-job.yaml",
"templates/jobs/create-user-job.yaml",
],
)
- assert len(docs) == 10
+ assert len(docs) == 11
for doc in docs:
component = doc["metadata"]["labels"]["component"]
- if component == "airflow-cleanup-pods":
+ if component in ["airflow-cleanup-pods", "database-cleanup"]:
priority =
doc["spec"]["jobTemplate"]["spec"]["template"]["spec"]["priorityClassName"]
else:
priority = doc["spec"]["template"]["spec"]["priorityClassName"]
@@ -525,6 +531,7 @@ class TestAirflowCommon:
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
},
show_only=[
"templates/flower/flower-deployment.yaml",
@@ -536,6 +543,7 @@ class TestAirflowCommon:
"templates/webserver/webserver-deployment.yaml",
"templates/workers/worker-deployment.yaml",
"templates/cleanup/cleanup-cronjob.yaml",
+ "templates/database-cleanup/database-cleanup-cronjob.yaml",
"templates/jobs/migrate-database-job.yaml",
"templates/jobs/create-user-job.yaml",
],
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py
b/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py
index 50891fe65d0..a8bcacb60e2 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py
@@ -56,6 +56,22 @@ class TestServiceAccountAnnotations:
"example": "cleanup",
},
),
+ (
+ {
+ "databaseCleanup": {
+ "enabled": True,
+ "serviceAccount": {
+ "annotations": {
+ "example": "database-cleanup",
+ },
+ },
+ },
+ },
+
"templates/database-cleanup/database-cleanup-serviceaccount.yaml",
+ {
+ "example": "database-cleanup",
+ },
+ ),
(
{
"scheduler": {
@@ -446,6 +462,20 @@ class TestServiceAccountAnnotations:
"example": "cleanup",
},
),
+ (
+ {
+ "databaseCleanup": {
+ "enabled": True,
+ "podAnnotations": {
+ "example": "database-cleanup",
+ },
+ }
+ },
+ "templates/database-cleanup/database-cleanup-cronjob.yaml",
+ {
+ "example": "database-cleanup",
+ },
+ ),
(
{
"redis": {
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
index 08fe577ef2e..5fc22f00feb 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
@@ -411,6 +411,7 @@ class TestBaseChartTest:
"ingress": {"enabled": True},
"networkPolicies": {"enabled": True},
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"dagProcessor": {"enabled": True},
"logs": {"persistence": {"enabled": True}},
@@ -429,6 +430,7 @@ class TestBaseChartTest:
kind_names_tuples = [
(f"{release_name}-airflow-cleanup", "ServiceAccount",
"airflow-cleanup-pods"),
+ (f"{release_name}-airflow-database-cleanup", "ServiceAccount",
"database-cleanup"),
(f"{release_name}-config", "ConfigMap", "config"),
(f"{release_name}-airflow-create-user-job", "ServiceAccount",
"create-user-job"),
(f"{release_name}-airflow-flower", "ServiceAccount", "flower"),
@@ -446,6 +448,9 @@ class TestBaseChartTest:
(f"{release_name}-cleanup", "CronJob", "airflow-cleanup-pods"),
(f"{release_name}-cleanup-role", "Role", None),
(f"{release_name}-cleanup-rolebinding", "RoleBinding", None),
+ (f"{release_name}-database-cleanup", "CronJob",
"database-cleanup"),
+ (f"{release_name}-database-cleanup-role", "Role", None),
+ (f"{release_name}-database-cleanup-rolebinding", "RoleBinding",
None),
(f"{release_name}-create-user", "Job", "create-user-job"),
(f"{release_name}-fernet-key", "Secret", None),
(f"{release_name}-flower", "Deployment", "flower"),
@@ -532,6 +537,7 @@ class TestBaseChartTest:
"redis": {"enabled": True},
"networkPolicies": {"enabled": True},
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"postgresql": {"enabled": False}, # We won't check the
objects created by the postgres chart
},
diff --git a/helm-tests/tests/helm_tests/security/test_rbac.py
b/helm-tests/tests/helm_tests/security/test_rbac.py
index 819483459b3..6ddd078e726 100644
--- a/helm-tests/tests/helm_tests/security/test_rbac.py
+++ b/helm-tests/tests/helm_tests/security/test_rbac.py
@@ -50,15 +50,18 @@ DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES = [
("Job", "test-rbac-create-user"),
("Job", "test-rbac-run-airflow-migrations"),
("CronJob", "test-rbac-cleanup"),
+ ("CronJob", "test-rbac-database-cleanup"),
]
RBAC_ENABLED_KIND_NAME_TUPLES = [
("Role", "test-rbac-pod-launcher-role"),
("Role", "test-rbac-cleanup-role"),
+ ("Role", "test-rbac-database-cleanup-role"),
("Role", "test-rbac-pod-log-reader-role"),
("RoleBinding", "test-rbac-pod-launcher-rolebinding"),
("RoleBinding", "test-rbac-pod-log-reader-rolebinding"),
("RoleBinding", "test-rbac-cleanup-rolebinding"),
+ ("RoleBinding", "test-rbac-database-cleanup-rolebinding"),
]
SERVICE_ACCOUNT_NAME_TUPLES = [
@@ -66,6 +69,7 @@ SERVICE_ACCOUNT_NAME_TUPLES = [
("ServiceAccount", "test-rbac-scheduler"),
("ServiceAccount", "test-rbac-triggerer"),
("ServiceAccount", "test-rbac-pgbouncer"),
+ ("ServiceAccount", "test-rbac-database-cleanup"),
("ServiceAccount", "test-rbac-flower"),
("ServiceAccount", "test-rbac-statsd"),
("ServiceAccount", "test-rbac-create-user-job"),
@@ -80,6 +84,7 @@ CUSTOM_SERVICE_ACCOUNT_NAMES = (
(CUSTOM_WORKER_NAME := "TestWorker"),
(CUSTOM_TRIGGERER_NAME := "TestTriggerer"),
(CUSTOM_CLEANUP_NAME := "TestCleanup"),
+ (CUSTOM_DATABASE_CLEANUP_NAME := "TestDatabaseCleanup"),
(CUSTOM_FLOWER_NAME := "TestFlower"),
(CUSTOM_PGBOUNCER_NAME := "TestPGBouncer"),
(CUSTOM_STATSD_NAME := "TestStatsd"),
@@ -174,6 +179,12 @@ class TestRBAC:
"create": False,
},
},
+ "databaseCleanup": {
+ "enabled": True,
+ "serviceAccount": {
+ "create": False,
+ },
+ },
"pgbouncer": {
"enabled": True,
"serviceAccount": {
@@ -211,6 +222,7 @@ class TestRBAC:
"executor": "CeleryExecutor,KubernetesExecutor",
"rbac": {"create": False},
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"workers": {"useWorkerDedicatedServiceAccounts":
dedicated_workers_sa},
@@ -252,6 +264,12 @@ class TestRBAC:
"create": False,
},
},
+ "databaseCleanup": {
+ "enabled": True,
+ "serviceAccount": {
+ "create": False,
+ },
+ },
"scheduler": {"serviceAccount": {"create": False}},
"dagProcessor": {"serviceAccount": {"create": False}},
"webserver": {"serviceAccount": {"create": False}},
@@ -289,6 +307,7 @@ class TestRBAC:
"fullnameOverride": "test-rbac",
"executor": "CeleryExecutor,KubernetesExecutor",
"cleanup": {"enabled": True},
+ "databaseCleanup": {"enabled": True},
"flower": {"enabled": True},
"pgbouncer": {"enabled": True},
"workers": {"useWorkerDedicatedServiceAccounts":
dedicated_workers_sa},
@@ -318,6 +337,12 @@ class TestRBAC:
"name": CUSTOM_CLEANUP_NAME,
},
},
+ "databaseCleanup": {
+ "enabled": True,
+ "serviceAccount": {
+ "name": CUSTOM_DATABASE_CLEANUP_NAME,
+ },
+ },
"scheduler": {"serviceAccount": {"name":
CUSTOM_SCHEDULER_NAME}},
"dagProcessor": {"serviceAccount": {"name":
CUSTOM_DAG_PROCESSOR_NAME}},
"apiServer": {"serviceAccount": {"name":
CUSTOM_API_SERVER_NAME}},
@@ -396,6 +421,12 @@ class TestRBAC:
"name": CUSTOM_CLEANUP_NAME,
},
},
+ "databaseCleanup": {
+ "enabled": True,
+ "serviceAccount": {
+ "name": CUSTOM_DATABASE_CLEANUP_NAME,
+ },
+ },
"scheduler": {"serviceAccount": {"name":
CUSTOM_SCHEDULER_NAME}},
"dagProcessor": {"serviceAccount": {"name":
CUSTOM_DAG_PROCESSOR_NAME}},
"apiServer": {"serviceAccount": {"name":
CUSTOM_API_SERVER_NAME}},
