This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6c1cefe0597 Add database cleanup options and remove deprecated
securityContext field (#58663)
6c1cefe0597 is described below
commit 6c1cefe0597c62599bf609321a1e0b62029b61d5
Author: Jorge Rocamora <[email protected]>
AuthorDate: Tue Nov 25 22:18:48 2025 +0100
Add database cleanup options and remove deprecated securityContext field
(#58663)
Co-authored-by: Your Name <[email protected]>
---
chart/values.schema.json | 42 ++++++++++++-------
chart/values.yaml | 19 ++++++---
...se_cleanup_pods.py => test_database_cleanup.py} | 49 ++++++++++++++++++++--
3 files changed, 88 insertions(+), 22 deletions(-)
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 852fb9a1bd6..149b362758b 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -8978,6 +8978,33 @@
"type": "integer",
"default": 90
},
+ "skipArchive": {
+ "description": "Don't preserve purged records in an
archive table.",
+ "type": "boolean",
+ "default": false
+ },
+ "tables": {
+ "description": "Table names to perform maintenance on.
Supported values in:
https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html#clean";,
+ "type": "array",
+ "uniqueItems": true,
+ "items": {
+ "type": "string"
+ },
+ "default": []
+ },
+ "batchSize": {
+ "description": "Maximum number of rows to delete or
archive in a single transaction.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null
+ },
+ "verbose": {
+ "description": "Make logging output more verbose.",
+ "type": "boolean",
+ "default": true
+ },
"command": {
"description": "Command to use when running the database
cleanup cronjob (templated).",
"type": [
@@ -9002,7 +9029,7 @@
},
"default": [
"-c",
- "CLEAN_TS=$(date -d \"-{{
.Values.databaseCleanup.retentionDays }} days\" +\"%Y-%m-%dT%H:%M:%S\"); echo
\"Cleaning up metadata DB entries older than ${CLEAN_TS}\"; exec airflow db
clean --clean-before-timestamp \"${CLEAN_TS}\" --yes --verbose"
+ "CLEAN_TS=$(date -d \"-{{
.Values.databaseCleanup.retentionDays }} days\" +\"%Y-%m-%dT%H:%M:%S\"); echo
\"Cleaning up metadata DB entries older than ${CLEAN_TS}\"; exec airflow db
clean --clean-before-timestamp \"${CLEAN_TS}\" --yes {{- if
.Values.databaseCleanup.skipArchive }} --skip-archive{{ end }} {{- if
.Values.databaseCleanup.verbose }} --verbose{{ end }} {{- with
.Values.databaseCleanup.batchSize }} --batch-size {{ . }}{{ end }} {{- with
.Values.databaseCle [...]
]
},
"jobAnnotations": {
@@ -9120,19 +9147,6 @@
}
}
},
- "securityContext": {
- "description": "Security context for the database cleanup
job pod (deprecated, use `securityContexts` instead). If not set, the values
from `securityContext` will be used.",
- "type": "object",
- "$ref":
"#/definitions/io.k8s.api.core.v1.PodSecurityContext",
- "default": {},
- "examples": [
- {
- "runAsUser": 50000,
- "runAsGroup": 0,
- "fsGroup": 0
- }
- ]
- },
"containerLifecycleHooks": {
"description": "Container Lifecycle Hooks definition for
the database cleanup. If not set, the values from global
`containerLifecycleHooks` will be used.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index 8d5e2361e86..b4d8d8f1561 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -2889,10 +2889,23 @@ databaseCleanup:
- >-
CLEAN_TS=$(date -d "-{{ .Values.databaseCleanup.retentionDays }} days"
+"%Y-%m-%dT%H:%M:%S");
echo "Cleaning up metadata DB entries older than ${CLEAN_TS}";
- exec airflow db clean --clean-before-timestamp "${CLEAN_TS}" --yes
--verbose
+ exec airflow db clean --clean-before-timestamp "${CLEAN_TS}" --yes
+ {{- if .Values.databaseCleanup.skipArchive }} --skip-archive{{ end }}
+ {{- if .Values.databaseCleanup.verbose }} --verbose{{ end }}
+ {{- with .Values.databaseCleanup.batchSize }} --batch-size {{ . }}{{ end
}}
+ {{- with .Values.databaseCleanup.tables }} --tables {{ . | join "," }}{{
end }}
# Number of days to retain entries in the metadata database.
retentionDays: 90
+ # Don't preserve purged records in an archive table
+ skipArchive: false
+ # Table names to perform maintenance on. Supported values in:
+ #
https://airflow.apache.org/docs/apache-airflow/stable/cli-and-env-variables-ref.html#clean
+ tables: []
+ # Maximum number of rows to delete or archive in a single transaction
+ batchSize: ~
+ # Make logging output more verbose
+ verbose: true
# jobAnnotations are annotations on the database cleanup CronJob
jobAnnotations: {}
@@ -2931,10 +2944,6 @@ databaseCleanup:
# Annotations to add to database cleanup cronjob kubernetes service
account.
annotations: {}
- # When not set, the values defined in the global securityContext will be used
- securityContext: {}
- # runAsUser: 50000
- # runAsGroup: 0
env: []
# Detailed default security context for database cleanup for container level
diff --git
a/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup_pods.py
b/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py
similarity index 90%
rename from
helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup_pods.py
rename to helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py
index 60ef5f4f4f9..2d449259461 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup_pods.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py
@@ -22,7 +22,7 @@ from chart_utils.helm_template_generator import render_chart
class TestDatabaseCleanupDeployment:
- """Tests database cleanup pods deployments."""
+ """Tests database cleanup deployments."""
def test_should_have_a_schedule_with_defaults(self):
doc = render_chart(
@@ -69,8 +69,8 @@ class TestDatabaseCleanupDeployment:
assert doc["spec"]["schedule"] == schedule_result
-class TestDatabaseCleanupPods:
- """Tests cleanup of pods."""
+class TestDatabaseCleanup:
+ """Tests database cleanup."""
def test_should_create_cronjob_for_enabled_cleanup(self):
docs = render_chart(
@@ -213,6 +213,49 @@ class TestDatabaseCleanupPods:
'CLEAN_TS=$(date -d "-90 days" +"%Y-%m-%dT%H:%M:%S"); echo
"Cleaning up metadata DB entries older than ${CLEAN_TS}"; exec airflow db clean
--clean-before-timestamp "${CLEAN_TS}" --yes --verbose',
]
+ @pytest.mark.parametrize(
+ ("retention", "skip_archive", "verbose", "batch_size", "tables",
"command_args"),
+ [
+ (90, False, False, None, None, ""),
+ (91, True, False, None, None, " --skip-archive"),
+ (92, False, True, None, None, " --verbose"),
+ (93, False, False, 200, None, " --batch-size 200"),
+ (94, False, False, None, ["xcom"], " --tables xcom"),
+ (
+ 95,
+ True,
+ True,
+ 500,
+ ["task_instance", "log"],
+ " --skip-archive --verbose --batch-size 500 --tables
task_instance,log",
+ ),
+ ],
+ )
+ def test_cleanup_command_options(
+ self, retention, skip_archive, verbose, batch_size, tables,
command_args
+ ):
+ docs = render_chart(
+ values={
+ "databaseCleanup": {
+ "enabled": True,
+ "retentionDays": retention,
+ "skipArchive": skip_archive,
+ "verbose": verbose,
+ "batchSize": batch_size,
+ "tables": tables,
+ }
+ },
+
show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"],
+ )
+
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].command",
docs[0]) == [
+ "bash"
+ ]
+ assert
jmespath.search("spec.jobTemplate.spec.template.spec.containers[0].args",
docs[0]) == [
+ "-c",
+ f'CLEAN_TS=$(date -d "-{retention} days" +"%Y-%m-%dT%H:%M:%S");
echo "Cleaning up metadata DB entries older than ${{CLEAN_TS}}"; exec airflow
db clean --clean-before-timestamp "${{CLEAN_TS}}" --yes{command_args}',
+ ]
+
def test_should_add_extraEnvs(self):
docs = render_chart(
values={
