This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new de01a6ba55d feat(docker): add minute-level log retention to clean-logs
script (#61855)
de01a6ba55d is described below
commit de01a6ba55d026432b27b46aa11b5ecfb0caf107
Author: Pablo Ugarte <[email protected]>
AuthorDate: Wed Mar 11 20:38:39 2026 +0100
feat(docker): add minute-level log retention to clean-logs script (#61855)
* feat(docker): add minute-level log retention support to clean-logs.sh
* feat(docker): import coauthored changes
* update inlined scripts in Dockerfile
* comments fixed
---
Dockerfile | 9 +++++---
.../dag-processor/dag-processor-deployment.yaml | 4 ++++
.../templates/scheduler/scheduler-deployment.yaml | 4 ++++
.../templates/triggerer/triggerer-deployment.yaml | 4 ++++
chart/templates/workers/worker-deployment.yaml | 4 ++++
chart/values.schema.json | 7 ++++++-
chart/values.yaml | 24 ++++++++++++++++++++++
scripts/docker/clean-logs.sh | 9 +++++---
8 files changed, 58 insertions(+), 7 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index aa3f1fba405..4115268e41a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1670,7 +1670,8 @@ COPY <<"EOF" /clean-logs.sh
set -euo pipefail
readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}"
-readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
+readonly RETENTION_DAYS="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
+readonly RETENTION_MINUTES="${AIRFLOW__LOG_RETENTION_MINUTES:-0}"
readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}"
readonly MAX_PERCENT="${AIRFLOW__LOG_MAX_SIZE_PERCENT:-0}"
@@ -1695,10 +1696,12 @@ fi
retention_days="${RETENTION}"
while true; do
- echo "Trimming airflow logs to ${retention_days} days."
+ total_retention_minutes=$(( (RETENTION_DAYS * 1440) + RETENTION_MINUTES ))
+ echo "Trimming airflow logs older than ${total_retention_minutes} minutes."
+
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
- -type f -mtime +"${retention_days}" -name '*.log' -print0 | \
+ -type f -mmin +"${total_retention_minutes}" -name '*.log' -print0 | \
xargs -0 rm -f || true
if [[ "$MAX_SIZE_BYTES" -gt 0 && "$retention_days" -ge 0 ]]; then
diff --git a/chart/templates/dag-processor/dag-processor-deployment.yaml
b/chart/templates/dag-processor/dag-processor-deployment.yaml
index c5045e6ecef..88311ad48c3 100644
--- a/chart/templates/dag-processor/dag-processor-deployment.yaml
+++ b/chart/templates/dag-processor/dag-processor-deployment.yaml
@@ -214,6 +214,10 @@ spec:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.dagProcessor.logGroomerSidecar.retentionDays
}}"
{{- end }}
+ {{- if .Values.dagProcessor.logGroomerSidecar.retentionMinutes }}
+ - name: AIRFLOW__LOG_RETENTION_MINUTES
+ value: "{{
.Values.dagProcessor.logGroomerSidecar.retentionMinutes }}"
+ {{- end }}
{{- if .Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{
.Values.dagProcessor.logGroomerSidecar.frequencyMinutes }}"
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml
b/chart/templates/scheduler/scheduler-deployment.yaml
index 3514180c874..2a476740088 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -288,6 +288,10 @@ spec:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.scheduler.logGroomerSidecar.retentionDays }}"
{{- end }}
+ {{- if .Values.scheduler.logGroomerSidecar.retentionMinutes }}
+ - name: AIRFLOW__LOG_RETENTION_MINUTES
+ value: "{{ .Values.scheduler.logGroomerSidecar.retentionMinutes
}}"
+ {{- end }}
{{- if .Values.scheduler.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.scheduler.logGroomerSidecar.frequencyMinutes
}}"
diff --git a/chart/templates/triggerer/triggerer-deployment.yaml
b/chart/templates/triggerer/triggerer-deployment.yaml
index 41a2f0d3d55..e4a394b3ad1 100644
--- a/chart/templates/triggerer/triggerer-deployment.yaml
+++ b/chart/templates/triggerer/triggerer-deployment.yaml
@@ -245,6 +245,10 @@ spec:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.triggerer.logGroomerSidecar.retentionDays }}"
{{- end }}
+ {{- if .Values.triggerer.logGroomerSidecar.retentionMinutes }}
+ - name: AIRFLOW__LOG_RETENTION_MINUTES
+ value: "{{ .Values.triggerer.logGroomerSidecar.retentionMinutes
}}"
+ {{- end }}
{{- if .Values.triggerer.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.triggerer.logGroomerSidecar.frequencyMinutes
}}"
diff --git a/chart/templates/workers/worker-deployment.yaml
b/chart/templates/workers/worker-deployment.yaml
index c810581bf74..96069aa1ff5 100644
--- a/chart/templates/workers/worker-deployment.yaml
+++ b/chart/templates/workers/worker-deployment.yaml
@@ -367,6 +367,10 @@ spec:
- name: AIRFLOW__LOG_RETENTION_DAYS
value: "{{ .Values.workers.logGroomerSidecar.retentionDays }}"
{{- end }}
+ {{- if .Values.workers.logGroomerSidecar.retentionMinutes }}
+ - name: AIRFLOW__LOG_RETENTION_MINUTES
+ value: "{{ .Values.workers.logGroomerSidecar.retentionMinutes }}"
+ {{- end }}
{{- if .Values.workers.logGroomerSidecar.frequencyMinutes }}
- name: AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES
value: "{{ .Values.workers.logGroomerSidecar.frequencyMinutes }}"
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 0891db13bd5..a6f38e0cf41 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -14139,10 +14139,15 @@
]
},
"retentionDays": {
- "description": "Number of days to retain the logs when
running the Airflow log groomer sidecar.",
+ "description": "Number of days to retain the logs when
running the Airflow log groomer sidecar. Total retention time is retentionDays
+ retentionMinutes.",
"type": "integer",
"default": 15
},
+ "retentionMinutes": {
+ "description": "Number of minutes to retain the logs when
running the Airflow log groomer sidecar. Total retention time is retentionDays
+ retentionMinutes.",
+ "type": "integer",
+ "default": 0
+ },
"frequencyMinutes": {
"description": "Number of minutes between attempts to
groom the Airflow logs in log groomer sidecar.",
"type": "integer",
diff --git a/chart/values.yaml b/chart/values.yaml
index 15fa5aeefdb..bd6cb56bedd 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -993,6 +993,12 @@ workers:
# Number of days to retain logs
retentionDays: 15
+ # Number of minutes to retain logs.
+ # This can be used for finer granularity than days.
+ # Total retention is retentionDays + retentionMinutes.
+ retentionMinutes: 0
+
+
# Frequency to attempt to groom logs (in minutes)
frequencyMinutes: 15
@@ -1472,6 +1478,12 @@ scheduler:
args: ["bash", "/clean-logs"]
# Number of days to retain logs
retentionDays: 15
+
+ # Number of minutes to retain logs.
+ # This can be used for finer granularity than days.
+ # Total retention is retentionDays + retentionMinutes.
+ retentionMinutes: 0
+
# frequency to attempt to groom logs, in minutes
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
@@ -2317,6 +2329,12 @@ triggerer:
args: ["bash", "/clean-logs"]
# Number of days to retain logs
retentionDays: 15
+
+ # Number of minutes to retain logs.
+ # This can be used for finer granularity than days.
+ # Total retention is retentionDays + retentionMinutes.
+ retentionMinutes: 0
+
# frequency to attempt to groom logs, in minutes
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
@@ -2548,6 +2566,12 @@ dagProcessor:
args: ["bash", "/clean-logs"]
# Number of days to retain logs
retentionDays: 15
+
+ # Number of minutes to retain logs.
+ # This can be used for finer granularity than days.
+ # Total retention is retentionDays + retentionMinutes.
+ retentionMinutes: 0
+
# frequency to attempt to groom logs, in minutes
frequencyMinutes: 15
# Max size of logs in bytes. 0 = disabled
diff --git a/scripts/docker/clean-logs.sh b/scripts/docker/clean-logs.sh
index 15370362695..7253424f461 100755
--- a/scripts/docker/clean-logs.sh
+++ b/scripts/docker/clean-logs.sh
@@ -20,7 +20,8 @@
set -euo pipefail
readonly DIRECTORY="${AIRFLOW_HOME:-/usr/local/airflow}"
-readonly RETENTION="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
+readonly RETENTION_DAYS="${AIRFLOW__LOG_RETENTION_DAYS:-15}"
+readonly RETENTION_MINUTES="${AIRFLOW__LOG_RETENTION_MINUTES:-0}"
readonly FREQUENCY="${AIRFLOW__LOG_CLEANUP_FREQUENCY_MINUTES:-15}"
readonly MAX_PERCENT="${AIRFLOW__LOG_MAX_SIZE_PERCENT:-0}"
@@ -45,10 +46,12 @@ fi
retention_days="${RETENTION}"
while true; do
- echo "Trimming airflow logs to ${retention_days} days."
+ total_retention_minutes=$(( (RETENTION_DAYS * 1440) + RETENTION_MINUTES ))
+ echo "Trimming airflow logs older than ${total_retention_minutes} minutes."
+
find "${DIRECTORY}"/logs \
-type d -name 'lost+found' -prune -o \
- -type f -mtime +"${retention_days}" -name '*.log' -print0 | \
+ -type f -mmin +"${total_retention_minutes}" -name '*.log' -print0 | \
xargs -0 rm -f || true
if [[ "$MAX_SIZE_BYTES" -gt 0 && "$retention_days" -ge 0 ]]; then
