This is an automated email from the ASF dual-hosted git repository.
eladkal pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f31271b497 Add priorityclassname to jobs (#39133)
f31271b497 is described below
commit f31271b497b773c907e9242e4c8e1758ac1a292d
Author: Aakcht <[email protected]>
AuthorDate: Sun Apr 21 12:02:40 2024 +0600
Add priorityclassname to jobs (#39133)
---
chart/templates/cleanup/cleanup-cronjob.yaml | 3 +++
chart/templates/jobs/create-user-job.yaml | 3 +++
chart/templates/jobs/migrate-database-job.yaml | 3 +++
chart/values.schema.json | 24 ++++++++++++++++++++++++
chart/values.yaml | 3 +++
helm_tests/airflow_aux/test_airflow_common.py | 13 +++++++++++--
6 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/chart/templates/cleanup/cleanup-cronjob.yaml
b/chart/templates/cleanup/cleanup-cronjob.yaml
index 073dd70001..5afab9462b 100644
--- a/chart/templates/cleanup/cleanup-cronjob.yaml
+++ b/chart/templates/cleanup/cleanup-cronjob.yaml
@@ -75,6 +75,9 @@ spec:
{{- end }}
spec:
restartPolicy: Never
+ {{- if .Values.cleanup.priorityClassName }}
+ priorityClassName: {{ .Values.cleanup.priorityClassName }}
+ {{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 12 }}
affinity: {{- toYaml $affinity | nindent 12 }}
{{- if .Values.schedulerName }}
diff --git a/chart/templates/jobs/create-user-job.yaml
b/chart/templates/jobs/create-user-job.yaml
index 1bd6fac667..70e31bdb36 100644
--- a/chart/templates/jobs/create-user-job.yaml
+++ b/chart/templates/jobs/create-user-job.yaml
@@ -75,6 +75,9 @@ spec:
spec:
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
+ {{- if .Values.createUserJob.priorityClassName }}
+ priorityClassName: {{ .Values.createUserJob.priorityClassName }}
+ {{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
diff --git a/chart/templates/jobs/migrate-database-job.yaml
b/chart/templates/jobs/migrate-database-job.yaml
index 6da44f7372..d5fadf84d9 100644
--- a/chart/templates/jobs/migrate-database-job.yaml
+++ b/chart/templates/jobs/migrate-database-job.yaml
@@ -75,6 +75,9 @@ spec:
spec:
securityContext: {{ $securityContext | nindent 8 }}
restartPolicy: OnFailure
+ {{- if .Values.migrateDatabaseJob.priorityClassName }}
+ priorityClassName: {{ .Values.migrateDatabaseJob.priorityClassName }}
+ {{- end }}
nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
affinity: {{- toYaml $affinity | nindent 8 }}
{{- if .Values.schedulerName }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index fbad50bfa8..9827737c59 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -3911,6 +3911,14 @@
}
]
},
+ "priorityClassName": {
+ "description": "Specify priority for the create user job
pod.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": null
+ },
"useHelmHooks": {
"description": "Specify if you want to use the default
Helm Hook annotations",
"type": "boolean",
@@ -4199,6 +4207,14 @@
}
}
},
+ "priorityClassName": {
+ "description": "Specify priority for the migrate database
job pod.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": null
+ },
"useHelmHooks": {
"description": "Specify if you want to use the default
Helm Hook annotations",
"type": "boolean",
@@ -6886,6 +6902,14 @@
"$ref":
"#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint"
}
},
+ "priorityClassName": {
+ "description": "Specify priority for cleanup pods.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": null
+ },
"podAnnotations": {
"description": "Annotations to add to cleanup pods.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index 57931c2db8..76070eb086 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -1050,6 +1050,7 @@ createUserJob:
affinity: {}
tolerations: []
topologySpreadConstraints: []
+ priorityClassName: ~
# In case you need to disable the helm hooks that create the jobs after
install.
# Disable this if you are using ArgoCD for example
useHelmHooks: true
@@ -1149,6 +1150,7 @@ migrateDatabaseJob:
affinity: {}
tolerations: []
topologySpreadConstraints: []
+ priorityClassName: ~
# In case you need to disable the helm hooks that create the jobs after
install.
# Disable this if you are using ArgoCD for example
useHelmHooks: true
@@ -2268,6 +2270,7 @@ cleanup:
affinity: {}
tolerations: []
topologySpreadConstraints: []
+ priorityClassName: ~
podAnnotations: {}
diff --git a/helm_tests/airflow_aux/test_airflow_common.py
b/helm_tests/airflow_aux/test_airflow_common.py
index 5257b490d2..13a72c2b3d 100644
--- a/helm_tests/airflow_aux/test_airflow_common.py
+++ b/helm_tests/airflow_aux/test_airflow_common.py
@@ -405,6 +405,9 @@ class TestAirflowCommon:
"dagProcessor": {"priorityClassName":
"low-priority-dag-processor"},
"webserver": {"priorityClassName": "low-priority-webserver"},
"workers": {"priorityClassName": "low-priority-worker"},
+ "cleanup": {"enabled": True, "priorityClassName":
"low-priority-airflow-cleanup-pods"},
+ "migrateDatabaseJob": {"priorityClassName":
"low-priority-run-airflow-migrations"},
+ "createUserJob": {"priorityClassName":
"low-priority-create-user-job"},
},
show_only=[
"templates/flower/flower-deployment.yaml",
@@ -415,12 +418,18 @@ class TestAirflowCommon:
"templates/dag-processor/dag-processor-deployment.yaml",
"templates/webserver/webserver-deployment.yaml",
"templates/workers/worker-deployment.yaml",
+ "templates/cleanup/cleanup-cronjob.yaml",
+ "templates/jobs/migrate-database-job.yaml",
+ "templates/jobs/create-user-job.yaml",
],
)
- assert 7 == len(docs)
+ assert 10 == len(docs)
for doc in docs:
component = doc["metadata"]["labels"]["component"]
- priority = doc["spec"]["template"]["spec"]["priorityClassName"]
+ if component == "airflow-cleanup-pods":
+ priority =
doc["spec"]["jobTemplate"]["spec"]["template"]["spec"]["priorityClassName"]
+ else:
+ priority = doc["spec"]["template"]["spec"]["priorityClassName"]
assert priority == f"low-priority-{component}"
