This is an automated email from the ASF dual-hosted git repository. eladkal pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new f31271b497 Add priorityclassname to jobs (#39133) f31271b497 is described below commit f31271b497b773c907e9242e4c8e1758ac1a292d Author: Aakcht <aak...@gmail.com> AuthorDate: Sun Apr 21 12:02:40 2024 +0600 Add priorityclassname to jobs (#39133) --- chart/templates/cleanup/cleanup-cronjob.yaml | 3 +++ chart/templates/jobs/create-user-job.yaml | 3 +++ chart/templates/jobs/migrate-database-job.yaml | 3 +++ chart/values.schema.json | 24 ++++++++++++++++++++++++ chart/values.yaml | 3 +++ helm_tests/airflow_aux/test_airflow_common.py | 13 +++++++++++-- 6 files changed, 47 insertions(+), 2 deletions(-) diff --git a/chart/templates/cleanup/cleanup-cronjob.yaml b/chart/templates/cleanup/cleanup-cronjob.yaml index 073dd70001..5afab9462b 100644 --- a/chart/templates/cleanup/cleanup-cronjob.yaml +++ b/chart/templates/cleanup/cleanup-cronjob.yaml @@ -75,6 +75,9 @@ spec: {{- end }} spec: restartPolicy: Never + {{- if .Values.cleanup.priorityClassName }} + priorityClassName: {{ .Values.cleanup.priorityClassName }} + {{- end }} nodeSelector: {{- toYaml $nodeSelector | nindent 12 }} affinity: {{- toYaml $affinity | nindent 12 }} {{- if .Values.schedulerName }} diff --git a/chart/templates/jobs/create-user-job.yaml b/chart/templates/jobs/create-user-job.yaml index 1bd6fac667..70e31bdb36 100644 --- a/chart/templates/jobs/create-user-job.yaml +++ b/chart/templates/jobs/create-user-job.yaml @@ -75,6 +75,9 @@ spec: spec: securityContext: {{ $securityContext | nindent 8 }} restartPolicy: OnFailure + {{- if .Values.createUserJob.priorityClassName }} + priorityClassName: {{ .Values.createUserJob.priorityClassName }} + {{- end }} nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} affinity: {{- toYaml $affinity | nindent 8 }} {{- if .Values.schedulerName }} diff --git a/chart/templates/jobs/migrate-database-job.yaml b/chart/templates/jobs/migrate-database-job.yaml index 6da44f7372..d5fadf84d9 100644 --- a/chart/templates/jobs/migrate-database-job.yaml +++ b/chart/templates/jobs/migrate-database-job.yaml @@ -75,6 +75,9 @@ spec: spec: securityContext: {{ $securityContext | nindent 8 }} restartPolicy: OnFailure + {{- if .Values.migrateDatabaseJob.priorityClassName }} + priorityClassName: {{ .Values.migrateDatabaseJob.priorityClassName }} + {{- end }} nodeSelector: {{- toYaml $nodeSelector | nindent 8 }} affinity: {{- toYaml $affinity | nindent 8 }} {{- if .Values.schedulerName }} diff --git a/chart/values.schema.json b/chart/values.schema.json index fbad50bfa8..9827737c59 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -3911,6 +3911,14 @@ } ] }, + "priorityClassName": { + "description": "Specify priority for the create user job pod.", + "type": [ + "string", + "null" + ], + "default": null + }, "useHelmHooks": { "description": "Specify if you want to use the default Helm Hook annotations", "type": "boolean", @@ -4199,6 +4207,14 @@ } } }, + "priorityClassName": { + "description": "Specify priority for the migrate database job pod.", + "type": [ + "string", + "null" + ], + "default": null + }, "useHelmHooks": { "description": "Specify if you want to use the default Helm Hook annotations", "type": "boolean", @@ -6886,6 +6902,14 @@ "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" } }, + "priorityClassName": { + "description": "Specify priority for cleanup pods.", + "type": [ + "string", + "null" + ], + "default": null + }, "podAnnotations": { "description": "Annotations to add to cleanup pods.", "type": "object", diff --git a/chart/values.yaml b/chart/values.yaml index 57931c2db8..76070eb086 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1050,6 +1050,7 @@ createUserJob: affinity: {} tolerations: [] topologySpreadConstraints: [] + priorityClassName: ~ # In case you need to disable the helm hooks that create the jobs after install. # Disable this if you are using ArgoCD for example useHelmHooks: true @@ -1149,6 +1150,7 @@ migrateDatabaseJob: affinity: {} tolerations: [] topologySpreadConstraints: [] + priorityClassName: ~ # In case you need to disable the helm hooks that create the jobs after install. # Disable this if you are using ArgoCD for example useHelmHooks: true @@ -2268,6 +2270,7 @@ cleanup: affinity: {} tolerations: [] topologySpreadConstraints: [] + priorityClassName: ~ podAnnotations: {} diff --git a/helm_tests/airflow_aux/test_airflow_common.py b/helm_tests/airflow_aux/test_airflow_common.py index 5257b490d2..13a72c2b3d 100644 --- a/helm_tests/airflow_aux/test_airflow_common.py +++ b/helm_tests/airflow_aux/test_airflow_common.py @@ -405,6 +405,9 @@ class TestAirflowCommon: "dagProcessor": {"priorityClassName": "low-priority-dag-processor"}, "webserver": {"priorityClassName": "low-priority-webserver"}, "workers": {"priorityClassName": "low-priority-worker"}, + "cleanup": {"enabled": True, "priorityClassName": "low-priority-airflow-cleanup-pods"}, + "migrateDatabaseJob": {"priorityClassName": "low-priority-run-airflow-migrations"}, + "createUserJob": {"priorityClassName": "low-priority-create-user-job"}, }, show_only=[ "templates/flower/flower-deployment.yaml", @@ -415,12 +418,18 @@ class TestAirflowCommon: "templates/dag-processor/dag-processor-deployment.yaml", "templates/webserver/webserver-deployment.yaml", "templates/workers/worker-deployment.yaml", + "templates/cleanup/cleanup-cronjob.yaml", + "templates/jobs/migrate-database-job.yaml", + "templates/jobs/create-user-job.yaml", ], ) - assert 7 == len(docs) + assert 10 == len(docs) for doc in docs: component = doc["metadata"]["labels"]["component"] - priority = doc["spec"]["template"]["spec"]["priorityClassName"] + if component == "airflow-cleanup-pods": + priority = doc["spec"]["jobTemplate"]["spec"]["template"]["spec"]["priorityClassName"] + else: + priority = doc["spec"]["template"]["spec"]["priorityClassName"] assert priority == f"low-priority-{component}"