This is an automated email from the ASF dual-hosted git repository.
dimberman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 7b3a00d Allow multiple schedulers in helm chart (#11330)
7b3a00d is described below
commit 7b3a00d4f97ada5e09bfc33c0e48b3b1833bfe56
Author: Daniel Imberman <[email protected]>
AuthorDate: Mon Oct 12 13:48:59 2020 -0700
Allow multiple schedulers in helm chart (#11330)
* Allow multiple schedulers in helm chart
* schema
* add docs
* add to readme
Co-authored-by: Daniel Imberman <[email protected]>
---
chart/README.md | 1 +
chart/templates/create-user-job.yaml | 8 ++++++++
chart/templates/scheduler/scheduler-deployment.yaml | 8 ++++----
chart/values.schema.json | 4 ++++
chart/values.yaml | 3 +++
5 files changed, 20 insertions(+), 4 deletions(-)
diff --git a/chart/README.md b/chart/README.md
index 4e25362..660a7ac 100644
--- a/chart/README.md
+++ b/chart/README.md
@@ -187,6 +187,7 @@ The following tables lists the configurable parameters of
the Airflow chart and
| `workers.safeToEvict` | Allow Kubernetes to
evict worker pods if needed (node downscaling)
| `true` |
| `scheduler.podDisruptionBudget.enabled` | Enable PDB on
Airflow scheduler
| `false` |
| `scheduler.podDisruptionBudget.config.maxUnavailable` | MaxUnavailable pods
for scheduler
| `1` |
+| `scheduler.replicas` | # of parallel
schedulers (Airflow 2.0 using Mysql 8+ or Postgres only)
| `1` |
| `scheduler.resources.limits.cpu` | CPU Limit of
scheduler
| `~` |
| `scheduler.resources.limits.memory` | Memory Limit of
scheduler
| `~` |
| `scheduler.resources.requests.cpu` | CPU Request of
scheduler
| `~` |
diff --git a/chart/templates/create-user-job.yaml
b/chart/templates/create-user-job.yaml
index b596359..d9e41b6 100644
--- a/chart/templates/create-user-job.yaml
+++ b/chart/templates/create-user-job.yaml
@@ -57,6 +57,14 @@ spec:
- name: {{ template "registry_secret" . }}
{{- end }}
containers:
+ - name: run-airflow-migrations
+ image: {{ template "airflow_image" . }}
+ imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
+ # Support running against 1.10.x and 2.0.0dev/master
+ args: ["bash", "-c", "airflow upgradedb || airflow db upgrade"]
+ env:
+ {{- include "custom_airflow_environment" . | indent 10 }}
+ {{- include "standard_airflow_environment" . | indent 10 }}
- name: create-user
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml
b/chart/templates/scheduler/scheduler-deployment.yaml
index c47c6f9..2db8eb2 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -46,7 +46,7 @@ spec:
{{- if $stateful }}
serviceName: {{ .Release.Name }}-scheduler
{{- end }}
- replicas: 1
+ replicas: {{ .Values.scheduler.replicas }}
selector:
matchLabels:
tier: airflow
@@ -90,11 +90,11 @@ spec:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
- - name: run-airflow-migrations
+ - name: wait-for-airflow-migrations
image: {{ template "airflow_image" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
- # Support running against 1.10.x and 2.0.0dev/master
- args: ["bash", "-c", "airflow upgradedb || airflow db upgrade"]
+ args:
+ {{- include "wait-for-migrations-command" . | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 9370c80..b2f7288 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -486,6 +486,10 @@
"description": "Airflow scheduler settings.",
"type": "object",
"properties": {
+ "replicas": {
+ "description": "Airflow 2.0 allows users to run multiple
schedulers. This feature is only recommended for Mysql 8+ and postgres",
+ "type": "integer"
+ },
"podDisruptionBudget": {
"description": "Scheduler pod disruption budget.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index e58ff27..8a16a8f 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -292,6 +292,9 @@ workers:
# Airflow scheduler settings
scheduler:
+ # Airflow 2.0 allows users to run multiple schedulers,
+ # However this feature is only recommended for MySQL 8+ and Postgres
+ replicas: 1
# Scheduler pod disruption budget
podDisruptionBudget:
enabled: false
