This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 681d2ba012 Added ability to enable/disable scheduler and webserver
(#36991)
681d2ba012 is described below
commit 681d2ba0128e3e6cbb044970ba97af8e2fb000f9
Author: Kj Justin <[email protected]>
AuthorDate: Mon Feb 5 08:09:10 2024 -0500
Added ability to enable/disable scheduler and webserver (#36991)
---------
Co-authored-by: Elad Kalif <[email protected]>
Co-authored-by: Jed Cunningham
<[email protected]>
---
.../templates/scheduler/scheduler-deployment.yaml | 3 ++-
.../scheduler/scheduler-networkpolicy.yaml | 2 ++
.../scheduler/scheduler-poddisruptionbudget.yaml | 2 ++
chart/templates/scheduler/scheduler-service.yaml | 2 ++
.../scheduler/scheduler-serviceaccount.yaml | 2 +-
.../templates/webserver/webserver-deployment.yaml | 2 ++
chart/templates/webserver/webserver-ingress.yaml | 2 ++
.../webserver/webserver-networkpolicy.yaml | 2 ++
.../webserver/webserver-poddisruptionbudget.yaml | 2 ++
chart/templates/webserver/webserver-service.yaml | 12 +++++++----
.../webserver/webserver-serviceaccount.yaml | 2 +-
chart/values.schema.json | 10 +++++++++
chart/values.yaml | 2 ++
docs/helm-chart/production-guide.rst | 24 ++++++++++++++++++++++
helm_tests/airflow_core/test_scheduler.py | 17 +++++++++++++++
helm_tests/webserver/test_webserver.py | 13 ++++++++++++
16 files changed, 92 insertions(+), 7 deletions(-)
diff --git a/chart/templates/scheduler/scheduler-deployment.yaml
b/chart/templates/scheduler/scheduler-deployment.yaml
index ccee1609fb..7ec2c46bed 100644
--- a/chart/templates/scheduler/scheduler-deployment.yaml
+++ b/chart/templates/scheduler/scheduler-deployment.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Scheduler Deployment/StatefulSet
#################################
-
+{{- if .Values.scheduler.enabled }}
# Are we using a local executor?
{{- $local := contains "Local" .Values.executor }}
# Is persistence enabled on the _workers_?
@@ -333,3 +333,4 @@ spec:
requests:
storage: {{ .Values.workers.persistence.size }}
{{- end }}
+ {{- end }}
diff --git a/chart/templates/scheduler/scheduler-networkpolicy.yaml
b/chart/templates/scheduler/scheduler-networkpolicy.yaml
index bbd985371e..f1c04ad4bc 100644
--- a/chart/templates/scheduler/scheduler-networkpolicy.yaml
+++ b/chart/templates/scheduler/scheduler-networkpolicy.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Scheduler NetworkPolicy
#################################
+{{- if .Values.scheduler.enabled }}
{{- if .Values.networkPolicies.enabled }}
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
@@ -55,3 +56,4 @@ spec:
port: {{ .Values.ports.workerLogs }}
{{- end }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml
b/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml
index 3c6d7983b1..c413b099f3 100644
--- a/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml
+++ b/chart/templates/scheduler/scheduler-poddisruptionbudget.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Scheduler PodDisruptionBudget
#################################
+{{- if .Values.scheduler.enabled }}
{{- if .Values.scheduler.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
@@ -42,3 +43,4 @@ spec:
release: {{ .Release.Name }}
{{- toYaml .Values.scheduler.podDisruptionBudget.config | nindent 2 }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/scheduler/scheduler-service.yaml
b/chart/templates/scheduler/scheduler-service.yaml
index b2bf2a87db..aa28563deb 100644
--- a/chart/templates/scheduler/scheduler-service.yaml
+++ b/chart/templates/scheduler/scheduler-service.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Scheduler Service
#################################
+{{- if .Values.scheduler.enabled }}
{{- if or (eq .Values.executor "LocalExecutor") (eq .Values.executor
"LocalKubernetesExecutor") }}
apiVersion: v1
kind: Service
@@ -46,3 +47,4 @@ spec:
port: {{ .Values.ports.workerLogs }}
targetPort: {{ .Values.ports.workerLogs }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/scheduler/scheduler-serviceaccount.yaml
b/chart/templates/scheduler/scheduler-serviceaccount.yaml
index 1a359ab182..310f168496 100644
--- a/chart/templates/scheduler/scheduler-serviceaccount.yaml
+++ b/chart/templates/scheduler/scheduler-serviceaccount.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Scheduler ServiceAccount
#################################
-{{- if .Values.scheduler.serviceAccount.create }}
+{{- if and .Values.scheduler.enabled .Values.scheduler.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{
.Values.scheduler.serviceAccount.automountServiceAccountToken }}
diff --git a/chart/templates/webserver/webserver-deployment.yaml
b/chart/templates/webserver/webserver-deployment.yaml
index 1fa06b0724..bb6c9d4b53 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Webserver Deployment
#################################
+{{- if .Values.webserver.enabled }}
{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.webserver.affinity .Values.affinity }}
{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }}
@@ -291,3 +292,4 @@ spec:
{{- if .Values.webserver.extraVolumes }}
{{- tpl (toYaml .Values.webserver.extraVolumes) . | nindent 8 }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/webserver/webserver-ingress.yaml
b/chart/templates/webserver/webserver-ingress.yaml
index 451a57cbc0..6851be45ec 100644
--- a/chart/templates/webserver/webserver-ingress.yaml
+++ b/chart/templates/webserver/webserver-ingress.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Webserver Ingress
#################################
+{{- if .Values.webserver.enabled }}
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
{{- $fullname := (include "airflow.fullname" .) }}
apiVersion: networking.k8s.io/v1
@@ -109,3 +110,4 @@ spec:
ingressClassName: {{ .Values.ingress.web.ingressClassName }}
{{- end }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/webserver/webserver-networkpolicy.yaml
b/chart/templates/webserver/webserver-networkpolicy.yaml
index d0be9e3283..8945d9c2b6 100644
--- a/chart/templates/webserver/webserver-networkpolicy.yaml
+++ b/chart/templates/webserver/webserver-networkpolicy.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Webserver NetworkPolicy
#################################
+{{- if .Values.webserver.enabled }}
{{- if .Values.networkPolicies.enabled }}
{{- $from := or .Values.webserver.networkPolicy.ingress.from
.Values.webserver.extraNetworkPolicies }}
apiVersion: networking.k8s.io/v1
@@ -55,3 +56,4 @@ spec:
{{- end }}
{{- end }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/webserver/webserver-poddisruptionbudget.yaml
b/chart/templates/webserver/webserver-poddisruptionbudget.yaml
index 840219a56e..f5670ca6e7 100644
--- a/chart/templates/webserver/webserver-poddisruptionbudget.yaml
+++ b/chart/templates/webserver/webserver-poddisruptionbudget.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Webserver PodDisruptionBudget
#################################
+{{- if .Values.webserver.enabled }}
{{- if .Values.webserver.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
@@ -42,3 +43,4 @@ spec:
release: {{ .Release.Name }}
{{- toYaml .Values.webserver.podDisruptionBudget.config | nindent 2 }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/webserver/webserver-service.yaml
b/chart/templates/webserver/webserver-service.yaml
index a3c49d32a5..c2bdb6b0b5 100644
--- a/chart/templates/webserver/webserver-service.yaml
+++ b/chart/templates/webserver/webserver-service.yaml
@@ -20,6 +20,7 @@
################################
## Airflow Webserver Service
#################################
+{{- if .Values.webserver.enabled }}
apiVersion: v1
kind: Service
metadata:
@@ -43,10 +44,12 @@ spec:
component: webserver
release: {{ .Release.Name }}
ports:
- {{- range .Values.webserver.service.ports }}
- - {{ range $key, $val := . }}
- {{- $key }}: {{ tpl (toString $val) $ }}
- {{ end }}
+ ports:
+ {{ range .Values.webserver.service.ports }}
+ -
+ {{- range $key, $val := . }}
+ {{ $key }}: {{ tpl (toString $val) $ }}
+ {{- end }}
{{- end }}
{{- if .Values.webserver.service.loadBalancerIP }}
loadBalancerIP: {{ .Values.webserver.service.loadBalancerIP }}
@@ -54,3 +57,4 @@ spec:
{{- if .Values.webserver.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges: {{- toYaml
.Values.webserver.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/webserver/webserver-serviceaccount.yaml
b/chart/templates/webserver/webserver-serviceaccount.yaml
index 853e376a5d..a971da6e32 100644
--- a/chart/templates/webserver/webserver-serviceaccount.yaml
+++ b/chart/templates/webserver/webserver-serviceaccount.yaml
@@ -20,7 +20,7 @@
######################################
## Airflow Webserver ServiceAccount
######################################
-{{- if .Values.webserver.serviceAccount.create }}
+{{- if and .Values.webserver.enabled .Values.webserver.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
automountServiceAccountToken: {{
.Values.webserver.serviceAccount.automountServiceAccountToken }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index f6613adb96..3f9e3fc868 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -2193,6 +2193,11 @@
"x-docsSection": "Scheduler",
"additionalProperties": false,
"properties": {
+ "enabled": {
+ "description": "Enable scheduler",
+ "type": "boolean",
+ "default": true
+ },
"hostAliases": {
"description": "HostAliases for the scheduler pod.",
"items": {
@@ -4212,6 +4217,11 @@
"x-docsSection": "Webserver",
"additionalProperties": false,
"properties": {
+ "enabled": {
+ "description": "Enable webserver",
+ "type": "boolean",
+ "default": true
+ },
"configMapAnnotations": {
"description": "Extra annotations to apply to the
webserver configmap.",
"type": "object",
diff --git a/chart/values.yaml b/chart/values.yaml
index fddcf67c87..9a7b7352c1 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -790,6 +790,7 @@ workers:
# Airflow scheduler settings
scheduler:
+ enabled: true
# hostAliases for the scheduler pod
hostAliases: []
# - ip: "127.0.0.1"
@@ -1150,6 +1151,7 @@ migrateDatabaseJob:
# Airflow webserver settings
webserver:
+ enabled: true
# Add custom annotations to the webserver configmap
configMapAnnotations: {}
# hostAliases for the webserver pod
diff --git a/docs/helm-chart/production-guide.rst
b/docs/helm-chart/production-guide.rst
index 6451205611..4261c9f9ad 100644
--- a/docs/helm-chart/production-guide.rst
+++ b/docs/helm-chart/production-guide.rst
@@ -208,11 +208,35 @@ They match, right? Good. Now, add the public key to your
values. It'll look some
github.com ssh-rsa AAAA...1/wsjk=
+External Scheduler
+^^^^^^^^^^^^^^^^^^
+
+To use an external Scheduler instance:
+
+.. code-block:: yaml
+
+ scheduler:
+ enabled: false
+
+Ensure that your external webserver/scheduler is connected to the same redis
host. This will ensure the scheduler is aware of the workers deployed in the
helm-chart.
+
Accessing the Airflow UI
------------------------
How you access the Airflow UI will depend on your environment; however, the
chart does support various options:
+External Webserver
+^^^^^^^^^^^^^^^^^^
+
+To use an external Webserver:
+
+.. code-block:: yaml
+
+ webserver:
+ enabled: false
+
+Ensure that your external webserver/scheduler is connected to the same redis
host. This will ensure the scheduler is aware of the workers deployed in the
helm-chart.
+
Ingress
^^^^^^^
diff --git a/helm_tests/airflow_core/test_scheduler.py
b/helm_tests/airflow_core/test_scheduler.py
index addaaf896e..42980b8865 100644
--- a/helm_tests/airflow_core/test_scheduler.py
+++ b/helm_tests/airflow_core/test_scheduler.py
@@ -901,3 +901,20 @@ class TestSchedulerServiceAccount:
show_only=["templates/scheduler/scheduler-serviceaccount.yaml"],
)
assert jmespath.search("automountServiceAccountToken", docs[0]) is
False
+
+
+class TestSchedulerCreation:
+ """Tests scheduler deployment creation."""
+
+ def test_can_be_disabled(self):
+ """
+ Scheduler should be able to be disabled if the users desires.
+
+ For example, user may be disabled when using scheduler and having it
deployed on another host.
+ """
+ docs = render_chart(
+ values={"scheduler": {"enabled": False}},
+ show_only=["templates/scheduler/scheduler-deployment.yaml"],
+ )
+
+ assert 0 == len(docs)
diff --git a/helm_tests/webserver/test_webserver.py
b/helm_tests/webserver/test_webserver.py
index 7fd7e60257..42a89bcc5c 100644
--- a/helm_tests/webserver/test_webserver.py
+++ b/helm_tests/webserver/test_webserver.py
@@ -25,6 +25,19 @@ from tests.charts.helm_template_generator import render_chart
class TestWebserverDeployment:
"""Tests webserver deployment."""
+ def test_can_be_disabled(self):
+ """
+ Webserver should be able to be disabled if the users desires.
+
+ For example, user may be disabled when using webserver and having it
deployed on another host.
+ """
+ docs = render_chart(
+ values={"webserver": {"enabled": False}},
+ show_only=["templates/webserver/webserver-deployment.yaml"],
+ )
+
+ assert 0 == len(docs)
+
def
test_should_add_host_header_to_liveness_and_readiness_and_startup_probes(self):
docs = render_chart(
values={
