This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new caddbf3 Chart: Avoid `git-sync` sidecar on Websever when
Airflow>=2.0.0 (#15814)
caddbf3 is described below
commit caddbf3aa04096033502d7da7eafaf830737a9b9
Author: Kaxil Naik <[email protected]>
AuthorDate: Thu May 13 12:46:23 2021 +0100
Chart: Avoid `git-sync` sidecar on Websever when Airflow>=2.0.0 (#15814)
For `apache-airflow>=2.0.0`, DAG Serialization is enabled by default
and we don't need to have a sidecar on Websserver.
Previously this was done using `gitSync.excludeWebserver`. However
with https://github.com/apache/airflow/pull/15627 - we now have
`airflowVerson` so we can just do a comparison of the version.
---
.../templates/webserver/webserver-deployment.yaml | 6 +--
chart/tests/test_git_sync_webserver.py | 49 ++++++++++++++++++----
chart/values.schema.json | 4 --
chart/values.yaml | 6 ---
docs/helm-chart/manage-dags-files.rst | 14 +++----
5 files changed, 50 insertions(+), 29 deletions(-)
diff --git a/chart/templates/webserver/webserver-deployment.yaml
b/chart/templates/webserver/webserver-deployment.yaml
index 852e514..e9f3e23 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -108,7 +108,7 @@ spec:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
containers:
-{{- if and (.Values.dags.gitSync.enabled) (not
.Values.dags.persistence.enabled) (not .Values.dags.gitSync.excludeWebserver) }}
+{{- if and (.Values.dags.gitSync.enabled) (not
.Values.dags.persistence.enabled) (semverCompare "<2.0.0"
.Values.airflowVersion) }}
{{- include "git_sync_container" . | indent 8 }}
{{- end }}
- name: webserver
@@ -134,7 +134,7 @@ spec:
subPath: airflow_local_settings.py
readOnly: true
{{- end }}
-{{- if or (and .Values.dags.gitSync.enabled (not
.Values.dags.gitSync.excludeWebserver)) .Values.dags.persistence.enabled }}
+{{- if or (and .Values.dags.gitSync.enabled (semverCompare "<2.0.0"
.Values.airflowVersion)) .Values.dags.persistence.enabled }}
- name: dags
mountPath: {{ template "airflow_dags_mount_path" . }}
{{- end }}
@@ -195,7 +195,7 @@ spec:
- name: dags
persistentVolumeClaim:
claimName: {{ template "airflow_dags_volume_claim" . }}
- {{- else if and (.Values.dags.gitSync.enabled) (not
.Values.dags.gitSync.excludeWebserver) }}
+ {{- else if and (.Values.dags.gitSync.enabled) (semverCompare "<2.0.0"
.Values.airflowVersion) }}
- name: dags
emptyDir: {}
{{- if .Values.dags.gitSync.sshKeySecret }}
diff --git a/chart/tests/test_git_sync_webserver.py
b/chart/tests/test_git_sync_webserver.py
index 9f5a830..569b316 100644
--- a/chart/tests/test_git_sync_webserver.py
+++ b/chart/tests/test_git_sync_webserver.py
@@ -26,7 +26,10 @@ from tests.helm_template_generator import render_chart
class GitSyncWebserverTest(unittest.TestCase):
def
test_should_add_dags_volume_to_the_webserver_if_git_sync_and_persistence_is_enabled(self):
docs = render_chart(
- values={"dags": {"gitSync": {"enabled": True}, "persistence":
{"enabled": True}}},
+ values={
+ "airflowVersion": "1.10.14",
+ "dags": {"gitSync": {"enabled": True}, "persistence":
{"enabled": True}},
+ },
show_only=["templates/webserver/webserver-deployment.yaml"],
)
@@ -34,7 +37,10 @@ class GitSyncWebserverTest(unittest.TestCase):
def
test_should_add_dags_volume_to_the_webserver_if_git_sync_is_enabled_and_persistence_is_disabled(self):
docs = render_chart(
- values={"dags": {"gitSync": {"enabled": True}, "persistence":
{"enabled": False}}},
+ values={
+ "airflowVersion": "1.10.14",
+ "dags": {"gitSync": {"enabled": True}, "persistence":
{"enabled": False}},
+ },
show_only=["templates/webserver/webserver-deployment.yaml"],
)
@@ -43,10 +49,11 @@ class GitSyncWebserverTest(unittest.TestCase):
def
test_should_add_git_sync_container_to_webserver_if_persistence_is_not_enabled_but_git_sync_is(self):
docs = render_chart(
values={
+ "airflowVersion": "1.10.14",
"dags": {
"gitSync": {"enabled": True, "containerName": "git-sync"},
"persistence": {"enabled": False},
- }
+ },
},
show_only=["templates/webserver/webserver-deployment.yaml"],
)
@@ -63,18 +70,44 @@ class GitSyncWebserverTest(unittest.TestCase):
"spec.template.spec.serviceAccountName", docs[0]
)
- @parameterized.expand([(True,), (False,)])
- def test_git_sync_with_exclude_webserver(self, exclude_webserver):
+ @parameterized.expand(
+ [
+ (
+ "2.0.0",
+ True,
+ ),
+ (
+ "2.0.2",
+ True,
+ ),
+ (
+ "1.10.14",
+ False,
+ ),
+ (
+ "1.9.0",
+ False,
+ ),
+ (
+ "2.1.0",
+ True,
+ ),
+ ],
+ )
+ def test_git_sync_with_different_airflow_versions(self, airflow_version,
exclude_webserver):
"""
- If that dags.gitSync.excludeWebserver=True - git sync related
containers, volume mounts & volumes
+ If Airflow >= 2.0.0 - git sync related containers, volume mounts &
volumes
are not created.
"""
docs = render_chart(
values={
+ "airflowVersion": airflow_version,
"dags": {
- "gitSync": {"enabled": True, "excludeWebserver":
exclude_webserver},
+ "gitSync": {
+ "enabled": True,
+ },
"persistence": {"enabled": False},
- }
+ },
},
show_only=["templates/webserver/webserver-deployment.yaml"],
)
diff --git a/chart/values.schema.json b/chart/values.schema.json
index ab85567..2d33783 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -1677,10 +1677,6 @@
"description": "Enable Git sync.",
"type": "boolean"
},
- "excludeWebserver": {
- "description": "Disable Git sync on webserver as
it is not needed when DAG Serialization is enabled.",
- "type": "boolean"
- },
"repo": {
"description": "Git repository.",
"type": "string"
diff --git a/chart/values.yaml b/chart/values.yaml
index 2760c22..d8f3493 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -955,12 +955,6 @@ dags:
gitSync:
enabled: false
- # Change it to true when DAG Serialization is turned on. This will exclude
git sync containers
- # from Webserver as DAGs are fetched from the DB. DAG Serialization was
introduced in
- # 1.10.7 and is optional for <2.0.0.
- #
https://airflow.apache.org/docs/apache-airflow/1.10.15/dag-serialization.html
- excludeWebserver: false
-
# git repo clone url
# ssh examples ssh://[email protected]/apache/airflow.git
# [email protected]:apache/airflow.git
diff --git a/docs/helm-chart/manage-dags-files.rst
b/docs/helm-chart/manage-dags-files.rst
index 53a5f98..8a4cfba 100644
--- a/docs/helm-chart/manage-dags-files.rst
+++ b/docs/helm-chart/manage-dags-files.rst
@@ -82,7 +82,7 @@ Mounting DAGs using Git-Sync sidecar with Persistence enabled
This option will use a Persistent Volume Claim with an access mode of
``ReadWriteMany``.
The scheduler pod will sync DAGs from a git repository onto the PVC every
configured number of
-seconds. The other pods will read the synced DAGs. Not all volume plugins
have support for
+seconds. The other pods will read the synced DAGs. Not all volume plugins have
support for
``ReadWriteMany`` access mode.
Refer `Persistent Volume Access Modes
<https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes>`__
for details.
@@ -96,17 +96,11 @@ for details.
# by setting the dags.persistence.* and dags.gitSync.* values
# Please refer to values.yaml for details
-When using ``apache-airflow>=2.0.0``, :ref:`DAG Serialization
<apache-airflow:dag-serialization>` is enabled by default,
-hence Webserver does not need access to DAG files, so you can turn off
``git-sync`` for Webserver by setting
-``dags.gitSync.excludeWebserver`` to ``true``.
-This is also recommended when enabling DAG Serialization for
``apache-airflow>=1.10.11,<2``.
-
.. code-block:: bash
helm upgrade airflow . \
--set dags.persistence.enabled=true \
--set dags.gitSync.enabled=true \
- --set dags.gitSync.excludeWebserver=true
# you can also override the other persistence or gitSync values
# by setting the dags.persistence.* and dags.gitSync.* values
# Please refer to values.yaml for details
@@ -114,7 +108,8 @@ This is also recommended when enabling DAG Serialization
for ``apache-airflow>=1
Mounting DAGs using Git-Sync sidecar without Persistence
--------------------------------------------------------
-This option will use an always running Git-Sync sidecar on every scheduler,
webserver and worker pods.
+This option will use an always running Git-Sync sidecar on every scheduler,
webserver (if ``airflowVersion < 2.0.0``)
+and worker pods.
The Git-Sync sidecar containers will sync DAGs from a git repository every
configured number of
seconds. If you are using the ``KubernetesExecutor``, Git-sync will run as an
init container on your worker pods.
@@ -127,6 +122,9 @@ seconds. If you are using the ``KubernetesExecutor``,
Git-sync will run as an in
# by setting the dags.gitSync.* values
# Refer values.yaml for details
+When using ``apache-airflow>=2.0.0``, :ref:`DAG Serialization
<apache-airflow:dag-serialization>` is enabled by default,
+hence Webserver does not need access to DAG files, so ``git-sync`` sidecar is
not run on Webserver.
+
Mounting DAGs from an externally populated PVC
----------------------------------------------
