This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e89b1aac0a Support git-sync v4 in Helm Chart (#34731)
e89b1aac0a is described below
commit e89b1aac0aacba7f8aeca068a1644ef8463bb972
Author: Junyoung Park <[email protected]>
AuthorDate: Thu Jan 4 04:53:03 2024 +0900
Support git-sync v4 in Helm Chart (#34731)
---------
Co-authored-by: Elad Kalif <[email protected]>
---
chart/templates/_helpers.yaml | 40 +++++++++++
chart/values.schema.json | 19 ++++-
chart/values.yaml | 14 +++-
helm_tests/airflow_aux/test_pod_template_file.py | 33 ++++++++-
helm_tests/other/test_git_sync_scheduler.py | 89 ++++++++++++++++++++++++
5 files changed, 188 insertions(+), 7 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 719b6647b5..69e7dae634 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -202,16 +202,26 @@ If release name contains chart name it will be used as a
full name.
{{- if .Values.dags.gitSync.sshKeySecret }}
- name: GIT_SSH_KEY_FILE
value: "/etc/git-secret/ssh"
+ - name: GITSYNC_SSH_KEY_FILE
+ value: "/etc/git-secret/ssh"
- name: GIT_SYNC_SSH
value: "true"
+ - name: GITSYNC_SSH
+ value: "true"
{{- if .Values.dags.gitSync.knownHosts }}
- name: GIT_KNOWN_HOSTS
value: "true"
+ - name: GITSYNC_SSH_KNOWN_HOSTS
+ value: "true"
- name: GIT_SSH_KNOWN_HOSTS_FILE
value: "/etc/git-secret/known_hosts"
+ - name: GITSYNC_SSH_KNOWN_HOSTS_FILE
+ value: "/etc/git-secret/known_hosts"
{{- else }}
- name: GIT_KNOWN_HOSTS
value: "false"
+ - name: GITSYNC_SSH_KNOWN_HOSTS
+ value: "false"
{{- end }}
{{ else if .Values.dags.gitSync.credentialsSecret }}
- name: GIT_SYNC_USERNAME
@@ -219,33 +229,63 @@ If release name contains chart name it will be used as a
full name.
secretKeyRef:
name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
key: GIT_SYNC_USERNAME
+ - name: GITSYNC_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
+ key: GITSYNC_USERNAME
- name: GIT_SYNC_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
key: GIT_SYNC_PASSWORD
+ - name: GITSYNC_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Values.dags.gitSync.credentialsSecret | quote }}
+ key: GITSYNC_PASSWORD
{{- end }}
- name: GIT_SYNC_REV
value: {{ .Values.dags.gitSync.rev | quote }}
+ - name: GITSYNC_REF
+ value: {{ .Values.dags.gitSync.ref | quote }}
- name: GIT_SYNC_BRANCH
value: {{ .Values.dags.gitSync.branch | quote }}
- name: GIT_SYNC_REPO
value: {{ .Values.dags.gitSync.repo | quote }}
+ - name: GITSYNC_REPO
+ value: {{ .Values.dags.gitSync.repo | quote }}
- name: GIT_SYNC_DEPTH
value: {{ .Values.dags.gitSync.depth | quote }}
+ - name: GITSYNC_DEPTH
+ value: {{ .Values.dags.gitSync.depth | quote }}
- name: GIT_SYNC_ROOT
value: "/git"
+ - name: GITSYNC_ROOT
+ value: "/git"
- name: GIT_SYNC_DEST
value: "repo"
+ - name: GITSYNC_LINK
+ value: "repo"
- name: GIT_SYNC_ADD_USER
value: "true"
+ - name: GITSYNC_ADD_USER
+ value: "true"
+ {{- if .Values.dags.gitSync.wait }}
- name: GIT_SYNC_WAIT
value: {{ .Values.dags.gitSync.wait | quote }}
+ {{- end }}
+ - name: GITSYNC_PERIOD
+ value: {{ .Values.dags.gitSync.period | quote }}
- name: GIT_SYNC_MAX_SYNC_FAILURES
value: {{ .Values.dags.gitSync.maxFailures | quote }}
+ - name: GITSYNC_MAX_FAILURES
+ value: {{ .Values.dags.gitSync.maxFailures | quote }}
{{- if .is_init }}
- name: GIT_SYNC_ONE_TIME
value: "true"
+ - name: GITSYNC_ONE_TIME
+ value: "true"
{{- end }}
{{- with .Values.dags.gitSync.env }}
{{- toYaml . | nindent 4 }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 41501484bc..0d931609c0 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -776,7 +776,7 @@
"tag": {
"description": "The gitSync image tag.",
"type": "string",
- "default": "v3.6.9"
+ "default": "v4.1.0"
},
"pullPolicy": {
"description": "The gitSync image pull policy.",
@@ -7136,6 +7136,11 @@
"type": "string",
"default": "HEAD"
},
+ "ref": {
+ "description": "Git revision branch, tag, or
hash.",
+ "type": "string",
+ "default": "v2-2-stable"
+ },
"depth": {
"description": "Repository depth.",
"type": "integer",
@@ -7153,8 +7158,16 @@
},
"wait": {
"description": "Interval between git sync attempts
in seconds. High values are more likely to cause DAGs to become out of sync
between different components. Low values cause more traffic to the remote git
repository.",
- "type": "integer",
- "default": 5
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null
+ },
+ "period": {
+ "description": "Interval between git sync attempts
in Go-style duration string. High values are more likely to cause DAGs to
become out of sync between different components. Low values cause more traffic
to the remote git repository.",
+ "type": "string",
+ "default": "5s"
},
"containerName": {
"description": "Git sync container name.",
diff --git a/chart/values.yaml b/chart/values.yaml
index e0ac00f071..e67a1f6f64 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -121,7 +121,7 @@ images:
pullPolicy: IfNotPresent
gitSync:
repository: registry.k8s.io/git-sync/git-sync
- tag: v3.6.9
+ tag: v4.1.0
pullPolicy: IfNotPresent
# Select certain nodes for airflow pods.
@@ -2420,6 +2420,8 @@ dags:
repo: https://github.com/apache/airflow.git
branch: v2-2-stable
rev: HEAD
+ # The git revision (branch, tag, or hash) to check out, v4 only
+ ref: v2-2-stable
depth: 1
# the number of consecutive failures allowed before aborting
maxFailures: 0
@@ -2434,8 +2436,12 @@ dags:
# metadata:
# name: git-credentials
# data:
+ # # For git-sync v3
# GIT_SYNC_USERNAME: <base64_encoded_git_username>
# GIT_SYNC_PASSWORD: <base64_encoded_git_password>
+ # # For git-sync v4
+ # GITSYNC_USERNAME: <base64_encoded_git_username>
+ # GITSYNC_PASSWORD: <base64_encoded_git_password>
# and specify the name of the secret below
#
# credentialsSecret: git-credentials
@@ -2464,7 +2470,11 @@ dags:
# interval between git sync attempts in seconds
# high values are more likely to cause DAGs to become out of sync between
different components
# low values cause more traffic to the remote git repository
- wait: 5
+ # Go-style duration string (e.g. "100ms" or "0.1s" = 100ms).
+ # For backwards compatibility, wait will be used if it is specified.
+ period: 5s
+ wait: ~
+
containerName: git-sync
uid: 65533
diff --git a/helm_tests/airflow_aux/test_pod_template_file.py
b/helm_tests/airflow_aux/test_pod_template_file.py
index c1a831e6b1..4002dfa3a8 100644
--- a/helm_tests/airflow_aux/test_pod_template_file.py
+++ b/helm_tests/airflow_aux/test_pod_template_file.py
@@ -70,10 +70,12 @@ class TestPodTemplateFile:
"gitSync": {
"enabled": True,
"containerName": "git-sync-test",
- "wait": 66,
+ "wait": None,
+ "period": "66s",
"maxFailures": 70,
"subPath": "path1/path2",
"rev": "HEAD",
+ "ref": "test-branch",
"depth": 1,
"repo": "https://github.com/apache/airflow.git";,
"branch": "test-branch",
@@ -95,15 +97,23 @@ class TestPodTemplateFile:
"imagePullPolicy": "Always",
"env": [
{"name": "GIT_SYNC_REV", "value": "HEAD"},
+ {"name": "GITSYNC_REF", "value": "test-branch"},
{"name": "GIT_SYNC_BRANCH", "value": "test-branch"},
{"name": "GIT_SYNC_REPO", "value":
"https://github.com/apache/airflow.git"},
+ {"name": "GITSYNC_REPO", "value":
"https://github.com/apache/airflow.git"},
{"name": "GIT_SYNC_DEPTH", "value": "1"},
+ {"name": "GITSYNC_DEPTH", "value": "1"},
{"name": "GIT_SYNC_ROOT", "value": "/git"},
+ {"name": "GITSYNC_ROOT", "value": "/git"},
{"name": "GIT_SYNC_DEST", "value": "repo"},
+ {"name": "GITSYNC_LINK", "value": "repo"},
{"name": "GIT_SYNC_ADD_USER", "value": "true"},
- {"name": "GIT_SYNC_WAIT", "value": "66"},
+ {"name": "GITSYNC_ADD_USER", "value": "true"},
+ {"name": "GITSYNC_PERIOD", "value": "66s"},
{"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"},
+ {"name": "GITSYNC_MAX_FAILURES", "value": "70"},
{"name": "GIT_SYNC_ONE_TIME", "value": "true"},
+ {"name": "GITSYNC_ONE_TIME", "value": "true"},
],
"volumeMounts": [{"mountPath": "/git", "name": "dags"}],
"resources": {},
@@ -185,12 +195,21 @@ class TestPodTemplateFile:
assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in
jmespath.search(
"spec.initContainers[0].env", docs[0]
)
+ assert {"name": "GITSYNC_SSH_KEY_FILE", "value":
"/etc/git-secret/ssh"} in jmespath.search(
+ "spec.initContainers[0].env", docs[0]
+ )
assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search(
"spec.initContainers[0].env", docs[0]
)
+ assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search(
+ "spec.initContainers[0].env", docs[0]
+ )
assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in
jmespath.search(
"spec.initContainers[0].env", docs[0]
)
+ assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in
jmespath.search(
+ "spec.initContainers[0].env", docs[0]
+ )
assert {
"name": "git-sync-ssh-key",
"mountPath": "/etc/git-secret/ssh",
@@ -256,6 +275,16 @@ class TestPodTemplateFile:
"valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key":
"GIT_SYNC_PASSWORD"}},
} in jmespath.search("spec.initContainers[0].env", docs[0])
+ # Testing git-sync v4
+ assert {
+ "name": "GITSYNC_USERNAME",
+ "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key":
"GITSYNC_USERNAME"}},
+ } in jmespath.search("spec.initContainers[0].env", docs[0])
+ assert {
+ "name": "GITSYNC_PASSWORD",
+ "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key":
"GITSYNC_PASSWORD"}},
+ } in jmespath.search("spec.initContainers[0].env", docs[0])
+
def
test_should_set_the_dags_volume_claim_correctly_when_using_an_existing_claim(self):
docs = render_chart(
values={"dags": {"persistence": {"enabled": True, "existingClaim":
"test-claim"}}},
diff --git a/helm_tests/other/test_git_sync_scheduler.py
b/helm_tests/other/test_git_sync_scheduler.py
index 91039954f8..7bf7b83ae5 100644
--- a/helm_tests/other/test_git_sync_scheduler.py
+++ b/helm_tests/other/test_git_sync_scheduler.py
@@ -52,6 +52,66 @@ class TestGitSyncSchedulerTest:
)
def test_validate_the_git_sync_container_spec(self):
+ docs = render_chart(
+ values={
+ "images": {
+ "gitSync": {
+ "repository": "test-registry/test-repo",
+ "tag": "test-tag",
+ "pullPolicy": "Always",
+ }
+ },
+ "dags": {
+ "gitSync": {
+ "enabled": True,
+ "containerName": "git-sync-test",
+ "wait": None,
+ "period": "66s",
+ "maxFailures": 70,
+ "subPath": "path1/path2",
+ "rev": "HEAD",
+ "ref": "test-branch",
+ "depth": 1,
+ "repo": "https://github.com/apache/airflow.git";,
+ "branch": "test-branch",
+ "sshKeySecret": None,
+ "credentialsSecret": None,
+ "knownHosts": None,
+ },
+ "persistence": {"enabled": True},
+ },
+ },
+ show_only=["templates/scheduler/scheduler-deployment.yaml"],
+ )
+
+ assert {
+ "name": "git-sync-test",
+ "securityContext": {"runAsUser": 65533},
+ "image": "test-registry/test-repo:test-tag",
+ "imagePullPolicy": "Always",
+ "env": [
+ {"name": "GIT_SYNC_REV", "value": "HEAD"},
+ {"name": "GITSYNC_REF", "value": "test-branch"},
+ {"name": "GIT_SYNC_BRANCH", "value": "test-branch"},
+ {"name": "GIT_SYNC_REPO", "value":
"https://github.com/apache/airflow.git"},
+ {"name": "GITSYNC_REPO", "value":
"https://github.com/apache/airflow.git"},
+ {"name": "GIT_SYNC_DEPTH", "value": "1"},
+ {"name": "GITSYNC_DEPTH", "value": "1"},
+ {"name": "GIT_SYNC_ROOT", "value": "/git"},
+ {"name": "GITSYNC_ROOT", "value": "/git"},
+ {"name": "GIT_SYNC_DEST", "value": "repo"},
+ {"name": "GITSYNC_LINK", "value": "repo"},
+ {"name": "GIT_SYNC_ADD_USER", "value": "true"},
+ {"name": "GITSYNC_ADD_USER", "value": "true"},
+ {"name": "GITSYNC_PERIOD", "value": "66s"},
+ {"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"},
+ {"name": "GITSYNC_MAX_FAILURES", "value": "70"},
+ ],
+ "volumeMounts": [{"mountPath": "/git", "name": "dags"}],
+ "resources": {},
+ } == jmespath.search("spec.template.spec.containers[1]", docs[0])
+
+ def test_validate_the_git_sync_container_spec_if_wait_specified(self):
docs = render_chart(
values={
"images": {
@@ -66,9 +126,11 @@ class TestGitSyncSchedulerTest:
"enabled": True,
"containerName": "git-sync-test",
"wait": 66,
+ "period": "66s",
"maxFailures": 70,
"subPath": "path1/path2",
"rev": "HEAD",
+ "ref": "test-branch",
"depth": 1,
"repo": "https://github.com/apache/airflow.git";,
"branch": "test-branch",
@@ -89,14 +151,22 @@ class TestGitSyncSchedulerTest:
"imagePullPolicy": "Always",
"env": [
{"name": "GIT_SYNC_REV", "value": "HEAD"},
+ {"name": "GITSYNC_REF", "value": "test-branch"},
{"name": "GIT_SYNC_BRANCH", "value": "test-branch"},
{"name": "GIT_SYNC_REPO", "value":
"https://github.com/apache/airflow.git"},
+ {"name": "GITSYNC_REPO", "value":
"https://github.com/apache/airflow.git"},
{"name": "GIT_SYNC_DEPTH", "value": "1"},
+ {"name": "GITSYNC_DEPTH", "value": "1"},
{"name": "GIT_SYNC_ROOT", "value": "/git"},
+ {"name": "GITSYNC_ROOT", "value": "/git"},
{"name": "GIT_SYNC_DEST", "value": "repo"},
+ {"name": "GITSYNC_LINK", "value": "repo"},
{"name": "GIT_SYNC_ADD_USER", "value": "true"},
+ {"name": "GITSYNC_ADD_USER", "value": "true"},
{"name": "GIT_SYNC_WAIT", "value": "66"},
+ {"name": "GITSYNC_PERIOD", "value": "66s"},
{"name": "GIT_SYNC_MAX_SYNC_FAILURES", "value": "70"},
+ {"name": "GITSYNC_MAX_FAILURES", "value": "70"},
],
"volumeMounts": [{"mountPath": "/git", "name": "dags"}],
"resources": {},
@@ -121,12 +191,21 @@ class TestGitSyncSchedulerTest:
assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in
jmespath.search(
"spec.template.spec.containers[1].env", docs[0]
)
+ assert {"name": "GITSYNC_SSH_KEY_FILE", "value":
"/etc/git-secret/ssh"} in jmespath.search(
+ "spec.template.spec.containers[1].env", docs[0]
+ )
assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search(
"spec.template.spec.containers[1].env", docs[0]
)
+ assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search(
+ "spec.template.spec.containers[1].env", docs[0]
+ )
assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in
jmespath.search(
"spec.template.spec.containers[1].env", docs[0]
)
+ assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in
jmespath.search(
+ "spec.template.spec.containers[1].env", docs[0]
+ )
assert {
"name": "git-sync-ssh-key",
"secret": {"secretName": "ssh-secret", "defaultMode": 288},
@@ -173,6 +252,16 @@ class TestGitSyncSchedulerTest:
"valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key":
"GIT_SYNC_PASSWORD"}},
} in jmespath.search("spec.template.spec.containers[1].env", docs[0])
+ # Testing git-sync v4
+ assert {
+ "name": "GITSYNC_USERNAME",
+ "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key":
"GITSYNC_USERNAME"}},
+ } in jmespath.search("spec.template.spec.containers[1].env", docs[0])
+ assert {
+ "name": "GITSYNC_PASSWORD",
+ "valueFrom": {"secretKeyRef": {"name": "user-pass-secret", "key":
"GITSYNC_PASSWORD"}},
+ } in jmespath.search("spec.template.spec.containers[1].env", docs[0])
+
def
test_should_set_the_volume_claim_correctly_when_using_an_existing_claim(self):
docs = render_chart(
values={"dags": {"persistence": {"enabled": True, "existingClaim":
"test-claim"}}},
