This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new db63de6 Chart: Fix updating from ``KubernetesExecutor`` to
``CeleryExecutor`` (#16242)
db63de6 is described below
commit db63de626f53c9e0242f0752bb996d0e32ebf6ea
Author: Jed Cunningham <[email protected]>
AuthorDate: Sun Jun 6 16:07:31 2021 -0600
Chart: Fix updating from ``KubernetesExecutor`` to ``CeleryExecutor``
(#16242)
We will create these secrets (if necessary) _even if_ we aren't
currently using CeleryExecutor or CeleryKubernetesExecutor. As we are
relying on the "pre-install" hack to prevent changing randomly
generated passwords, updating the executor later doesn't give us the
opportunity to deploy them when we need them. We will always deploy
them defensively to make the executor update path actually work.
---
chart/templates/secrets/redis-secrets.yaml | 9 +++++++--
chart/tests/test_basic_helm_chart.py | 20 ++++++++++----------
chart/tests/test_redis.py | 9 +++++++++
chart/values.schema.json | 6 +++---
chart/values.yaml | 3 +++
5 files changed, 32 insertions(+), 15 deletions(-)
diff --git a/chart/templates/secrets/redis-secrets.yaml
b/chart/templates/secrets/redis-secrets.yaml
index f36fcbe..44745bf 100644
--- a/chart/templates/secrets/redis-secrets.yaml
+++ b/chart/templates/secrets/redis-secrets.yaml
@@ -15,11 +15,17 @@
# specific language governing permissions and limitations
# under the License.
+# We will create these secrets (if necessary) _even if_ we aren't
+# currently using CeleryExecutor or CeleryKubernetesExecutor. As we are
+# relying on the "pre-install" hack to prevent changing randomly generated
passwords,
+# updating the executor later doesn't give us the opportunity to deploy them
+# when we need them. We will always deploy them defensively to make the
executor
+# update path actually work.
+
################################
## Airflow Redis Password Secret
#################################
{{- $random_redis_password := randAlphaNum 10 }}
-{{- if or (eq .Values.executor "CeleryExecutor") (eq .Values.executor
"CeleryKubernetesExecutor") }}
{{- if and .Values.redis.enabled (not .Values.redis.passwordSecretName) }}
# If passwordSecretName is not set, we will either use the set password, or
use the generated one
kind: Secret
@@ -73,4 +79,3 @@ data:
connection: {{ (printf "%s" .Values.data.brokerUrl) | b64enc | quote }}
{{- end }}
{{- end }}
-{{- end }}
diff --git a/chart/tests/test_basic_helm_chart.py
b/chart/tests/test_basic_helm_chart.py
index 926a26c..65eeb1d 100644
--- a/chart/tests/test_basic_helm_chart.py
+++ b/chart/tests/test_basic_helm_chart.py
@@ -40,29 +40,32 @@ class TestBaseChartTest(unittest.TestCase):
"fullnameOverride": "TEST-BASIC",
},
)
- list_of_kind_names_tuples = [
+ list_of_kind_names_tuples = {
(k8s_object['kind'], k8s_object['metadata']['name']) for
k8s_object in k8s_objects
- ]
- assert list_of_kind_names_tuples == [
- ('ServiceAccount', 'TEST-BASIC-flower'),
+ }
+ assert list_of_kind_names_tuples == {
('ServiceAccount', 'TEST-BASIC-create-user-job'),
+ ('ServiceAccount', 'TEST-BASIC-flower'),
('ServiceAccount', 'TEST-BASIC-migrate-database-job'),
('ServiceAccount', 'TEST-BASIC-redis'),
('ServiceAccount', 'TEST-BASIC-scheduler'),
('ServiceAccount', 'TEST-BASIC-statsd'),
('ServiceAccount', 'TEST-BASIC-webserver'),
('ServiceAccount', 'TEST-BASIC-worker'),
- ('Secret', 'TEST-BASIC-postgresql'),
('Secret', 'TEST-BASIC-airflow-metadata'),
('Secret', 'TEST-BASIC-airflow-result-backend'),
+ ('Secret', 'TEST-BASIC-broker-url'),
+ ('Secret', 'TEST-BASIC-fernet-key'),
+ ('Secret', 'TEST-BASIC-postgresql'),
+ ('Secret', 'TEST-BASIC-redis-password'),
('ConfigMap', 'TEST-BASIC-airflow-config'),
('Role', 'TEST-BASIC-pod-launcher-role'),
('Role', 'TEST-BASIC-pod-log-reader-role'),
('RoleBinding', 'TEST-BASIC-pod-launcher-rolebinding'),
('RoleBinding', 'TEST-BASIC-pod-log-reader-rolebinding'),
+ ('Service', 'TEST-BASIC-flower'),
('Service', 'TEST-BASIC-postgresql-headless'),
('Service', 'TEST-BASIC-postgresql'),
- ('Service', 'TEST-BASIC-flower'),
('Service', 'TEST-BASIC-redis'),
('Service', 'TEST-BASIC-statsd'),
('Service', 'TEST-BASIC-webserver'),
@@ -74,12 +77,9 @@ class TestBaseChartTest(unittest.TestCase):
('StatefulSet', 'TEST-BASIC-postgresql'),
('StatefulSet', 'TEST-BASIC-redis'),
('StatefulSet', 'TEST-BASIC-worker'),
- ('Secret', 'TEST-BASIC-fernet-key'),
- ('Secret', 'TEST-BASIC-redis-password'),
- ('Secret', 'TEST-BASIC-broker-url'),
('Job', 'TEST-BASIC-create-user'),
('Job', 'TEST-BASIC-run-airflow-migrations'),
- ]
+ }
assert OBJECT_COUNT_IN_BASIC_DEPLOYMENT == len(k8s_objects)
for k8s_object in k8s_objects:
labels = jmespath.search('metadata.labels', k8s_object) or {}
diff --git a/chart/tests/test_redis.py b/chart/tests/test_redis.py
index 067f285..2af8f6b 100644
--- a/chart/tests/test_redis.py
+++ b/chart/tests/test_redis.py
@@ -233,6 +233,15 @@ class RedisTest(unittest.TestCase):
self.assert_broker_url_env(k8s_obj_by_key,
expected_broker_url_secret_name)
+ def test_default_redis_secrets_created_with_non_celery_executor(self):
+ # We want to make sure default redis secrets (if needed) are still
+ # created during install, as they are marked "pre-install".
+ # See note in templates/secrets/redis-secrets.yaml for more.
+ docs = render_chart(
+ values={"executor": "KubernetesExecutor"},
show_only=["templates/secrets/redis-secrets.yaml"]
+ )
+ assert 2 == len(docs)
+
def test_should_create_valid_affinity_tolerations_and_node_selector(self):
docs = render_chart(
values={
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 6e424d1..3e64b03 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -747,7 +747,7 @@
]
},
"brokerUrl": {
- "description": "Direct url to the redis broker (when using
an external redis instance).",
+ "description": "Direct url to the redis broker (when using
an external redis instance) (can only be set during install, not upgrade).",
"type": [
"string",
"null"
@@ -758,7 +758,7 @@
}
},
"fernetKey": {
- "description": "The Fernet key used to encrypt passwords.",
+ "description": "The Fernet key used to encrypt passwords (can only
be set during install, not upgrade).",
"type": [
"string",
"null"
@@ -2234,7 +2234,7 @@
"default": null
},
"password": {
- "description": "If password is set, create secret with it,
else generate a new one on install.",
+ "description": "If password is set, create secret with it,
else generate a new one on install (can only be set during install, not
upgrade).",
"type": [
"string",
"null"
diff --git a/chart/values.yaml b/chart/values.yaml
index 6d1d157..43e2331 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -259,9 +259,11 @@ data:
# port: 5432
# db: postgres
# sslmode: disable
+ # Note: brokerUrl can only be set during install, not upgrade
brokerUrl: ~
# Fernet key settings
+# Note: fernetKey can only be set during install, not upgrade
fernetKey: ~
fernetKeySecretName: ~
@@ -864,6 +866,7 @@ redis:
# Else, if password is set, create secret with it,
# Otherwise a new password will be generated on install
+ # Note: password can only be set during install, not upgrade.
password: ~
# This setting tells kubernetes that its ok to evict
