This is an automated email from the ASF dual-hosted git repository. dimberman pushed a commit to branch v1-10-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 24103e32e5ddb7bc8eea1b64afd4cd7ea1d403e2 Author: Ash Berlin-Taylor <[email protected]> AuthorDate: Thu Jul 16 14:33:07 2020 +0100 Helm chart can now place arbitrary config settings in to airflow.cfg (#9816) Rather than only allowing specific pre-determined config settings, this change allows the user to place _any_ config setting they like in the generated airflow.cfg, including overwriting the "generated defaults". This providers a nicer interface for the users of the chart (even if the could already set these via the env vars). (cherry picked from commit e4790d58b2c5a90567a1712e6115562579e24dbd) --- chart/templates/configmap.yaml | 112 +++-------------------------------------- chart/values.yaml | 84 +++++++++++++++++++++++++++++-- scripts/ci/libraries/_kind.sh | 3 +- 3 files changed, 91 insertions(+), 108 deletions(-) diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 00332b7..b24e474 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -30,113 +30,17 @@ metadata: heritage: {{ .Release.Service }} {{- with .Values.labels }} {{ toYaml . | indent 4 }} -{{- end }} +{{- end -}} +{{- $Global := . }} data: # These are system-specified config overrides. - airflow.cfg: | - [core] - dags_folder = {{ include "airflow_dags" . }} - load_examples = False - colored_console_log = False - executor = {{ .Values.executor }} -{{- if .Values.elasticsearch.enabled }} - remote_logging = True -{{- end }} - - [logging] - logging_level = DEBUG - [webserver] - enable_proxy_fix = True - expose_config = True - rbac = True - -{{- if eq .Values.executor "CeleryExecutor" }} - [celery] - default_queue = celery -{{- end }} - - [scheduler] - scheduler_heartbeat_sec = 5 -{{- if .Values.statsd.enabled }} - statsd_on = True - statsd_port = 9125 - statsd_prefix = airflow - statsd_host = {{ printf "%s-statsd" .Release.Name }} -{{- end }} - # Restart Scheduler every 41460 seconds (11 hours 31 minutes) - # The odd time is chosen so it is not always restarting on the same "hour" boundary - run_duration = 41460 - -{{- if .Values.elasticsearch.enabled }} - [elasticsearch] - # The elasticsearch variables were updated to the shorter names in v1.10.4 - write_stdout = True - json_format = True - log_id_template = {dag_id}_{task_id}_{execution_date}_{try_number} - - [elasticsearch_configs] - max_retries = 3 - timeout = 30 - retry_timeout = True -{{- end }} - -{{- if eq .Values.executor "KubernetesExecutor" }} - [kubernetes] - namespace = {{ .Release.Namespace }} - airflow_configmap = {{ include "airflow_config" . }} - airflow_local_settings_configmap = {{ include "airflow_config" . }} - worker_container_repository = {{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }} - worker_container_tag = {{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }} - worker_container_image_pull_policy = {{ .Values.images.airflow.pullPolicy }} - worker_service_account_name = {{ .Release.Name }}-worker-serviceaccount - image_pull_secrets = {{ template "registry_secret" . }} - dags_in_image = {{ if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}False{{ else }}True{{ end }} - delete_worker_pods = True - run_as_user = {{ .Values.uid }} - fs_group = {{ .Values.gid }} - {{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }} - git_dags_folder_mount_point = {{ include "airflow_dags_mount_path" . }} - dags_volume_mount_point = {{ include "airflow_dags_mount_path" . }} - {{- if .Values.dags.persistence.enabled }} - dags_volume_claim = {{ .Release.Name }}-dags - dags_volume_subpath = {{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }} - {{- else }} - git_repo = {{ .Values.dags.gitSync.repo }} - git_branch = {{ .Values.dags.gitSync.branch }} - git_sync_rev = {{ .Values.dags.gitSync.rev }} - git_sync_depth = {{ .Values.dags.gitSync.depth }} - git_sync_root = {{ .Values.dags.gitSync.root }} - git_sync_dest = {{ .Values.dags.gitSync.dest }} - git_sync_container_repository = {{ .Values.dags.gitSync.containerRepository }} - git_sync_container_tag = {{ .Values.dags.gitSync.containerTag }} - git_sync_init_container_name = {{ .Values.dags.gitSync.containerName }} - git_sync_run_as_user = {{ .Values.uid }} - {{- if .Values.dags.gitSync.knownHosts }} - git_ssh_known_hosts_configmap_name = {{ include "airflow_config" . }} - {{- end }} - {{- if .Values.dags.gitSync.sshKeySecret }} - git_ssh_key_secret_name = {{ .Values.dags.gitSync.sshKeySecret }} - {{- else if .Values.dags.gitSync.credentialsSecret }} - git_sync_credentials_secret = {{ .Values.dags.gitSync.credentialsSecret }} - {{- end }} + airflow.cfg: |- + {{- range $section, $settings := .Values.config }} + [{{ $section }}] + {{- range $key, $val := $settings }} + {{ $key }} = {{ tpl ($val | toString) $Global }} {{- end }} - {{- end }} - - [kubernetes_secrets] - AIRFLOW__CORE__SQL_ALCHEMY_CONN = {{ printf "%s=connection" (include "airflow_metadata_secret" .) }} - AIRFLOW__CORE__FERNET_KEY = {{ printf "%s=fernet-key" (include "fernet_key_secret" .) }} -{{- if .Values.elasticsearch.enabled }} - AIRFLOW__ELASTICSEARCH__ELASTICSEARCH_HOST = {{ printf "%s=connection" (include "elasticsearch_secret" .) }} -{{- end }} - - [kubernetes_labels] - tier = airflow - component = worker - release = {{ .Release.Name }} - {{- range $key, $value := .Values.labels }} - {{ printf "%s = %s" $key $value }} - {{- end }} -{{- end }} + {{ end }} {{- if .Values.webserver.webserverConfig }} webserver_config.py: | diff --git a/chart/values.yaml b/chart/values.yaml index 8fa7ab6..45ea84a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -435,9 +435,87 @@ postgresql: postgresqlPassword: postgres postgresqlUsername: postgres -# Authentication backend used for the experimental API -api: - authBackend: airflow.api.auth.backend.deny_all +# Config settings to go into the mounted airflow.cfg +# +# Please note that these values are passed through the `tpl` function, so are +# all subject to being rendered as go templates. If you need to include a +# litera `{{` in a value, it must be expessed like this: +# +# a: '{{ "{{ not a template }}" }}' +# +# yamllint disable rule:line-length +config: + core: + dags_folder: '{{ include "airflow_dags" . }}' + load_examples: 'False' + colored_console_log: 'False' + executor: '{{ .Values.executor }}' + remote_logging: '{{- ternary "True" "False" .Values.elasticsearch.enabled }}' + # Authentication backend used for the experimental API + api: + auth_backend: airflow.api.auth.backend.deny_all + logging: + logging_level: DEBUG + webserver: + enable_proxy_fix: 'True' + expose_config: 'True' + rbac: 'True' + + celery: + default_queue: celery + + scheduler: + scheduler_heartbeat_sec: 5 + statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}' + statsd_port: 9125 + statsd_prefix: airflow + statsd_host: '{{ printf "%s-statsd" .Release.Name }}' + + # Restart Scheduler every 41460 seconds (11 hours 31 minutes) + # The odd time is chosen so it is not always restarting on the same "hour" boundary + run_duration: 41460 + elasticsearch: + json_format: 'True' + log_id_template: "{dag_id}_{task_id}_{execution_date}_{try_number}" + elasticsearch_configs: + max_retries: 3 + timeout: 30 + retry_timeout: 'True' + + kubernetes: + namespace: '{{ .Release.Namespace }}' + airflow_configmap: '{{ include "airflow_config" . }}' + airflow_local_settings_configmap: '{{ include "airflow_config" . }}' + worker_container_repository: '{{ .Values.images.airflow.repository | default .Values.defaultAirflowRepository }}' + worker_container_tag: '{{ .Values.images.airflow.tag | default .Values.defaultAirflowTag }}' + worker_container_image_pull_policy: '{{ .Values.images.airflow.pullPolicy }}' + worker_service_account_name: '{{ .Release.Name }}-worker-serviceaccount' + image_pull_secrets: '{{ template "registry_secret" . }}' + dags_in_image: '{{ ternary "False" "True" (or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled) }}' + delete_worker_pods: 'True' + run_as_user: '{{ .Values.uid }}' + fs_group: '{{ .Values.gid }}' + git_dags_folder_mount_point: '{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}{{ include "airflow_dags_mount_path" . }}{{end}}' + dags_volume_mount_point: '{{- if or .Values.dags.gitSync.enabled .Values.dags.persistence.enabled }}{{ include "airflow_dags_mount_path" . }}{{ end }}' + dags_volume_claim: '{{- if .Values.dags.persistence.enabled }}{{ include "airflow_dags_volume_claim" . }}{{ end }}' + dags_volume_subpath: '{{- if .Values.dags.persistence.enabled }}{{.Values.dags.gitSync.dest }}/{{ .Values.dags.gitSync.subPath }}{{ end }}' + git_repo: '{{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }}{{ .Values.dags.gitSync.repo }}{{ end }}' + git_branch: '{{ .Values.dags.gitSync.branch }}' + git_sync_rev: '{{ .Values.dags.gitSync.rev }}' + git_sync_depth: '{{ .Values.dags.gitSync.depth }}' + git_sync_root: '{{ .Values.dags.gitSync.root }}' + git_sync_dest: '{{ .Values.dags.gitSync.dest }}' + git_sync_container_repository: '{{ .Values.dags.gitSync.containerRepository }}' + git_sync_container_tag: '{{ .Values.dags.gitSync.containerTag }}' + git_sync_init_container_name: '{{ .Values.dags.gitSync.containerName }}' + git_sync_run_as_user: '{{ .Values.uid }}' + git_ssh_known_hosts_configmap_name: '{{- if .Values.dags.gitSync.knownHosts }}{{ include "airflow_config" . }}{{ end }}' + git_ssh_key_secret_name: '{{- if .Values.dags.gitSync.sshKeySecret }}{{ .Values.dags.gitSync.sshKeySecret }}{{ end }}' + git_sync_credentials_secret: '{{- if .Values.dags.gitSync.credentialsSecret }}{{ .Values.dags.gitSync.credentialsSecret }}{{ end }}' + kubernetes_secrets: + AIRFLOW__CORE__SQL_ALCHEMY_CONN: '{{ printf "%s=connection" (include "airflow_metadata_secret" .) }}' + AIRFLOW__CORE__FERNET_KEY: '{{ printf "%s=fernet-key" (include "fernet_key_secret" .) }}' +# yamllint enable rule:line-length # Git sync dags: diff --git a/scripts/ci/libraries/_kind.sh b/scripts/ci/libraries/_kind.sh index 4f5ebe5..f24b3aa 100644 --- a/scripts/ci/libraries/_kind.sh +++ b/scripts/ci/libraries/_kind.sh @@ -294,7 +294,8 @@ function deploy_airflow_with_helm() { --set "defaultAirflowRepository=${DOCKERHUB_USER}/${DOCKERHUB_REPO}" \ --set "images.airflow.repository=${DOCKERHUB_USER}/${DOCKERHUB_REPO}" \ --set "images.airflow.tag=${AIRFLOW_PROD_BASE_TAG}" -v 1 \ - --set "defaultAirflowTag=${AIRFLOW_PROD_BASE_TAG}" -v 1 + --set "defaultAirflowTag=${AIRFLOW_PROD_BASE_TAG}" -v 1 \ + --set "config.api.auth_backend=airflow.api.auth.backend.default" echo popd || exit 1 }
