This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 87a3db98fc5 Start the api-server in AF3, webserver in AF2 (#47085)
87a3db98fc5 is described below
commit 87a3db98fc57eebea7c61cf04efd94c555058820
Author: Jed Cunningham <[email protected]>
AuthorDate: Wed Feb 26 11:21:24 2025 -0700
Start the api-server in AF3, webserver in AF2 (#47085)
* Start the api-server in AF3, webserver in AF2
Let's start the right "webserver" depending on the Airflow version -
api-server in AF3 and webserver in AF2.
* Fix k8s tests?
* Fix health request method
---
chart/templates/_helpers.yaml | 4 +-
.../api-server/api-server-deployment.yaml | 122 ++++++++++-----------
.../api-server/api-server-networkpolicy.yaml | 10 +-
.../api-server/api-server-poddisruptionbudget.yaml | 8 +-
chart/templates/api-server/api-server-service.yaml | 18 +--
.../api-server/api-server-serviceaccount.yaml | 12 +-
chart/templates/configmaps/configmap.yaml | 2 +-
.../templates/webserver/webserver-deployment.yaml | 2 +-
chart/templates/webserver/webserver-hpa.yaml | 4 +-
chart/templates/webserver/webserver-ingress.yaml | 2 +-
.../webserver/webserver-networkpolicy.yaml | 2 +-
.../webserver/webserver-poddisruptionbudget.yaml | 2 +-
chart/templates/webserver/webserver-service.yaml | 2 +-
.../webserver/webserver-serviceaccount.yaml | 2 +
chart/values.schema.json | 12 +-
chart/values.yaml | 8 +-
.../src/airflow_breeze/utils/kubernetes_utils.py | 36 +++---
helm_tests/airflow_aux/test_basic_helm_chart.py | 16 ++-
helm_tests/airflow_core/test_api_server.py | 68 ++++++------
helm_tests/security/test_rbac.py | 42 +++++--
kubernetes_tests/test_base.py | 24 ++--
scripts/ci/kubernetes/nodeport.yaml | 6 +-
22 files changed, 215 insertions(+), 189 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index 718674f8074..c76cefa843e 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -620,8 +620,8 @@ server_tls_key_file = /etc/pgbouncer/server.key
{{/* Create the name of the API server service account to use */}}
-{{- define "_apiServer.serviceAccountName" -}}
- {{- include "_serviceAccountName" (merge (dict "key" "_apiServer"
"nameSuffix" "api-server" ) .) -}}
+{{- define "apiServer.serviceAccountName" -}}
+ {{- include "_serviceAccountName" (merge (dict "key" "apiServer"
"nameSuffix" "api-server" ) .) -}}
{{- end }}
{{/* Create the name of the redis service account to use */}}
diff --git a/chart/templates/api-server/api-server-deployment.yaml
b/chart/templates/api-server/api-server-deployment.yaml
index 3ce0fb9ac26..b4cf2cd4461 100644
--- a/chart/templates/api-server/api-server-deployment.yaml
+++ b/chart/templates/api-server/api-server-deployment.yaml
@@ -21,15 +21,15 @@
## Airflow API Server Deployment
#################################
{{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
-{{- $nodeSelector := or .Values._apiServer.nodeSelector .Values.nodeSelector }}
-{{- $affinity := or .Values._apiServer.affinity .Values.affinity }}
-{{- $tolerations := or .Values._apiServer.tolerations .Values.tolerations }}
-{{- $topologySpreadConstraints := or
.Values._apiServer.topologySpreadConstraints .Values.topologySpreadConstraints
}}
-{{- $revisionHistoryLimit := or .Values._apiServer.revisionHistoryLimit
.Values.revisionHistoryLimit }}
-{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values._apiServer) }}
-{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values._apiServer) }}
-{{- $containerSecurityContextWaitForMigrations := include
"containerSecurityContext" (list . .Values._apiServer.waitForMigrations) }}
-{{- $containerLifecycleHooks := or .Values._apiServer.containerLifecycleHooks
.Values.containerLifecycleHooks }}
+{{- $nodeSelector := or .Values.apiServer.nodeSelector .Values.nodeSelector }}
+{{- $affinity := or .Values.apiServer.affinity .Values.affinity }}
+{{- $tolerations := or .Values.apiServer.tolerations .Values.tolerations }}
+{{- $topologySpreadConstraints := or
.Values.apiServer.topologySpreadConstraints .Values.topologySpreadConstraints }}
+{{- $revisionHistoryLimit := or .Values.apiServer.revisionHistoryLimit
.Values.revisionHistoryLimit }}
+{{- $securityContext := include "airflowPodSecurityContext" (list .
.Values.apiServer) }}
+{{- $containerSecurityContext := include "containerSecurityContext" (list .
.Values.apiServer) }}
+{{- $containerSecurityContextWaitForMigrations := include
"containerSecurityContext" (list . .Values.apiServer.waitForMigrations) }}
+{{- $containerLifecycleHooks := or .Values.apiServer.containerLifecycleHooks
.Values.containerLifecycleHooks }}
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -43,17 +43,17 @@ metadata:
{{- with .Values.labels }}
{{- toYaml . | nindent 4 }}
{{- end }}
- {{- if .Values._apiServer.annotations }}
- annotations: {{- toYaml .Values._apiServer.annotations | nindent 4 }}
+ {{- if .Values.apiServer.annotations }}
+ annotations: {{- toYaml .Values.apiServer.annotations | nindent 4 }}
{{- end }}
spec:
- replicas: {{ .Values._apiServer.replicas }}
+ replicas: {{ .Values.apiServer.replicas }}
{{- if $revisionHistoryLimit }}
revisionHistoryLimit: {{ $revisionHistoryLimit }}
{{- end }}
strategy:
- {{- if .Values._apiServer.strategy }}
- {{- toYaml .Values._apiServer.strategy | nindent 4 }}
+ {{- if .Values.apiServer.strategy }}
+ {{- toYaml .Values.apiServer.strategy | nindent 4 }}
{{- else }}
# Here we define the rolling update strategy
# - maxSurge define how many pod we can add at a time
@@ -78,8 +78,8 @@ spec:
tier: airflow
component: api-server
release: {{ .Release.Name }}
- {{- if or (.Values.labels) (.Values._apiServer.labels) }}
- {{- mustMerge .Values._apiServer.labels .Values.labels | toYaml |
nindent 8 }}
+ {{- if or (.Values.labels) (.Values.apiServer.labels) }}
+ {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml |
nindent 8 }}
{{- end }}
annotations:
checksum/metadata-secret: {{ include (print $.Template.BasePath
"/secrets/metadata-connection-secret.yaml") . | sha256sum }}
@@ -90,16 +90,16 @@ spec:
{{- if .Values.airflowPodAnnotations }}
{{- toYaml .Values.airflowPodAnnotations | nindent 8 }}
{{- end }}
- {{- if .Values._apiServer.podAnnotations }}
- {{- toYaml .Values._apiServer.podAnnotations | nindent 8 }}
+ {{- if .Values.apiServer.podAnnotations }}
+ {{- toYaml .Values.apiServer.podAnnotations | nindent 8 }}
{{- end }}
spec:
- {{- if .Values._apiServer.hostAliases }}
- hostAliases: {{- toYaml .Values._apiServer.hostAliases | nindent 8 }}
+ {{- if .Values.apiServer.hostAliases }}
+ hostAliases: {{- toYaml .Values.apiServer.hostAliases | nindent 8 }}
{{- end }}
- serviceAccountName: {{ include "_apiServer.serviceAccountName" . }}
- {{- if .Values._apiServer.priorityClassName }}
- priorityClassName: {{ .Values._apiServer.priorityClassName }}
+ serviceAccountName: {{ include "apiServer.serviceAccountName" . }}
+ {{- if .Values.apiServer.priorityClassName }}
+ priorityClassName: {{ .Values.apiServer.priorityClassName }}
{{- end }}
{{- if .Values.schedulerName }}
schedulerName: {{ .Values.schedulerName }}
@@ -127,9 +127,9 @@ spec:
- name: {{ template "registry_secret" . }}
{{- end }}
initContainers:
- {{- if .Values._apiServer.waitForMigrations.enabled }}
+ {{- if .Values.apiServer.waitForMigrations.enabled }}
- name: wait-for-airflow-migrations
- resources: {{- toYaml .Values._apiServer.resources | nindent 12 }}
+ resources: {{- toYaml .Values.apiServer.resources | nindent 12 }}
image: {{ template "airflow_image_for_migrations" . }}
imagePullPolicy: {{ .Values.images.airflow.pullPolicy }}
securityContext: {{ $containerSecurityContextWaitForMigrations |
nindent 12 }}
@@ -138,20 +138,20 @@ spec:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
- {{- if .Values._apiServer.extraVolumeMounts }}
- {{- tpl (toYaml .Values._apiServer.extraVolumeMounts) . |
nindent 12 }}
+ {{- if .Values.apiServer.extraVolumeMounts }}
+ {{- tpl (toYaml .Values.apiServer.extraVolumeMounts) . | nindent
12 }}
{{- end }}
args: {{- include "wait-for-migrations-command" . | indent 10 }}
envFrom: {{- include "custom_airflow_environment_from" . | default
"\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
- {{- if .Values._apiServer.waitForMigrations.env }}
- {{- tpl (toYaml .Values._apiServer.waitForMigrations.env) $ |
nindent 12 }}
+ {{- if .Values.apiServer.waitForMigrations.env }}
+ {{- tpl (toYaml .Values.apiServer.waitForMigrations.env) $ |
nindent 12 }}
{{- end }}
{{- end }}
- {{- if .Values._apiServer.extraInitContainers }}
- {{- toYaml .Values._apiServer.extraInitContainers | nindent 8 }}
+ {{- if .Values.apiServer.extraInitContainers }}
+ {{- toYaml .Values.apiServer.extraInitContainers | nindent 8 }}
{{- end }}
containers:
- name: api-server
@@ -161,13 +161,13 @@ spec:
{{- if $containerLifecycleHooks }}
lifecycle: {{- tpl (toYaml $containerLifecycleHooks) . | nindent 12
}}
{{- end }}
- {{- if .Values._apiServer.command }}
- command: {{ tpl (toYaml .Values._apiServer.command) . | nindent 12 }}
+ {{- if .Values.apiServer.command }}
+ command: {{ tpl (toYaml .Values.apiServer.command) . | nindent 12 }}
{{- end }}
- {{- if .Values._apiServer.args }}
- args: {{- tpl (toYaml .Values._apiServer.args) . | nindent 12 }}
+ {{- if .Values.apiServer.args }}
+ args: {{- tpl (toYaml .Values.apiServer.args) . | nindent 12 }}
{{- end }}
- resources: {{- toYaml .Values._apiServer.resources | nindent 12 }}
+ resources: {{- toYaml .Values.apiServer.resources | nindent 12 }}
volumeMounts:
{{- include "airflow_config_mount" . | nindent 12 }}
{{- if .Values.logs.persistence.enabled }}
@@ -177,48 +177,48 @@ spec:
{{- if .Values.volumeMounts }}
{{- toYaml .Values.volumeMounts | nindent 12 }}
{{- end }}
- {{- if .Values._apiServer.extraVolumeMounts }}
- {{- tpl (toYaml .Values._apiServer.extraVolumeMounts) . |
nindent 12 }}
+ {{- if .Values.apiServer.extraVolumeMounts }}
+ {{- tpl (toYaml .Values.apiServer.extraVolumeMounts) . | nindent
12 }}
{{- end }}
ports:
- name: api-server
- containerPort: {{ .Values.ports._apiServer }}
+ containerPort: {{ .Values.ports.apiServer }}
livenessProbe:
httpGet:
path: /public/version
- port: {{ .Values.ports._apiServer }}
- scheme: {{ .Values._apiServer.livenessProbe.scheme | default
"http" }}
- initialDelaySeconds: {{
.Values._apiServer.livenessProbe.initialDelaySeconds }}
- timeoutSeconds: {{ .Values._apiServer.livenessProbe.timeoutSeconds
}}
- failureThreshold: {{
.Values._apiServer.livenessProbe.failureThreshold }}
- periodSeconds: {{ .Values._apiServer.livenessProbe.periodSeconds }}
+ port: {{ .Values.ports.apiServer }}
+ scheme: {{ .Values.apiServer.livenessProbe.scheme | default
"http" }}
+ initialDelaySeconds: {{
.Values.apiServer.livenessProbe.initialDelaySeconds }}
+ timeoutSeconds: {{ .Values.apiServer.livenessProbe.timeoutSeconds
}}
+ failureThreshold: {{
.Values.apiServer.livenessProbe.failureThreshold }}
+ periodSeconds: {{ .Values.apiServer.livenessProbe.periodSeconds }}
readinessProbe:
httpGet:
path: /public/version
- port: {{ .Values.ports._apiServer }}
- scheme: {{ .Values._apiServer.readinessProbe.scheme | default
"http" }}
- initialDelaySeconds: {{
.Values._apiServer.readinessProbe.initialDelaySeconds }}
- timeoutSeconds: {{
.Values._apiServer.readinessProbe.timeoutSeconds }}
- failureThreshold: {{
.Values._apiServer.readinessProbe.failureThreshold }}
- periodSeconds: {{ .Values._apiServer.readinessProbe.periodSeconds
}}
+ port: {{ .Values.ports.apiServer }}
+ scheme: {{ .Values.apiServer.readinessProbe.scheme | default
"http" }}
+ initialDelaySeconds: {{
.Values.apiServer.readinessProbe.initialDelaySeconds }}
+ timeoutSeconds: {{ .Values.apiServer.readinessProbe.timeoutSeconds
}}
+ failureThreshold: {{
.Values.apiServer.readinessProbe.failureThreshold }}
+ periodSeconds: {{ .Values.apiServer.readinessProbe.periodSeconds }}
startupProbe:
httpGet:
path: /public/version
- port: {{ .Values.ports._apiServer }}
- scheme: {{ .Values._apiServer.startupProbe.scheme | default
"http" }}
- timeoutSeconds: {{ .Values._apiServer.startupProbe.timeoutSeconds
}}
- failureThreshold: {{
.Values._apiServer.startupProbe.failureThreshold }}
- periodSeconds: {{ .Values._apiServer.startupProbe.periodSeconds }}
+ port: {{ .Values.ports.apiServer }}
+ scheme: {{ .Values.apiServer.startupProbe.scheme | default
"http" }}
+ timeoutSeconds: {{ .Values.apiServer.startupProbe.timeoutSeconds }}
+ failureThreshold: {{
.Values.apiServer.startupProbe.failureThreshold }}
+ periodSeconds: {{ .Values.apiServer.startupProbe.periodSeconds }}
envFrom: {{- include "custom_airflow_environment_from" . | default
"\n []" | indent 10 }}
env:
{{- include "custom_airflow_environment" . | indent 10 }}
{{- include "standard_airflow_environment" . | indent 10 }}
- {{- include "container_extra_envs" (list . .Values._apiServer.env)
| indent 10 }}
+ {{- include "container_extra_envs" (list . .Values.apiServer.env)
| indent 10 }}
{{- if and (.Values.dags.gitSync.enabled) (not
.Values.dags.persistence.enabled) (semverCompare "<2.0.0"
.Values.airflowVersion) }}
{{- include "git_sync_container" . | nindent 8 }}
{{- end }}
- {{- if .Values._apiServer.extraContainers }}
- {{- tpl (toYaml .Values._apiServer.extraContainers) . | nindent 8 }}
+ {{- if .Values.apiServer.extraContainers }}
+ {{- tpl (toYaml .Values.apiServer.extraContainers) . | nindent 8 }}
{{- end }}
volumes:
- name: config
@@ -234,7 +234,7 @@ spec:
{{- if .Values.volumes }}
{{- toYaml .Values.volumes | nindent 8 }}
{{- end }}
- {{- if .Values._apiServer.extraVolumes }}
- {{- tpl (toYaml .Values._apiServer.extraVolumes) . | nindent 8 }}
+ {{- if .Values.apiServer.extraVolumes }}
+ {{- tpl (toYaml .Values.apiServer.extraVolumes) . | nindent 8 }}
{{- end }}
{{- end }}
diff --git a/chart/templates/api-server/api-server-networkpolicy.yaml
b/chart/templates/api-server/api-server-networkpolicy.yaml
index d648ea4baa2..af460181120 100644
--- a/chart/templates/api-server/api-server-networkpolicy.yaml
+++ b/chart/templates/api-server/api-server-networkpolicy.yaml
@@ -32,8 +32,8 @@ metadata:
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
- {{- if or (.Values.labels) (.Values._apiServer.labels) }}
- {{- mustMerge .Values._apiServer.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- if or (.Values.labels) (.Values.apiServer.labels) }}
+ {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent
4 }}
{{- end }}
spec:
podSelector:
@@ -43,11 +43,11 @@ spec:
release: {{ .Release.Name }}
policyTypes:
- Ingress
- {{- if .Values._apiServer.networkPolicy.ingress.from }}
+ {{- if .Values.apiServer.networkPolicy.ingress.from }}
ingress:
- - from: {{- toYaml .Values._apiServer.networkPolicy.ingress.from | nindent
6 }}
+ - from: {{- toYaml .Values.apiServer.networkPolicy.ingress.from | nindent
6 }}
ports:
- {{ range .Values._apiServer.networkPolicy.ingress.ports }}
+ {{ range .Values.apiServer.networkPolicy.ingress.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
diff --git a/chart/templates/api-server/api-server-poddisruptionbudget.yaml
b/chart/templates/api-server/api-server-poddisruptionbudget.yaml
index c8d9249e4ac..7d0b162e41e 100644
--- a/chart/templates/api-server/api-server-poddisruptionbudget.yaml
+++ b/chart/templates/api-server/api-server-poddisruptionbudget.yaml
@@ -21,7 +21,7 @@
## Airflow api-server PodDisruptionBudget
#################################
{{- if semverCompare ">=3.0.0" .Values.airflowVersion }}
-{{- if .Values._apiServer.podDisruptionBudget.enabled }}
+{{- if .Values.apiServer.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
@@ -32,8 +32,8 @@ metadata:
release: {{ .Release.Name }}
chart: {{ .Chart.Name }}
heritage: {{ .Release.Service }}
- {{- if or (.Values.labels) (.Values._apiServer.labels) }}
- {{- mustMerge .Values._apiServer.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- if or (.Values.labels) (.Values.apiServer.labels) }}
+ {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent
4 }}
{{- end }}
spec:
selector:
@@ -41,6 +41,6 @@ spec:
tier: airflow
component: api-server
release: {{ .Release.Name }}
- {{- toYaml .Values._apiServer.podDisruptionBudget.config | nindent 2 }}
+ {{- toYaml .Values.apiServer.podDisruptionBudget.config | nindent 2 }}
{{- end }}
{{- end }}
diff --git a/chart/templates/api-server/api-server-service.yaml
b/chart/templates/api-server/api-server-service.yaml
index 71ad37f2ff7..0a360aee085 100644
--- a/chart/templates/api-server/api-server-service.yaml
+++ b/chart/templates/api-server/api-server-service.yaml
@@ -31,29 +31,29 @@ metadata:
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
- {{- if or (.Values.labels) (.Values._apiServer.labels) }}
- {{- mustMerge .Values._apiServer.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- if or (.Values.labels) (.Values.apiServer.labels) }}
+ {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent
4 }}
{{- end }}
- {{- with .Values._apiServer.service.annotations }}
+ {{- with .Values.apiServer.service.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
spec:
- type: {{ .Values._apiServer.service.type }}
+ type: {{ .Values.apiServer.service.type }}
selector:
tier: airflow
component: api-server
release: {{ .Release.Name }}
ports:
- {{ range .Values._apiServer.service.ports }}
+ {{ range .Values.apiServer.service.ports }}
-
{{- range $key, $val := . }}
{{ $key }}: {{ tpl (toString $val) $ }}
{{- end }}
{{- end }}
- {{- if .Values._apiServer.service.loadBalancerIP }}
- loadBalancerIP: {{ .Values._apiServer.service.loadBalancerIP }}
+ {{- if .Values.apiServer.service.loadBalancerIP }}
+ loadBalancerIP: {{ .Values.apiServer.service.loadBalancerIP }}
{{- end }}
- {{- if .Values._apiServer.service.loadBalancerSourceRanges }}
- loadBalancerSourceRanges: {{- toYaml
.Values._apiServer.service.loadBalancerSourceRanges | nindent 4 }}
+ {{- if .Values.apiServer.service.loadBalancerSourceRanges }}
+ loadBalancerSourceRanges: {{- toYaml
.Values.apiServer.service.loadBalancerSourceRanges | nindent 4 }}
{{- end }}
{{- end }}
diff --git a/chart/templates/api-server/api-server-serviceaccount.yaml
b/chart/templates/api-server/api-server-serviceaccount.yaml
index b797a7caadf..3b864d01602 100644
--- a/chart/templates/api-server/api-server-serviceaccount.yaml
+++ b/chart/templates/api-server/api-server-serviceaccount.yaml
@@ -20,22 +20,22 @@
######################################
## Airflow api-server ServiceAccount
######################################
-{{- if and .Values._apiServer.serviceAccount.create (semverCompare ">=3.0.0"
.Values.airflowVersion) }}
+{{- if and .Values.apiServer.serviceAccount.create (semverCompare ">=3.0.0"
.Values.airflowVersion) }}
apiVersion: v1
kind: ServiceAccount
-automountServiceAccountToken: {{
.Values._apiServer.serviceAccount.automountServiceAccountToken }}
+automountServiceAccountToken: {{
.Values.apiServer.serviceAccount.automountServiceAccountToken }}
metadata:
- name: {{ include "_apiServer.serviceAccountName" . }}
+ name: {{ include "apiServer.serviceAccountName" . }}
labels:
tier: airflow
component: api-server
release: {{ .Release.Name }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
heritage: {{ .Release.Service }}
- {{- if or (.Values.labels) (.Values._apiServer.labels) }}
- {{- mustMerge .Values._apiServer.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- if or (.Values.labels) (.Values.apiServer.labels) }}
+ {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent
4 }}
{{- end }}
- {{- with .Values._apiServer.serviceAccount.annotations }}
+ {{- with .Values.apiServer.serviceAccount.annotations }}
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
diff --git a/chart/templates/configmaps/configmap.yaml
b/chart/templates/configmaps/configmap.yaml
index 783881d82dd..de898709321 100644
--- a/chart/templates/configmaps/configmap.yaml
+++ b/chart/templates/configmaps/configmap.yaml
@@ -42,7 +42,7 @@ data:
{{- if semverCompare ">=3.0.0" .Values.airflowVersion -}}
{{- $config := merge .Values.config ( dict "core" dict )}}
{{- if not (hasKey $config.core "execution_api_server_url") -}}
- {{- $_ := set $config.core "execution_api_server_url" (printf
"http://%s-api-server:%d/execution/"; (include "airflow.fullname" .) (int
.Values.ports._apiServer)) -}}
+ {{- $_ := set $config.core "execution_api_server_url" (printf
"http://%s-api-server:%d/execution/"; (include "airflow.fullname" .) (int
.Values.ports.apiServer)) -}}
{{- end -}}
{{- end -}}
# These are system-specified config overrides.
diff --git a/chart/templates/webserver/webserver-deployment.yaml
b/chart/templates/webserver/webserver-deployment.yaml
index dd3c70acf45..e14b13e4582 100644
--- a/chart/templates/webserver/webserver-deployment.yaml
+++ b/chart/templates/webserver/webserver-deployment.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Webserver Deployment
#################################
-{{- if .Values.webserver.enabled }}
+{{- if and .Values.webserver.enabled (semverCompare "<3.0.0"
.Values.airflowVersion) }}
{{- $nodeSelector := or .Values.webserver.nodeSelector .Values.nodeSelector }}
{{- $affinity := or .Values.webserver.affinity .Values.affinity }}
{{- $tolerations := or .Values.webserver.tolerations .Values.tolerations }}
diff --git a/chart/templates/webserver/webserver-hpa.yaml
b/chart/templates/webserver/webserver-hpa.yaml
index dbd1eb611d2..2c4ba1ba0f7 100644
--- a/chart/templates/webserver/webserver-hpa.yaml
+++ b/chart/templates/webserver/webserver-hpa.yaml
@@ -20,7 +20,8 @@
################################
## Airflow Webserver HPA
#################################
-{{- if and (.Values.webserver.enabled) (.Values.webserver.hpa.enabled) }}
+{{- if semverCompare "<3.0.0" .Values.airflowVersion }}
+{{- if and .Values.webserver.enabled .Values.webserver.hpa.enabled }}
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
@@ -47,3 +48,4 @@ spec:
behavior: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
+{{- end }}
diff --git a/chart/templates/webserver/webserver-ingress.yaml
b/chart/templates/webserver/webserver-ingress.yaml
index 6851be45ec1..f65f1841f86 100644
--- a/chart/templates/webserver/webserver-ingress.yaml
+++ b/chart/templates/webserver/webserver-ingress.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Webserver Ingress
#################################
-{{- if .Values.webserver.enabled }}
+{{- if and .Values.webserver.enabled (semverCompare "<3.0.0"
.Values.airflowVersion) }}
{{- if or .Values.ingress.web.enabled .Values.ingress.enabled }}
{{- $fullname := (include "airflow.fullname" .) }}
apiVersion: networking.k8s.io/v1
diff --git a/chart/templates/webserver/webserver-networkpolicy.yaml
b/chart/templates/webserver/webserver-networkpolicy.yaml
index 8945d9c2b62..037132fc166 100644
--- a/chart/templates/webserver/webserver-networkpolicy.yaml
+++ b/chart/templates/webserver/webserver-networkpolicy.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Webserver NetworkPolicy
#################################
-{{- if .Values.webserver.enabled }}
+{{- if and .Values.webserver.enabled (semverCompare "<3.0.0"
.Values.airflowVersion) }}
{{- if .Values.networkPolicies.enabled }}
{{- $from := or .Values.webserver.networkPolicy.ingress.from
.Values.webserver.extraNetworkPolicies }}
apiVersion: networking.k8s.io/v1
diff --git a/chart/templates/webserver/webserver-poddisruptionbudget.yaml
b/chart/templates/webserver/webserver-poddisruptionbudget.yaml
index f5670ca6e74..eaa420d21d2 100644
--- a/chart/templates/webserver/webserver-poddisruptionbudget.yaml
+++ b/chart/templates/webserver/webserver-poddisruptionbudget.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Webserver PodDisruptionBudget
#################################
-{{- if .Values.webserver.enabled }}
+{{- if and .Values.webserver.enabled (semverCompare "<3.0.0"
.Values.airflowVersion) }}
{{- if .Values.webserver.podDisruptionBudget.enabled }}
apiVersion: policy/v1
kind: PodDisruptionBudget
diff --git a/chart/templates/webserver/webserver-service.yaml
b/chart/templates/webserver/webserver-service.yaml
index 844c7dca393..fab8d281fcb 100644
--- a/chart/templates/webserver/webserver-service.yaml
+++ b/chart/templates/webserver/webserver-service.yaml
@@ -20,7 +20,7 @@
################################
## Airflow Webserver Service
#################################
-{{- if .Values.webserver.enabled }}
+{{- if and .Values.webserver.enabled (semverCompare "<3.0.0"
.Values.airflowVersion) }}
apiVersion: v1
kind: Service
metadata:
diff --git a/chart/templates/webserver/webserver-serviceaccount.yaml
b/chart/templates/webserver/webserver-serviceaccount.yaml
index a971da6e327..8bd392adfa1 100644
--- a/chart/templates/webserver/webserver-serviceaccount.yaml
+++ b/chart/templates/webserver/webserver-serviceaccount.yaml
@@ -20,6 +20,7 @@
######################################
## Airflow Webserver ServiceAccount
######################################
+{{- if semverCompare "<3.0.0" .Values.airflowVersion }}
{{- if and .Values.webserver.enabled .Values.webserver.serviceAccount.create }}
apiVersion: v1
kind: ServiceAccount
@@ -39,3 +40,4 @@ metadata:
annotations: {{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}
+{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index 1586e767916..e25b0a11a79 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -4684,8 +4684,8 @@
}
}
},
- "_apiServer": {
- "description": "Airflow API server settings. Experimental / for
dev purpose only.",
+ "apiServer": {
+ "description": "Airflow API server settings. Airflow 3+ only.",
"type": "object",
"x-docsSection": "API Server",
"additionalProperties": false,
@@ -4957,7 +4957,7 @@
},
"default": [
{
- "port": "{{
.Values.ports._apiServer }}"
+ "port": "{{
.Values.ports.apiServer }}"
}
],
"examples": [
@@ -5184,7 +5184,7 @@
"default": [
{
"name": "api-server",
- "port": "{{ .Values.ports._apiServer }}"
+ "port": "{{ .Values.ports.apiServer }}"
}
],
"examples": [
@@ -5367,7 +5367,7 @@
}
},
"webserver": {
- "description": "Airflow webserver settings.",
+ "description": "Airflow webserver settings. Airflow 2 only.",
"type": "object",
"x-docsSection": "Webserver",
"additionalProperties": false,
@@ -8183,7 +8183,7 @@
"type": "integer",
"default": 8080
},
- "_apiServer": {
+ "apiServer": {
"description": "API server port.",
"type": "integer",
"default": 9091
diff --git a/chart/values.yaml b/chart/values.yaml
index 070d7c3c68c..f326b1a8266 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -1253,7 +1253,7 @@ migrateDatabaseJob:
applyCustomEnv: true
env: []
-_apiServer:
+apiServer:
# Labels specific to workers objects and pods
labels: {}
@@ -1281,7 +1281,7 @@ _apiServer:
annotations: {}
ports:
- name: api-server
- port: "{{ .Values.ports._apiServer }}"
+ port: "{{ .Values.ports.apiServer }}"
loadBalancerIP: ~
## Limit load balancer source ips to list of CIDRs
@@ -1320,7 +1320,7 @@ _apiServer:
from: []
# Ports for webserver NetworkPolicy ingress (if `from` is set)
ports:
- - port: "{{ .Values.ports._apiServer }}"
+ - port: "{{ .Values.ports.apiServer }}"
resources: {}
# limits:
@@ -2540,7 +2540,7 @@ ports:
statsdScrape: 9102
pgbouncer: 6543
pgbouncerScrape: 9127
- _apiServer: 9091
+ apiServer: 9091
# Define any ResourceQuotas for namespace
quotas: {}
diff --git a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
index a7780debb2f..47db0747722 100644
--- a/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/kubernetes_utils.py
@@ -476,10 +476,8 @@ def get_k8s_env(python: str, kubernetes_version: str,
executor: str | None = Non
new_env["KINDCONFIG"] = str(
get_kind_cluster_config_path(python=python,
kubernetes_version=kubernetes_version)
)
- api_server_port, web_server_port = _get_kubernetes_port_numbers(
- python=python, kubernetes_version=kubernetes_version
- )
- new_env["CLUSTER_FORWARDED_PORT"] = str(web_server_port)
+ _, api_server_port = _get_kubernetes_port_numbers(python=python,
kubernetes_version=kubernetes_version)
+ new_env["CLUSTER_FORWARDED_PORT"] = str(api_server_port)
kubectl_cluster_name = get_kubectl_cluster_name(python=python,
kubernetes_version=kubernetes_version)
if executor:
new_env["PS1"] = f"({kubectl_cluster_name}:{executor})> "
@@ -524,16 +522,16 @@ def set_random_cluster_ports(python: str,
kubernetes_version: str, output: Outpu
The sockets should be closed just before creating the cluster.
"""
forwarded_port_number = _get_free_port()
- api_server_port = _get_free_port()
+ k8s_api_server_port = _get_free_port()
get_console(output=output).print(
- f"[info]Random ports: API: {api_server_port}, Web:
{forwarded_port_number}"
+ f"[info]Random ports: K8S API: {k8s_api_server_port}, API Server:
{forwarded_port_number}"
)
cluster_conf_path = get_kind_cluster_config_path(python=python,
kubernetes_version=kubernetes_version)
config = (
(AIRFLOW_SOURCES_ROOT / "scripts" / "ci" / "kubernetes" /
"kind-cluster-conf.yaml")
.read_text()
.replace("{{FORWARDED_PORT_NUMBER}}", str(forwarded_port_number))
- .replace("{{API_SERVER_PORT}}", str(api_server_port))
+ .replace("{{API_SERVER_PORT}}", str(k8s_api_server_port))
)
cluster_conf_path.write_text(config)
get_console(output=output).print(f"[info]Config created in
{cluster_conf_path}:\n")
@@ -545,9 +543,9 @@ def _get_kubernetes_port_numbers(python: str,
kubernetes_version: str) -> tuple[
conf = _get_kind_cluster_config_content(python=python,
kubernetes_version=kubernetes_version)
if conf is None:
return 0, 0
- api_server_port = conf["networking"]["apiServerPort"]
- web_server_port = conf["nodes"][1]["extraPortMappings"][0]["hostPort"]
- return api_server_port, web_server_port
+ k8s_api_server_port = conf["networking"]["apiServerPort"]
+ api_server_port = conf["nodes"][1]["extraPortMappings"][0]["hostPort"]
+ return k8s_api_server_port, api_server_port
def _attempt_to_connect(port_number: int, output: Output | None, wait_seconds:
int = 0) -> bool:
@@ -558,18 +556,18 @@ def _attempt_to_connect(port_number: int, output: Output
| None, wait_seconds: i
for attempt in itertools.count(1):
get_console(output=output).print(f"[info]Connecting to
localhost:{port_number}. Num try: {attempt}")
try:
- response = requests.head(f"http://localhost:{port_number}/health";)
+ response =
requests.get(f"http://localhost:{port_number}/public/monitor/health";)
except ConnectionError:
get_console(output=output).print(
- f"The webserver is not yet ready at
http://localhost:{port_number}/health "
+ f"The api server is not yet ready at
http://localhost:{port_number}/public/monitor/health "
)
except Exception as e:
get_console(output=output).print(f"[info]Error when connecting to
localhost:{port_number} : {e}")
else:
if response.status_code == 200:
get_console(output=output).print(
- "[success]Established connection to webserver at "
- f"http://localhost:{port_number}/health and it is healthy."
+ "[success]Established connection to api server at "
+ f"http://localhost:{port_number}/public/monitor/health and
it is healthy."
)
return True
else:
@@ -590,19 +588,19 @@ def _attempt_to_connect(port_number: int, output: Output
| None, wait_seconds: i
def print_cluster_urls(
python: str, kubernetes_version: str, output: Output | None,
wait_time_in_seconds: int = 0
):
- api_server_port, web_server_port = _get_kubernetes_port_numbers(
+ k8s_api_server_port, api_server_port = _get_kubernetes_port_numbers(
python=python, kubernetes_version=kubernetes_version
)
get_console(output=output).print(
- f"\n[info]KinD Cluster API server URL:
[/]http://localhost:{api_server_port}";
+ f"\n[info]KinD Cluster API server URL:
[/]http://localhost:{k8s_api_server_port}";
)
- if _attempt_to_connect(port_number=web_server_port, output=output,
wait_seconds=wait_time_in_seconds):
+ if _attempt_to_connect(port_number=api_server_port, output=output,
wait_seconds=wait_time_in_seconds):
get_console(output=output).print(
- f"[info]Airflow Web server URL:
[/]http://localhost:{web_server_port} (admin/admin)\n"
+ f"[info]Airflow API server URL:
[/]http://localhost:{api_server_port} (admin/admin)\n"
)
else:
get_console(output=output).print(
- f"\n[warning]Airflow webserver is not available at port
{web_server_port}. "
+ f"\n[warning]Airflow API server is not available at port
{api_server_port}. "
f"Run `breeze k8s deploy-airflow --python {python}
--kubernetes-version {kubernetes_version}` "
"to (re)deploy airflow\n"
)
diff --git a/helm_tests/airflow_aux/test_basic_helm_chart.py
b/helm_tests/airflow_aux/test_basic_helm_chart.py
index 52ff7facf4b..4e6185e2db0 100644
--- a/helm_tests/airflow_aux/test_basic_helm_chart.py
+++ b/helm_tests/airflow_aux/test_basic_helm_chart.py
@@ -24,6 +24,7 @@ from unittest import mock
import jmespath
import pytest
+from packaging.version import parse as parse_version
from tests.charts.helm_template_generator import render_chart
@@ -107,7 +108,6 @@ class TestBaseChartTest:
("ServiceAccount", "test-basic-scheduler"),
("ServiceAccount", "test-basic-statsd"),
("ServiceAccount", "test-basic-triggerer"),
- ("ServiceAccount", "test-basic-webserver"),
("ServiceAccount", "test-basic-worker"),
("Secret", "test-basic-metadata"),
("Secret", "test-basic-broker-url"),
@@ -125,12 +125,10 @@ class TestBaseChartTest:
("Service", "test-basic-postgresql"),
("Service", "test-basic-redis"),
("Service", "test-basic-statsd"),
- ("Service", "test-basic-webserver"),
("Service", "test-basic-worker"),
("Deployment", "test-basic-scheduler"),
("Deployment", "test-basic-statsd"),
(self.default_trigger_obj(version), "test-basic-triggerer"),
- ("Deployment", "test-basic-webserver"),
("StatefulSet", "test-basic-postgresql"),
("StatefulSet", "test-basic-redis"),
("StatefulSet", "test-basic-worker"),
@@ -139,7 +137,7 @@ class TestBaseChartTest:
}
if version == "2.3.2":
expected.add(("Secret", "test-basic-result-backend"))
- if version == "3.0.0":
+ if version != "default" and parse_version(version) >=
parse_version("3.0.0"):
expected.update(
(
("Deployment", "test-basic-api-server"),
@@ -150,7 +148,15 @@ class TestBaseChartTest:
("Service", "test-basic-triggerer"),
)
)
- elif version == "default":
+ else:
+ expected.update(
+ (
+ ("Deployment", "test-basic-webserver"),
+ ("Service", "test-basic-webserver"),
+ ("ServiceAccount", "test-basic-webserver"),
+ )
+ )
+ if version == "default":
expected.add(("Service", "test-basic-triggerer"))
assert list_of_kind_names_tuples == expected
assert len(k8s_objects) == len(expected)
diff --git a/helm_tests/airflow_core/test_api_server.py
b/helm_tests/airflow_core/test_api_server.py
index f78630dfef4..08fba280046 100644
--- a/helm_tests/airflow_core/test_api_server.py
+++ b/helm_tests/airflow_core/test_api_server.py
@@ -39,9 +39,9 @@ class TestAPIServerDeployment:
[(8, 10), (10, 8), (8, None), (None, 10), (None, None)],
)
def test_revision_history_limit(self, revision_history_limit,
global_revision_history_limit):
- values = {"_apiServer": {}}
+ values = {"apiServer": {}}
if revision_history_limit:
- values["_apiServer"]["revisionHistoryLimit"] =
revision_history_limit
+ values["apiServer"]["revisionHistoryLimit"] =
revision_history_limit
if global_revision_history_limit:
values["revisionHistoryLimit"] = global_revision_history_limit
docs = render_chart(
@@ -54,7 +54,7 @@ class TestAPIServerDeployment:
def
test_should_add_scheme_to_liveness_and_readiness_and_startup_probes(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"livenessProbe": {"scheme": "HTTPS"},
"readinessProbe": {"scheme": "HTTPS"},
"startupProbe": {"scheme": "HTTPS"},
@@ -77,7 +77,7 @@ class TestAPIServerDeployment:
docs = render_chart(
values={
"executor": "CeleryExecutor",
- "_apiServer": {
+ "apiServer": {
"extraContainers": [
{"name": "{{.Chart.Name}}", "image":
"test-registry/test-repo:test-tag"}
],
@@ -94,7 +94,7 @@ class TestAPIServerDeployment:
def test_should_add_extraEnvs(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
},
},
@@ -108,7 +108,7 @@ class TestAPIServerDeployment:
def test_should_add_extra_volume_and_extra_volume_mount(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}",
"emptyDir": {}}],
"extraVolumeMounts": [
{"name": "test-volume-{{ .Chart.Name }}", "mountPath":
"/opt/test"}
@@ -146,7 +146,7 @@ class TestAPIServerDeployment:
def test_should_add_extraEnvs_to_wait_for_migration_container(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"waitForMigrations": {
"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}],
},
@@ -171,7 +171,7 @@ class TestAPIServerDeployment:
def test_disable_wait_for_migration(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"waitForMigrations": {"enabled": False},
},
},
@@ -185,7 +185,7 @@ class TestAPIServerDeployment:
def test_should_add_extra_init_containers(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"extraInitContainers": [
{"name": "test-init-container", "image":
"test-registry/test-repo:test-tag"}
],
@@ -202,7 +202,7 @@ class TestAPIServerDeployment:
def test_should_add_component_specific_labels(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"labels": {"test_label": "test_label_value"},
},
},
@@ -215,7 +215,7 @@ class TestAPIServerDeployment:
def test_should_create_valid_affinity_tolerations_and_node_selector(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"affinity": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
@@ -300,7 +300,7 @@ class TestAPIServerDeployment:
}
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"affinity": expected_affinity,
"tolerations": [
{"key": "dynamic-pods", "operator": "Equal", "value":
"true", "effect": "NoSchedule"}
@@ -412,7 +412,7 @@ class TestAPIServerDeployment:
def testapi_server_resources_are_configurable(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"resources": {
"limits": {"cpu": "200m", "memory": "128Mi"},
"requests": {"cpu": "300m", "memory": "169Mi"},
@@ -447,7 +447,7 @@ class TestAPIServerDeployment:
def test_api_server_security_contexts_are_configurable(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"securityContexts": {
"pod": {
"fsGroup": 1000,
@@ -480,7 +480,7 @@ class TestAPIServerDeployment:
with pytest.raises(CalledProcessError, match="Additional property
securityContext is not allowed"):
render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"securityContext": {
"fsGroup": 1000,
"runAsGroup": 1001,
@@ -518,7 +518,7 @@ class TestAPIServerDeployment:
)
def test_update_strategy(self, airflow_version, strategy,
expected_strategy):
docs = render_chart(
- values={"airflowVersion": airflow_version, "_apiServer":
{"strategy": expected_strategy}},
+ values={"airflowVersion": airflow_version, "apiServer":
{"strategy": expected_strategy}},
show_only=["templates/api-server/api-server-deployment.yaml"],
)
@@ -540,7 +540,7 @@ class TestAPIServerDeployment:
@pytest.mark.parametrize("args", [None, ["custom", "args"]])
def test_command_and_args_overrides(self, command, args):
docs = render_chart(
- values={"_apiServer": {"command": command, "args": args}},
+ values={"apiServer": {"command": command, "args": args}},
show_only=["templates/api-server/api-server-deployment.yaml"],
)
@@ -550,7 +550,7 @@ class TestAPIServerDeployment:
def test_command_and_args_overrides_are_templated(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"command": ["{{ .Release.Name }}"],
"args": ["{{ .Release.Service }}"],
}
@@ -563,7 +563,7 @@ class TestAPIServerDeployment:
def test_should_add_component_specific_annotations(self):
docs = render_chart(
- values={"_apiServer": {"annotations": {"test_annotation":
"test_annotation_value"}}},
+ values={"apiServer": {"annotations": {"test_annotation":
"test_annotation_value"}}},
show_only=["templates/api-server/api-server-deployment.yaml"],
)
assert "annotations" in jmespath.search("metadata", docs[0])
@@ -571,7 +571,7 @@ class TestAPIServerDeployment:
def test_api_server_pod_hostaliases(self):
docs = render_chart(
- values={"_apiServer": {"hostAliases": [{"ip": "127.0.0.1",
"hostnames": ["foo.local"]}]}},
+ values={"apiServer": {"hostAliases": [{"ip": "127.0.0.1",
"hostnames": ["foo.local"]}]}},
show_only=["templates/api-server/api-server-deployment.yaml"],
)
@@ -600,8 +600,8 @@ class TestAPIServerService:
def test_overrides(self):
docs = render_chart(
values={
- "ports": {"_apiServer": 9000},
- "_apiServer": {
+ "ports": {"apiServer": 9000},
+ "apiServer": {
"service": {
"type": "LoadBalancer",
"loadBalancerIP": "127.0.0.1",
@@ -628,7 +628,7 @@ class TestAPIServerService:
{
"name": "{{ .Release.Name }}",
"protocol": "UDP",
- "port": "{{ .Values.ports._apiServer }}",
+ "port": "{{ .Values.ports.apiServer }}",
}
],
[{"name": "release-name", "protocol": "UDP", "port": 9091}],
@@ -636,7 +636,7 @@ class TestAPIServerService:
([{"name": "only_sidecar", "port": "{{ int 9000 }}"}], [{"name":
"only_sidecar", "port": 9000}]),
(
[
- {"name": "api-server", "port": "{{
.Values.ports._apiServer }}"},
+ {"name": "api-server", "port": "{{ .Values.ports.apiServer
}}"},
{"name": "sidecar", "port": 80, "targetPort": "sidecar"},
],
[
@@ -648,7 +648,7 @@ class TestAPIServerService:
)
def test_ports_overrides(self, ports, expected_ports):
docs = render_chart(
- values={"_apiServer": {"service": {"ports": ports}}},
+ values={"apiServer": {"service": {"ports": ports}}},
show_only=["templates/api-server/api-server-service.yaml"],
)
@@ -656,7 +656,7 @@ class TestAPIServerService:
def test_should_add_component_specific_labels(self):
docs = render_chart(
- values={"_apiServer": {"labels": {"test_label":
"test_label_value"}}},
+ values={"apiServer": {"labels": {"test_label":
"test_label_value"}}},
show_only=["templates/api-server/api-server-service.yaml"],
)
assert "test_label" in jmespath.search("metadata.labels", docs[0])
@@ -677,7 +677,7 @@ class TestAPIServerService:
)
def test_nodeport_service(self, ports, expected_ports):
docs = render_chart(
- values={"_apiServer": {"service": {"type": "NodePort", "ports":
ports}}},
+ values={"apiServer": {"service": {"type": "NodePort", "ports":
ports}}},
show_only=["templates/api-server/api-server-service.yaml"],
)
@@ -698,7 +698,7 @@ class TestAPIServerNetworkPolicy:
docs = render_chart(
values={
"networkPolicies": {"enabled": True},
- "_apiServer": {
+ "apiServer": {
"networkPolicy": {
"ingress": {
"from": [{"namespaceSelector": {"matchLabels":
{"release": "myrelease"}}}]
@@ -722,7 +722,7 @@ class TestAPIServerNetworkPolicy:
([{"port": "sidecar"}], [{"port": "sidecar"}]),
(
[
- {"port": "{{ .Values.ports._apiServer }}"},
+ {"port": "{{ .Values.ports.apiServer }}"},
{"port": 80},
],
[
@@ -736,7 +736,7 @@ class TestAPIServerNetworkPolicy:
docs = render_chart(
values={
"networkPolicies": {"enabled": True},
- "_apiServer": {
+ "apiServer": {
"networkPolicy": {
"ingress": {
"from": [{"namespaceSelector": {"matchLabels":
{"release": "myrelease"}}}],
@@ -754,7 +754,7 @@ class TestAPIServerNetworkPolicy:
docs = render_chart(
values={
"networkPolicies": {"enabled": True},
- "_apiServer": {
+ "apiServer": {
"labels": {"test_label": "test_label_value"},
},
},
@@ -770,7 +770,7 @@ class TestAPIServerServiceAccount:
def test_should_add_component_specific_labels(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"serviceAccount": {"create": True},
"labels": {"test_label": "test_label_value"},
},
@@ -783,7 +783,7 @@ class TestAPIServerServiceAccount:
def test_default_automount_service_account_token(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"serviceAccount": {"create": True},
},
},
@@ -794,7 +794,7 @@ class TestAPIServerServiceAccount:
def test_overridden_automount_service_account_token(self):
docs = render_chart(
values={
- "_apiServer": {
+ "apiServer": {
"serviceAccount": {"create": True,
"automountServiceAccountToken": False},
},
},
diff --git a/helm_tests/security/test_rbac.py b/helm_tests/security/test_rbac.py
index c494ebc7ee6..13938e12faa 100644
--- a/helm_tests/security/test_rbac.py
+++ b/helm_tests/security/test_rbac.py
@@ -20,6 +20,7 @@ from copy import copy
import jmespath
import pytest
+from packaging.version import parse as parse_version
from tests.charts.helm_template_generator import render_chart
@@ -33,14 +34,12 @@ DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES = [
("Service", "test-rbac-postgresql-hl"),
("Service", "test-rbac-postgresql"),
("Service", "test-rbac-statsd"),
- ("Service", "test-rbac-webserver"),
("Service", "test-rbac-flower"),
("Service", "test-rbac-pgbouncer"),
("Service", "test-rbac-redis"),
("Service", "test-rbac-worker"),
("Deployment", "test-rbac-scheduler"),
("Deployment", "test-rbac-statsd"),
- ("Deployment", "test-rbac-webserver"),
("Deployment", "test-rbac-flower"),
("Deployment", "test-rbac-pgbouncer"),
("StatefulSet", "test-rbac-postgresql"),
@@ -67,7 +66,6 @@ RBAC_ENABLED_KIND_NAME_TUPLES = [
SERVICE_ACCOUNT_NAME_TUPLES = [
("ServiceAccount", "test-rbac-cleanup"),
("ServiceAccount", "test-rbac-scheduler"),
- ("ServiceAccount", "test-rbac-webserver"),
("ServiceAccount", "test-rbac-worker"),
("ServiceAccount", "test-rbac-triggerer"),
("ServiceAccount", "test-rbac-pgbouncer"),
@@ -81,7 +79,6 @@ SERVICE_ACCOUNT_NAME_TUPLES = [
CUSTOM_SERVICE_ACCOUNT_NAMES = (
(CUSTOM_SCHEDULER_NAME := "TestScheduler"),
(CUSTOM_DAG_PROCESSOR_NAME := "TestDagProcessor"),
- (CUSTOM_WEBSERVER_NAME := "TestWebserver"),
(CUSTOM_API_SERVER_NAME := "TestAPISserver"),
(CUSTOM_WORKER_NAME := "TestWorker"),
(CUSTOM_TRIGGERER_NAME := "TestTriggerer"),
@@ -94,6 +91,7 @@ CUSTOM_SERVICE_ACCOUNT_NAMES = (
(CUSTOM_REDIS_NAME := "TestRedis"),
(CUSTOM_POSTGRESQL_NAME := "TestPostgresql"),
)
+CUSTOM_WEBSERVER_NAME = "TestWebserver"
parametrize_version = pytest.mark.parametrize("version", ["2.3.2", "2.4.0",
"3.0.0", "default"])
@@ -116,7 +114,7 @@ class TestRBAC:
tuples.append(("Deployment", "test-rbac-triggerer"))
if version == "2.3.2":
tuples.append(("Secret", "test-rbac-result-backend"))
- if version == "3.0.0":
+ if version != "default" and parse_version(version) >=
parse_version("3.0.0"):
tuples.extend(
(
("Service", "test-rbac-api-server"),
@@ -127,6 +125,16 @@ class TestRBAC:
if sa:
tuples.append(("ServiceAccount", "test-rbac-api-server"))
tuples.append(("ServiceAccount", "test-rbac-dag-processor"))
+ else:
+ tuples.extend(
+ (
+ ("Service", "test-rbac-webserver"),
+ ("Deployment", "test-rbac-webserver"),
+ )
+ )
+ if sa:
+ tuples.append(("ServiceAccount", "test-rbac-webserver"))
+
return tuples
@parametrize_version
@@ -153,7 +161,7 @@ class TestRBAC:
"scheduler": {"serviceAccount": {"create": False}},
"dagProcessor": {"serviceAccount": {"create": False}},
"webserver": {"serviceAccount": {"create": False}},
- "_apiServer": {"serviceAccount": {"create": False}},
+ "apiServer": {"serviceAccount": {"create": False}},
"workers": {"serviceAccount": {"create": False}},
"triggerer": {"serviceAccount": {"create": False}},
"statsd": {"serviceAccount": {"create": False}},
@@ -206,7 +214,7 @@ class TestRBAC:
"scheduler": {"serviceAccount": {"create": False}},
"dagProcessor": {"serviceAccount": {"create": False}},
"webserver": {"serviceAccount": {"create": False}},
- "_apiServer": {"serviceAccount": {"create": False}},
+ "apiServer": {"serviceAccount": {"create": False}},
"workers": {"serviceAccount": {"create": False}},
"triggerer": {"serviceAccount": {"create": False}},
"flower": {"enabled": True, "serviceAccount": {"create":
False}},
@@ -266,8 +274,7 @@ class TestRBAC:
},
"scheduler": {"serviceAccount": {"name":
CUSTOM_SCHEDULER_NAME}},
"dagProcessor": {"serviceAccount": {"name":
CUSTOM_DAG_PROCESSOR_NAME}},
- "webserver": {"serviceAccount": {"name":
CUSTOM_WEBSERVER_NAME}},
- "_apiServer": {"serviceAccount": {"name":
CUSTOM_API_SERVER_NAME}},
+ "apiServer": {"serviceAccount": {"name":
CUSTOM_API_SERVER_NAME}},
"workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}},
"triggerer": {"serviceAccount": {"name":
CUSTOM_TRIGGERER_NAME}},
"flower": {"enabled": True, "serviceAccount": {"name":
CUSTOM_FLOWER_NAME}},
@@ -291,6 +298,19 @@ class TestRBAC:
]
assert sorted(list_of_sa_names) == sorted(CUSTOM_SERVICE_ACCOUNT_NAMES)
+ def test_webserver_service_account_name_airflow_2(self):
+ k8s_objects = render_chart(
+ "test-rbac",
+ values={
+ "airflowVersion": "2.10.5",
+ "fullnameOverride": "test-rbac",
+ "webserver": {"serviceAccount": {"name":
CUSTOM_WEBSERVER_NAME}},
+ },
+ show_only=["templates/webserver/webserver-serviceaccount.yaml"],
+ )
+ sa_name = jmespath.search("metadata.name", k8s_objects[0])
+ assert sa_name == CUSTOM_WEBSERVER_NAME
+
def test_service_account_custom_names_in_objects(self):
k8s_objects = render_chart(
"test-rbac",
@@ -305,8 +325,7 @@ class TestRBAC:
},
"scheduler": {"serviceAccount": {"name":
CUSTOM_SCHEDULER_NAME}},
"dagProcessor": {"serviceAccount": {"name":
CUSTOM_DAG_PROCESSOR_NAME}},
- "webserver": {"serviceAccount": {"name":
CUSTOM_WEBSERVER_NAME}},
- "_apiServer": {"serviceAccount": {"name":
CUSTOM_API_SERVER_NAME}},
+ "apiServer": {"serviceAccount": {"name":
CUSTOM_API_SERVER_NAME}},
"workers": {"serviceAccount": {"name": CUSTOM_WORKER_NAME}},
"triggerer": {"serviceAccount": {"name":
CUSTOM_TRIGGERER_NAME}},
"flower": {"enabled": True, "serviceAccount": {"name":
CUSTOM_FLOWER_NAME}},
@@ -361,7 +380,6 @@ class TestRBAC:
service_account_names = [
"test-rbac-scheduler",
"test-rbac-dag-processor",
- "test-rbac-webserver",
"test-rbac-api-server",
"test-rbac-triggerer",
"test-rbac-migrate-database-job",
diff --git a/kubernetes_tests/test_base.py b/kubernetes_tests/test_base.py
index 2e0a96620a1..34c0871ff97 100644
--- a/kubernetes_tests/test_base.py
+++ b/kubernetes_tests/test_base.py
@@ -33,7 +33,7 @@ from requests.exceptions import RetryError
from urllib3.exceptions import MaxRetryError
from urllib3.util.retry import Retry
-CLUSTER_FORWARDED_PORT = os.environ.get("CLUSTER_FORWARDED_PORT") or "8080"
+CLUSTER_FORWARDED_PORT = os.environ.get("CLUSTER_FORWARDED_PORT") or "9091"
KUBERNETES_HOST_PORT = (os.environ.get("CLUSTER_HOST") or "localhost") + ":" +
CLUSTER_FORWARDED_PORT
EXECUTOR = os.environ.get("EXECUTOR")
@@ -62,7 +62,7 @@ class BaseK8STest:
self.test_id = f"{request.node.cls.__name__}_{request.node.name}"
self.session = self._get_session_with_retries()
try:
- self._ensure_airflow_webserver_is_healthy()
+ self._ensure_airflow_api_server_is_healthy()
yield
finally:
self.session.close()
@@ -135,25 +135,25 @@ class BaseK8STest:
session.mount("https://";, HTTPAdapter(max_retries=retries))
return session
- def _ensure_airflow_webserver_is_healthy(self):
+ def _ensure_airflow_api_server_is_healthy(self):
max_tries = 10
timeout_seconds = 5
for i in range(max_tries):
try:
response = self.session.get(
- f"http://{KUBERNETES_HOST_PORT}/health";,
+ f"http://{KUBERNETES_HOST_PORT}/public/monitor/health";,
timeout=1,
)
if response.status_code == 200:
- print("Airflow webserver is healthy!")
+ print("Airflow api server is healthy!")
return
except Exception as e:
- print(f"Exception when checking if webserver is healthy {e}")
+ print(f"Exception when checking if api server is healthy {e}")
if i < max_tries - 1:
print(f"Waiting {timeout_seconds} s and retrying.")
time.sleep(timeout_seconds)
raise Exception(
- f"Giving up. The webserver of Airflow was not healthy after
{max_tries} tries "
+ f"Giving up. The api server of Airflow was not healthy after
{max_tries} tries "
f"with {timeout_seconds} s delays"
)
@@ -167,7 +167,7 @@ class BaseK8STest:
# Check task state
try:
get_string = (
-
f"http://{host}/api/v1/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id}";
+
f"http://{host}/public/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id}";
)
print(f"Calling [monitor_task]#1 {get_string}")
result = self.session.get(get_string)
@@ -211,7 +211,7 @@ class BaseK8STest:
# Wait some time for the operator to complete
while tries < max_tries:
time.sleep(5)
- get_string = f"http://{host}/api/v1/dags/{dag_id}/dagRuns";
+ get_string = f"http://{host}/public/dags/{dag_id}/dagRuns";
print(f"Calling {get_string}")
# Get all dagruns
result = self.session.get(get_string)
@@ -238,7 +238,7 @@ class BaseK8STest:
# Maybe check if we can retrieve the logs, but then we need to extend
the API
def start_dag(self, dag_id, host):
- patch_string = f"http://{host}/api/v1/dags/{dag_id}";
+ patch_string = f"http://{host}/public/dags/{dag_id}";
print(f"Calling [start_dag]#1 {patch_string}")
max_attempts = 10
result = {}
@@ -262,7 +262,7 @@ class BaseK8STest:
result_json = str(result)
print(f"Received [start_dag]#1 {result_json}")
assert result.status_code == 200, f"Could not enable DAG:
{result_json}"
- post_string = f"http://{host}/api/v1/dags/{dag_id}/dagRuns";
+ post_string = f"http://{host}/public/dags/{dag_id}/dagRuns";
print(f"Calling [start_dag]#2 {post_string}")
logical_date = datetime.now(timezone.utc).isoformat()
@@ -277,7 +277,7 @@ class BaseK8STest:
time.sleep(1)
- get_string = f"http://{host}/api/v1/dags/{dag_id}/dagRuns";
+ get_string = f"http://{host}/public/dags/{dag_id}/dagRuns";
print(f"Calling [start_dag]#3 {get_string}")
result = self.session.get(get_string)
assert result.status_code == 200, f"Could not get DAGRuns:
{result.json()}"
diff --git a/scripts/ci/kubernetes/nodeport.yaml
b/scripts/ci/kubernetes/nodeport.yaml
index 84382814d90..79555c188f6 100644
--- a/scripts/ci/kubernetes/nodeport.yaml
+++ b/scripts/ci/kubernetes/nodeport.yaml
@@ -22,10 +22,10 @@ metadata:
spec:
type: NodePort
selector:
- component: webserver
+ component: api-server
release: airflow
tier: airflow
ports:
- - port: 8080
- targetPort: 8080
+ - port: 9091
+ targetPort: 9091
nodePort: 30007
