This is an automated email from the ASF dual-hosted git repository.
jscheffl pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 535e3cca49d Add optional OTel service to the Airflow Helm Chart
(#64902)
535e3cca49d is described below
commit 535e3cca49dd00287cac675c6f0ea664d5549648
Author: Christos Bisias <[email protected]>
AuthorDate: Tue May 5 21:58:22 2026 +0300
Add optional OTel service to the Airflow Helm Chart (#64902)
* add otel to helm chart
* use Kustomize for grafana, jaeger, prometheus
* enable specific service per flag + unit test
* remove grafana, jaeger and prometheus kustomization logic
* traces enabled and metrics disabled, by default
* remove otelCollector.enabled flag
* add statsd comments about otel metrics overriding the config
* make OTEL_METRIC_EXPORT_INTERVAL configurable and provide default value +
entry in the values.schema.json
* remove hardcoded value for metrics otel_port in values.yaml
* add option to override the configmap
* add otelCollector.args and make the config.yml file as the default
argument
* rename extraAnnotations to annotations in otel-collector-service.yaml
* parameterize the readiness and liveness probe values
* remove prometheus from the configmap
* update the default value for OTEL_TRACES_EXPORTER
* fix tests in airflow_aux + otel-collector-serviceaccount.yaml
* fix spellcheck errors in docs
* fix tests in security
* otel collector unit tests + networkpolicy file
* values.schema.json cleanup
* add a minimum to all integer configs in values.schema.json
* fix heading comments
* change config default to ~ from empty string
* fix static check error
---
chart/templates/_helpers.yaml | 27 ++
.../configmaps/otel-collector-configmap.yaml | 73 ++++
.../otel-collector/otel-collector-deployment.yaml | 115 +++++
.../otel-collector-networkpolicy.yaml | 59 +++
.../otel-collector/otel-collector-service.yaml | 55 +++
.../otel-collector-serviceaccount.yaml | 41 ++
chart/values.schema.json | 292 ++++++++++++-
chart/values.yaml | 121 +++++-
.../doc/images/output_testing_helm-tests.svg | 2 +-
.../doc/images/output_testing_helm-tests.txt | 2 +-
.../helm_tests/airflow_aux/test_airflow_common.py | 12 +
.../airflow_aux/test_basic_helm_chart.py | 9 +
.../tests/helm_tests/otel_collector/__init__.py | 16 +
.../otel_collector/test_labels_deployment.py | 113 +++++
.../otel_collector/test_labels_networkpolicy.py | 98 +++++
.../otel_collector/test_labels_service.py | 94 ++++
.../otel_collector/test_labels_serviceaccount.py | 94 ++++
.../otel_collector/test_otel_collector.py | 484 +++++++++++++++++++++
helm-tests/tests/helm_tests/security/test_rbac.py | 10 +
19 files changed, 1713 insertions(+), 4 deletions(-)
diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml
index b4ca3866720..d2aa1e093e4 100644
--- a/chart/templates/_helpers.yaml
+++ b/chart/templates/_helpers.yaml
@@ -141,6 +141,24 @@ If release name contains chart name it will be used as a
full name.
name: {{ template "opensearch_secret" . }}
key: connection
{{- end }}
+ {{- if or .Values.otelCollector.tracesEnabled
.Values.otelCollector.metricsEnabled }}
+ - name: OTEL_SERVICE_NAME
+ value: "airflow"
+ - name: OTEL_EXPORTER_OTLP_PROTOCOL
+ value: "http/protobuf"
+ {{- end }}
+ {{- if .Values.otelCollector.tracesEnabled }}
+ - name: OTEL_TRACES_EXPORTER
+ value: "otlp"
+ - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
+ value: "http://{{ include "airflow.fullname" . }}-otel-collector:{{
.Values.ports.otelCollectorOtlpHttp }}/v1/traces"
+ {{- end }}
+ {{- if .Values.otelCollector.metricsEnabled }}
+ - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT
+ value: "http://{{ include "airflow.fullname" . }}-otel-collector:{{
.Values.ports.otelCollectorOtlpHttp }}/v1/metrics"
+ - name: OTEL_METRIC_EXPORT_INTERVAL
+ value: {{ .Values.otelCollector.metricExportIntervalMs | quote }}
+ {{- end }}
{{- end }}
{{/* User defined Airflow environment variables */}}
@@ -404,6 +422,10 @@ If release name contains chart name it will be used as a
full name.
{{- printf "%s:%s" .Values.images.gitSync.repository
.Values.images.gitSync.tag }}
{{- end }}
+{{- define "otel_collector_image" -}}
+ {{- printf "%s:%s" .Values.images.otelCollector.repository
.Values.images.otelCollector.tag }}
+{{- end }}
+
{{- define "fernet_key_secret" -}}
{{- default (printf "%s-fernet-key" (include "airflow.fullname" .))
.Values.fernetKeySecretName }}
{{- end }}
@@ -680,6 +702,11 @@ server_tls_key_file = /etc/pgbouncer/server.key
{{- include "_serviceAccountName" (merge (dict "key" "statsd") .) -}}
{{- end }}
+{{/* Create the name of the OTel Collector service account to use */}}
+{{- define "otelCollector.serviceAccountName" -}}
+ {{- include "_serviceAccountName" (merge (dict "key" "otelCollector"
"nameSuffix" "otel-collector") .) -}}
+{{- end }}
+
{{/* Create the name of the create user job service account to use */}}
{{- define "createUserJob.serviceAccountName" -}}
{{- include "_serviceAccountName" (merge (dict "key" "createUserJob"
"nameSuffix" "create-user-job") .) -}}
diff --git a/chart/templates/configmaps/otel-collector-configmap.yaml
b/chart/templates/configmaps/otel-collector-configmap.yaml
new file mode 100644
index 00000000000..923ef40e871
--- /dev/null
+++ b/chart/templates/configmaps/otel-collector-configmap.yaml
@@ -0,0 +1,73 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+###########################
+## OTel Collector ConfigMap
+###########################
+{{- if or .Values.otelCollector.tracesEnabled
.Values.otelCollector.metricsEnabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ include "airflow.fullname" . }}-otel-collector
+ labels:
+ tier: airflow
+ component: config
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- with .Values.labels }}
+ {{- toYaml . | nindent 4 }}
+ {{- end }}
+data:
+ config.yml: |
+{{- if .Values.otelCollector.config }}
+{{ tpl .Values.otelCollector.config . | indent 4 }}
+{{- else }}
+ extensions:
+ health_check:
+ endpoint: 0.0.0.0:13133
+
+ receivers:
+ otlp:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:{{ .Values.ports.otelCollectorOtlpGrpc }}
+ http:
+ endpoint: 0.0.0.0:{{ .Values.ports.otelCollectorOtlpHttp }}
+
+ processors:
+ batch: {}
+
+ exporters:
+ logging:
+ verbosity: basic
+
+ service:
+ extensions: [health_check]
+ pipelines:
+ traces:
+ receivers: [otlp]
+ processors: [batch]
+ exporters: [logging]
+ metrics:
+ receivers: [otlp]
+ processors: [batch]
+ exporters: [logging]
+{{- end }}
+{{- end }}
diff --git a/chart/templates/otel-collector/otel-collector-deployment.yaml
b/chart/templates/otel-collector/otel-collector-deployment.yaml
new file mode 100644
index 00000000000..45cf3fddb68
--- /dev/null
+++ b/chart/templates/otel-collector/otel-collector-deployment.yaml
@@ -0,0 +1,115 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+############################
+## OTel Collector Deployment
+############################
+{{- if or .Values.otelCollector.tracesEnabled
.Values.otelCollector.metricsEnabled }}
+{{- $nodeSelector := or .Values.otelCollector.nodeSelector
.Values.nodeSelector }}
+{{- $affinity := or .Values.otelCollector.affinity .Values.affinity }}
+{{- $tolerations := or .Values.otelCollector.tolerations .Values.tolerations }}
+{{- $topologySpreadConstraints := or
.Values.otelCollector.topologySpreadConstraints
.Values.topologySpreadConstraints }}
+{{- $revisionHistoryLimit := include "airflow.revisionHistoryLimit" (list
.Values.otelCollector.revisionHistoryLimit .Values.revisionHistoryLimit) }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ include "airflow.fullname" . }}-otel-collector
+ labels:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- if or .Values.labels .Values.otelCollector.labels }}
+ {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- end }}
+ {{- with .Values.otelCollector.annotations }}
+ annotations: {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ replicas: 1
+ {{- if ne $revisionHistoryLimit "" }}
+ revisionHistoryLimit: {{ $revisionHistoryLimit }}
+ {{- end }}
+ selector:
+ matchLabels:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ template:
+ metadata:
+ labels:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ {{- if or .Values.labels .Values.otelCollector.labels }}
+ {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml |
nindent 8 }}
+ {{- end }}
+ annotations:
+ checksum/otel-collector-config: {{ include (print $.Template.BasePath
"/configmaps/otel-collector-configmap.yaml") . | sha256sum }}
+ {{- with .Values.otelCollector.podAnnotations }}
+ {{- tpl (toYaml .) $ | nindent 8 }}
+ {{- end }}
+ spec:
+ nodeSelector: {{- toYaml $nodeSelector | nindent 8 }}
+ affinity: {{- toYaml $affinity | nindent 8 }}
+ tolerations: {{- toYaml $tolerations | nindent 8 }}
+ topologySpreadConstraints: {{- toYaml $topologySpreadConstraints |
nindent 8 }}
+ terminationGracePeriodSeconds: {{
.Values.otelCollector.terminationGracePeriodSeconds }}
+ serviceAccountName: {{ include "otelCollector.serviceAccountName" . }}
+ {{- if .Values.otelCollector.priorityClassName }}
+ priorityClassName: {{ .Values.otelCollector.priorityClassName }}
+ {{- end }}
+ restartPolicy: Always
+ securityContext: {{- toYaml .Values.otelCollector.securityContexts.pod |
nindent 8 }}
+ containers:
+ - name: otel-collector
+ image: {{ template "otel_collector_image" . }}
+ imagePullPolicy: {{ .Values.images.otelCollector.pullPolicy }}
+ securityContext: {{- toYaml
.Values.otelCollector.securityContexts.container | nindent 12 }}
+ args: {{- tpl (toYaml .Values.otelCollector.args) . | nindent 12 }}
+ ports:
+ - name: otlp-grpc
+ containerPort: {{ .Values.ports.otelCollectorOtlpGrpc }}
+ protocol: TCP
+ - name: otlp-http
+ containerPort: {{ .Values.ports.otelCollectorOtlpHttp }}
+ protocol: TCP
+ livenessProbe:
+ httpGet:
+ path: /
+ port: 13133
+ initialDelaySeconds: {{
.Values.otelCollector.livenessProbe.initialDelaySeconds }}
+ periodSeconds: {{
.Values.otelCollector.livenessProbe.periodSeconds }}
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 13133
+ initialDelaySeconds: {{
.Values.otelCollector.readinessProbe.initialDelaySeconds }}
+ periodSeconds: {{
.Values.otelCollector.readinessProbe.periodSeconds }}
+ resources: {{- toYaml .Values.otelCollector.resources | nindent 12 }}
+ volumeMounts:
+ - name: config
+ mountPath: /etc/otel-collector
+ readOnly: true
+ volumes:
+ - name: config
+ configMap:
+ name: {{ include "airflow.fullname" . }}-otel-collector
+{{- end }}
diff --git a/chart/templates/otel-collector/otel-collector-networkpolicy.yaml
b/chart/templates/otel-collector/otel-collector-networkpolicy.yaml
new file mode 100644
index 00000000000..f58cb77fdec
--- /dev/null
+++ b/chart/templates/otel-collector/otel-collector-networkpolicy.yaml
@@ -0,0 +1,59 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+#######################################
+## Airflow OTel Collector NetworkPolicy
+#######################################
+{{- if and .Values.networkPolicies.enabled (or
.Values.otelCollector.tracesEnabled .Values.otelCollector.metricsEnabled) }}
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: {{ include "airflow.fullname" . }}-otel-collector-policy
+ labels:
+ tier: airflow
+ component: otel-collector-policy
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- if or .Values.labels .Values.otelCollector.labels }}
+ {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- end }}
+spec:
+ podSelector:
+ matchLabels:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ policyTypes:
+ - Ingress
+ ingress:
+ - from:
+ - podSelector:
+ matchLabels:
+ tier: airflow
+ release: {{ .Release.Name }}
+ {{- if .Values.otelCollector.extraNetworkPolicies }}
+ {{- toYaml .Values.otelCollector.extraNetworkPolicies | nindent 4 }}
+ {{- end }}
+ ports:
+ - protocol: TCP
+ port: {{ .Values.ports.otelCollectorOtlpHttp }}
+ - protocol: TCP
+ port: {{ .Values.ports.otelCollectorOtlpGrpc }}
+{{- end }}
diff --git a/chart/templates/otel-collector/otel-collector-service.yaml
b/chart/templates/otel-collector/otel-collector-service.yaml
new file mode 100644
index 00000000000..9d2ce3debed
--- /dev/null
+++ b/chart/templates/otel-collector/otel-collector-service.yaml
@@ -0,0 +1,55 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+#########################
+## OTel Collector Service
+#########################
+{{- if or .Values.otelCollector.tracesEnabled
.Values.otelCollector.metricsEnabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ include "airflow.fullname" . }}-otel-collector
+ labels:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- if or .Values.labels .Values.otelCollector.labels }}
+ {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- end }}
+ {{- with .Values.otelCollector.service.annotations }}
+ annotations: {{- toYaml . | nindent 4 }}
+ {{- end }}
+spec:
+ type: ClusterIP
+ selector:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ ports:
+ - name: otlp-grpc
+ protocol: TCP
+ port: {{ .Values.ports.otelCollectorOtlpGrpc }}
+ targetPort: {{ .Values.ports.otelCollectorOtlpGrpc }}
+ - name: otlp-http
+ protocol: TCP
+ port: {{ .Values.ports.otelCollectorOtlpHttp }}
+ targetPort: {{ .Values.ports.otelCollectorOtlpHttp }}
+{{- end }}
diff --git a/chart/templates/otel-collector/otel-collector-serviceaccount.yaml
b/chart/templates/otel-collector/otel-collector-serviceaccount.yaml
new file mode 100644
index 00000000000..379e85a8b20
--- /dev/null
+++ b/chart/templates/otel-collector/otel-collector-serviceaccount.yaml
@@ -0,0 +1,41 @@
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/}}
+
+########################################
+## Airflow OTel Collector ServiceAccount
+########################################
+{{- if and (or .Values.otelCollector.tracesEnabled
.Values.otelCollector.metricsEnabled)
.Values.otelCollector.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+automountServiceAccountToken: {{
.Values.otelCollector.serviceAccount.automountServiceAccountToken }}
+metadata:
+ name: {{ include "otelCollector.serviceAccountName" . }}
+ labels:
+ tier: airflow
+ component: otel-collector
+ release: {{ .Release.Name }}
+ chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ heritage: {{ .Release.Service }}
+ {{- if or .Values.labels .Values.otelCollector.labels }}
+ {{- mustMerge .Values.otelCollector.labels .Values.labels | toYaml |
nindent 4 }}
+ {{- end }}
+ {{- with .Values.otelCollector.serviceAccount.annotations }}
+ annotations: {{- toYaml . | nindent 4 }}
+ {{- end }}
+{{- end }}
diff --git a/chart/values.schema.json b/chart/values.schema.json
index d5612c1ab3f..4a645d85c22 100644
--- a/chart/values.schema.json
+++ b/chart/values.schema.json
@@ -17,6 +17,7 @@
"Flower",
"Redis",
"StatsD",
+ "OpenTelemetry",
"Jobs",
"Kubernetes",
"Ingress",
@@ -1088,6 +1089,28 @@
"default": "IfNotPresent"
}
}
+ },
+ "otelCollector": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "repository": {
+ "type": "string"
+ },
+ "tag": {
+ "type": "string"
+ },
+ "pullPolicy": {
+ "type": "string",
+ "enum": [
+ "Always",
+ "Never",
+ "IfNotPresent"
+ ],
+ "default": "IfNotPresent"
+ }
+ },
+ "description": "Configuration of the OTel Collector image."
}
}
},
@@ -8302,7 +8325,7 @@
}
},
"enabled": {
- "description": "Enable StatsD.",
+ "description": "Enable StatsD. When
`otelCollector.metricsEnabled` is true, [metrics] statsd_on is set to false in
the rendered Airflow config because Airflow can only export metrics to one
backend at a time.",
"type": "boolean",
"default": true
},
@@ -8611,6 +8634,263 @@
}
}
},
+ "otelCollector": {
+ "description": "OpenTelemetry Collector settings.",
+ "type": "object",
+ "x-docsSection": "OpenTelemetry",
+ "additionalProperties": false,
+ "properties": {
+ "tracesEnabled": {
+ "description": "Send Airflow traces to the OTel Collector.
Sets `[traces] otel_on` in the rendered Airflow config and deploys the
collector when either `tracesEnabled` or `metricsEnabled` is true.",
+ "type": "boolean",
+ "default": true
+ },
+ "metricsEnabled": {
+ "description": "Send Airflow metrics to the OTel
Collector. Sets `[metrics] otel_on` in the rendered Airflow config and forces
`[metrics] statsd_on` to False because Airflow can only export metrics to one
backend at a time.",
+ "type": "boolean",
+ "default": false
+ },
+ "metricExportIntervalMs": {
+ "description": "Interval (in milliseconds) at which the
OTel SDK exports metrics to the collector. Sets the
`OTEL_METRIC_EXPORT_INTERVAL` env var on Airflow pods.",
+ "type": "integer",
+ "default": 30000,
+ "minimum": 0
+ },
+ "config": {
+ "description": "Override the OTel Collector `config.yml`.
When set (non-empty), this string replaces the chart's default collector
config. The value is rendered with `tpl`, so values like `{{
.Values.ports.otelCollectorOtlpHttp }}` or `{{ include \"airflow.fullname\" .
}}` can be referenced from inside the string. Leave empty to use the chart
default.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": null
+ },
+ "args": {
+ "description": "Args to pass to the OTel Collector
container (templated).",
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "type": "string"
+ },
+ "default": [
+ "--config=/etc/otel-collector/config.yml"
+ ]
+ },
+ "revisionHistoryLimit": {
+ "description": "Number of old ReplicaSets to retain.",
+ "type": [
+ "integer",
+ "null"
+ ],
+ "default": null,
+ "x-docsSection": "Kubernetes"
+ },
+ "annotations": {
+ "description": "Annotations to add to the OTel Collector
deployment.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "terminationGracePeriodSeconds": {
+ "description": "Grace period for the OTel Collector to
finish after SIGTERM is sent from Kubernetes.",
+ "type": "integer",
+ "default": 30,
+ "minimum": 0
+ },
+ "livenessProbe": {
+ "description": "Liveness probe configuration for the OTel
Collector container.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "initialDelaySeconds": {
+ "description": "Number of seconds after the
container has started before the liveness probe is initiated.",
+ "type": "integer",
+ "default": 10,
+ "minimum": 0
+ },
+ "periodSeconds": {
+ "description": "How often (in seconds) to perform
the liveness probe.",
+ "type": "integer",
+ "default": 15,
+ "minimum": 0
+ }
+ }
+ },
+ "readinessProbe": {
+ "description": "Readiness probe configuration for the OTel
Collector container.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "initialDelaySeconds": {
+ "description": "Number of seconds after the
container has started before the readiness probe is initiated.",
+ "type": "integer",
+ "default": 10,
+ "minimum": 0
+ },
+ "periodSeconds": {
+ "description": "How often (in seconds) to perform
the readiness probe.",
+ "type": "integer",
+ "default": 15,
+ "minimum": 0
+ }
+ }
+ },
+ "resources": {
+ "description": "Resources for the OTel Collector pod.",
+ "type": "object",
+ "default": {},
+ "examples": [
+ {
+ "limits": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ },
+ "requests": {
+ "cpu": "100m",
+ "memory": "128Mi"
+ }
+ }
+ ],
+ "$ref":
"#/definitions/io.k8s.api.core.v1.ResourceRequirements"
+ },
+ "service": {
+ "description": "OTel Collector Service configuration.",
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "annotations": {
+ "description": "Annotations for the OTel Collector
Service.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "nodeSelector": {
+ "description": "Select certain nodes for OTel Collector
pods.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "affinity": {
+ "description": "Specify scheduling constraints for OTel
Collector pods.",
+ "type": "object",
+ "default": {},
+ "$ref": "#/definitions/io.k8s.api.core.v1.Affinity"
+ },
+ "tolerations": {
+ "description": "Specify Tolerations for OTel Collector
pods.",
+ "type": "array",
+ "default": [],
+ "items": {
+ "type": "object",
+ "$ref": "#/definitions/io.k8s.api.core.v1.Toleration"
+ }
+ },
+ "topologySpreadConstraints": {
+ "description": "Specify topology spread constraints for
OTel Collector pods.",
+ "type": "array",
+ "default": [],
+ "items": {
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint"
+ }
+ },
+ "priorityClassName": {
+ "description": "Specify priority for OTel Collector pods.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": null
+ },
+ "labels": {
+ "description": "Labels specific to OTel Collector objects
and pods.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "podAnnotations": {
+ "description": "Annotations to add to the OTel Collector
pods (templated).",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "securityContexts": {
+ "description": "Security context definition for the OTel
Collector.",
+ "type": "object",
+ "x-docsSection": "Kubernetes",
+ "properties": {
+ "pod": {
+ "description": "Pod security context definition
for the OTel Collector.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.PodSecurityContext",
+ "default": {},
+ "x-docsSection": "Kubernetes"
+ },
+ "container": {
+ "description": "Container security context
definition for the OTel Collector.",
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.core.v1.SecurityContext",
+ "default": {},
+ "x-docsSection": "Kubernetes"
+ }
+ }
+ },
+ "extraNetworkPolicies": {
+ "description": "Additional NetworkPolicies as needed for
the OTel Collector. Only used when `networkPolicies.enabled` is true.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "$ref":
"#/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer"
+ },
+ "default": []
+ },
+ "serviceAccount": {
+ "description": "Create ServiceAccount.",
+ "type": "object",
+ "properties": {
+ "automountServiceAccountToken": {
+ "description": "Specifies if the ServiceAccount's
API credentials should be auto-mounted onto OTel Collector pods. Defaults to
false because the chart's default collector config does not talk to the
Kubernetes API.",
+ "type": "boolean",
+ "default": false
+ },
+ "create": {
+ "description": "Specifies whether a ServiceAccount
should be created.",
+ "type": "boolean",
+ "default": true
+ },
+ "name": {
+ "description": "The name of the ServiceAccount to
use. If not set and create is true, a name is generated using the release
name.",
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": null
+ },
+ "annotations": {
+ "description": "Annotations to add to the OTel
Collector Kubernetes ServiceAccount.",
+ "type": "object",
+ "default": {},
+ "additionalProperties": {
+ "type": "string"
+ }
+ }
+ }
+ }
+ }
+ },
"pgbouncer": {
"description": "PgBouncer settings.",
"type": "object",
@@ -9853,6 +10133,16 @@
"description": "PgBouncer scrape port.",
"type": "integer",
"default": 9127
+ },
+ "otelCollectorOtlpHttp": {
+ "description": "OTel Collector OTLP HTTP port.",
+ "type": "integer",
+ "default": 4318
+ },
+ "otelCollectorOtlpGrpc": {
+ "description": "OTel Collector OTLP gRPC port.",
+ "type": "integer",
+ "default": 4317
}
}
},
diff --git a/chart/values.yaml b/chart/values.yaml
index 39e7a4de8aa..0df514c93ec 100644
--- a/chart/values.yaml
+++ b/chart/values.yaml
@@ -122,6 +122,10 @@ images:
repository: registry.k8s.io/git-sync/git-sync
tag: v4.4.2
pullPolicy: IfNotPresent
+ otelCollector:
+ repository: otel/opentelemetry-collector-contrib
+ tag: "0.70.0"
+ pullPolicy: IfNotPresent
# Select certain nodes for Airflow pods.
nodeSelector: {}
@@ -3067,6 +3071,9 @@ statsd:
# Add custom annotations to the StatsD ConfigMap
configMapAnnotations: {}
+ # When otelCollector.metricsEnabled is true, [metrics] statsd_on is set to
+ # False in the rendered Airflow config because Airflow can only export
metrics
+ # to one backend at a time.
enabled: true
# Max number of old ReplicaSets to retain
@@ -3179,6 +3186,111 @@ statsd:
# Environment variables to add to StatsD container
env: []
+# OpenTelemetry Collector settings
+otelCollector:
+ # Send Airflow traces to the OTel Collector (sets [traces] otel_on).
+ tracesEnabled: true
+
+ # Send Airflow metrics to the OTel Collector (sets [metrics] otel_on and
disables statsd).
+ metricsEnabled: false
+
+ # Default value for the OTEL_METRIC_EXPORT_INTERVAL env var on Airflow pods.
+ # Interval (in milliseconds) at which the OTel SDK exports metrics to the
collector.
+ metricExportIntervalMs: 30000
+
+ # Override the OTel Collector config.yml. When set (non-empty), this string
replaces
+ # the chart's default collector config. The value is rendered with `tpl`, so
you can
+ # reference values like `{{ .Values.ports.otelCollectorOtlpHttp }}` or
+ # `{{ include "airflow.fullname" . }}` from inside the string. Leave empty
to use
+ # the chart default.
+ config: ~
+ # config: |
+ # extensions:
+ # health_check:
+ # endpoint: 0.0.0.0:13133
+ # receivers:
+ # otlp:
+ # protocols:
+ # http:
+ # endpoint: 0.0.0.0:{{ .Values.ports.otelCollectorOtlpHttp }}
+ # processors:
+ # batch: {}
+ # exporters:
+ # logging:
+ # verbosity: basic
+ # service:
+ # extensions: [health_check]
+ # pipelines:
+ # traces:
+ # receivers: [otlp]
+ # processors: [batch]
+ # exporters: [logging]
+
+ # Args to pass to the OTel Collector container (templated).
+ args:
+ - "--config=/etc/otel-collector/config.yml"
+
+ # Max number of old ReplicaSets to retain
+ revisionHistoryLimit: ~
+
+ # Annotations to add to the OTel Collector Deployment
+ annotations: {}
+
+ # Grace period for OTel Collector to finish after SIGTERM
+ terminationGracePeriodSeconds: 30
+
+ livenessProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 15
+
+ readinessProbe:
+ initialDelaySeconds: 10
+ periodSeconds: 15
+
+ resources: {}
+ # limits:
+ # cpu: 100m
+ # memory: 128Mi
+ # requests:
+ # cpu: 100m
+ # memory: 128Mi
+
+ service:
+ annotations: {}
+
+ nodeSelector: {}
+ affinity: {}
+ tolerations: []
+ topologySpreadConstraints: []
+ priorityClassName: ~
+ labels: {}
+ podAnnotations: {}
+
+ securityContexts:
+ pod: {}
+ container: {}
+
+ # Additional ingress peers/rules for the OTel Collector NetworkPolicy.
+ # Only used when `networkPolicies.enabled` is true.
+ extraNetworkPolicies: []
+
+ serviceAccount:
+ # ref:
https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+ # The default OTel Collector config does not talk to the Kubernetes API,
so credentials
+ # are not auto-mounted. Flip to true if you override
`otelCollector.config` to use
+ # processors that need API access (e.g. `k8sattributes`).
+ automountServiceAccountToken: false
+
+ # Specifies whether a Service Account should be created
+ create: true
+
+ # The name of the Service Account to use.
+ # If not set and `create` is 'true', a name is generated using the release
name
+ name: ~
+
+ # Annotations to add to the OTel Collector Kubernetes ServiceAccount.
+ annotations: {}
+
# PgBouncer settings
pgbouncer:
# Enable PgBouncer
@@ -3574,6 +3686,8 @@ ports:
redisDB: 6379
statsdIngest: 9125
statsdScrape: 9102
+ otelCollectorOtlpHttp: 4318
+ otelCollectorOtlpGrpc: 4317
pgbouncer: 6543
pgbouncerScrape: 9127
apiServer: 8080
@@ -3793,10 +3907,15 @@ config:
remote_logging: '{{- ternary "True" "False" (or
.Values.elasticsearch.enabled .Values.opensearch.enabled) }}'
colored_console_log: 'False'
metrics:
- statsd_on: '{{ ternary "True" "False" .Values.statsd.enabled }}'
+ statsd_on: '{{ ternary "True" "False" (and .Values.statsd.enabled (not
.Values.otelCollector.metricsEnabled)) }}'
statsd_port: 9125
statsd_prefix: airflow
statsd_host: '{{ printf "%s-statsd" (include "airflow.fullname" .) }}'
+ otel_on: '{{ ternary "True" "False" .Values.otelCollector.metricsEnabled
}}'
+ otel_host: '{{ if .Values.otelCollector.metricsEnabled }}{{ printf
"%s-otel-collector" (include "airflow.fullname" .) }}{{ end }}'
+ otel_port: '{{ .Values.ports.otelCollectorOtlpHttp }}'
+ traces:
+ otel_on: '{{ ternary "True" "False" .Values.otelCollector.tracesEnabled }}'
fab:
enable_proxy_fix: 'True'
celery:
diff --git a/dev/breeze/doc/images/output_testing_helm-tests.svg
b/dev/breeze/doc/images/output_testing_helm-tests.svg
index d98995ed56a..f2cc5db19fb 100644
--- a/dev/breeze/doc/images/output_testing_helm-tests.svg
+++ b/dev/breeze/doc/images/output_testing_helm-tests.svg
@@ -142,7 +142,7 @@
</text><text class="breeze-testing-helm-tests-r1" x="1464" y="117.6"
textLength="12.2" clip-path="url(#breeze-testing-helm-tests-line-4)">
</text><text class="breeze-testing-helm-tests-r5" x="0" y="142"
textLength="24.4"
clip-path="url(#breeze-testing-helm-tests-line-5)">╭─</text><text
class="breeze-testing-helm-tests-r5" x="24.4" y="142" textLength="378.2"
clip-path="url(#breeze-testing-helm-tests-line-5)"> Flags for helms-tests command </text><text
class="breeze-testing-helm-tests-r5" x="402.6" y="142" textLength="1037"
clip-path="url(#breeze-testing-helm-tests-line-5)">───────────────────────────
[...]
</text><text class="breeze-testing-helm-tests-r5" x="0" y="166.4"
textLength="12.2"
clip-path="url(#breeze-testing-helm-tests-line-6)">│</text><text
class="breeze-testing-helm-tests-r4" x="24.4" y="166.4" textLength="244"
clip-path="url(#breeze-testing-helm-tests-line-6)">--test-type         </text><text
class="breeze-testing-helm-tests-r1" x="292.8" y="166.4" textLength="317.2"
clip-path="url(#breeze-testing-helm-tests-line-6)">Type of&# [...]
-</text><text class="breeze-testing-helm-tests-r5" x="0" y="190.8"
textLength="12.2"
clip-path="url(#breeze-testing-helm-tests-line-7)">│</text><text
class="breeze-testing-helm-tests-r6" x="292.8" y="190.8" textLength="597.8"
clip-path="url(#breeze-testing-helm-tests-line-7)">dagprocessor | other | redis | security | statsd)</text><text
class="breeze-testing-helm-tests-r5" x="1451.8" y="190.8" textLength="12.2"
clip-path="url(#breeze-testing-helm-te [...]
+</text><text class="breeze-testing-helm-tests-r5" x="0" y="190.8"
textLength="12.2"
clip-path="url(#breeze-testing-helm-tests-line-7)">│</text><text
class="breeze-testing-helm-tests-r6" x="292.8" y="190.8" textLength="805.2"
clip-path="url(#breeze-testing-helm-tests-line-7)">dagprocessor | otel_collector | other | redis | security | statsd)</text><text
class="breeze-testing-helm-tests-r5" x="1451.8" y="190.8" textLength="12.2"
clip-path=" [...]
</text><text class="breeze-testing-helm-tests-r5" x="0" y="215.2"
textLength="12.2"
clip-path="url(#breeze-testing-helm-tests-line-8)">│</text><text
class="breeze-testing-helm-tests-r4" x="24.4" y="215.2" textLength="244"
clip-path="url(#breeze-testing-helm-tests-line-8)">--test-timeout      </text><text
class="breeze-testing-helm-tests-r1" x="292.8" y="215.2" textLength="1146.8"
clip-path="url(#breeze-testing-helm-tests-line-8)">Test timeout in
[...]
</text><text class="breeze-testing-helm-tests-r5" x="0" y="239.6"
textLength="12.2"
clip-path="url(#breeze-testing-helm-tests-line-9)">│</text><text
class="breeze-testing-helm-tests-r5" x="292.8" y="239.6" textLength="158.6"
clip-path="url(#breeze-testing-helm-tests-line-9)">[default: 60]</text><text
class="breeze-testing-helm-tests-r6" x="463.6" y="239.6" textLength="244"
clip-path="url(#breeze-testing-helm-tests-line-9)">(INTEGER RANGE x>=0)</text><text
class="breeze- [...]
</text><text class="breeze-testing-helm-tests-r5" x="0" y="264"
textLength="12.2"
clip-path="url(#breeze-testing-helm-tests-line-10)">│</text><text
class="breeze-testing-helm-tests-r4" x="24.4" y="264" textLength="244"
clip-path="url(#breeze-testing-helm-tests-line-10)">--kubernetes-version</text><text
class="breeze-testing-helm-tests-r1" x="292.8" y="264" textLength="658.8"
clip-path="url(#breeze-testing-helm-tests-line-10)">Kubernetes version to validate helm t
[...]
diff --git a/dev/breeze/doc/images/output_testing_helm-tests.txt
b/dev/breeze/doc/images/output_testing_helm-tests.txt
index 98bf55d3fd8..5331937a401 100644
--- a/dev/breeze/doc/images/output_testing_helm-tests.txt
+++ b/dev/breeze/doc/images/output_testing_helm-tests.txt
@@ -1 +1 @@
-09ffed0009ecde363528f90bc5d8ab57
+02ba09ea9213ae046fe814563630054d
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
b/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
index 0ce1123c874..a46ea30bee2 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_airflow_common.py
@@ -349,6 +349,10 @@ class TestAirflowCommon:
"AIRFLOW__CORE__FERNET_KEY",
"AIRFLOW_CONN_AIRFLOW_DB",
"AIRFLOW__CELERY__BROKER_URL",
+ "OTEL_SERVICE_NAME",
+ "OTEL_EXPORTER_OTLP_PROTOCOL",
+ "OTEL_TRACES_EXPORTER",
+ "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
]
expected_vars_in_worker = ["DUMB_INIT_SETSID"] + expected_vars
for doc in docs:
@@ -377,6 +381,10 @@ class TestAirflowCommon:
"AIRFLOW__API__SECRET_KEY",
"AIRFLOW__API_AUTH__JWT_SECRET",
"AIRFLOW__CELERY__BROKER_URL",
+ "OTEL_SERVICE_NAME",
+ "OTEL_EXPORTER_OTLP_PROTOCOL",
+ "OTEL_TRACES_EXPORTER",
+ "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
]
expected_vars_no_jwt = [
"AIRFLOW_HOME",
@@ -385,6 +393,10 @@ class TestAirflowCommon:
"AIRFLOW_CONN_AIRFLOW_DB",
"AIRFLOW__API__SECRET_KEY",
"AIRFLOW__CELERY__BROKER_URL",
+ "OTEL_SERVICE_NAME",
+ "OTEL_EXPORTER_OTLP_PROTOCOL",
+ "OTEL_TRACES_EXPORTER",
+ "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
]
for doc in docs:
component = doc["metadata"]["labels"]["component"]
diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
index 74b921765c7..76ebc5e87a8 100644
--- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
+++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py
@@ -34,6 +34,7 @@ OBJECTS_STD_NAMING = {
("ServiceAccount", "test-basic-airflow-redis"),
("ServiceAccount", "test-basic-airflow-scheduler"),
("ServiceAccount", "test-basic-airflow-statsd"),
+ ("ServiceAccount", "test-basic-airflow-otel-collector"),
("ServiceAccount", "test-basic-airflow-triggerer"),
("ServiceAccount", "test-basic-airflow-worker"),
("Secret", "test-basic-airflow-api-secret-key"),
@@ -45,6 +46,7 @@ OBJECTS_STD_NAMING = {
("Secret", "test-basic-postgresql"),
("ConfigMap", "test-basic-airflow-config"),
("ConfigMap", "test-basic-airflow-statsd"),
+ ("ConfigMap", "test-basic-airflow-otel-collector"),
("Role", "test-basic-airflow-pod-launcher-role"),
("Role", "test-basic-airflow-pod-log-reader-role"),
("RoleBinding", "test-basic-airflow-pod-launcher-rolebinding"),
@@ -52,6 +54,7 @@ OBJECTS_STD_NAMING = {
("Service", "test-basic-airflow-api-server"),
("Service", "test-basic-airflow-redis"),
("Service", "test-basic-airflow-statsd"),
+ ("Service", "test-basic-airflow-otel-collector"),
("Service", "test-basic-airflow-triggerer"),
("Service", "test-basic-airflow-worker"),
("Service", "test-basic-postgresql"),
@@ -60,6 +63,7 @@ OBJECTS_STD_NAMING = {
("Deployment", "test-basic-airflow-dag-processor"),
("Deployment", "test-basic-airflow-scheduler"),
("Deployment", "test-basic-airflow-statsd"),
+ ("Deployment", "test-basic-airflow-otel-collector"),
("StatefulSet", "test-basic-airflow-redis"),
("StatefulSet", "test-basic-airflow-worker"),
("StatefulSet", "test-basic-airflow-triggerer"),
@@ -94,6 +98,7 @@ class TestBaseChartTest:
("ServiceAccount", "test-basic-redis"),
("ServiceAccount", "test-basic-scheduler"),
("ServiceAccount", "test-basic-statsd"),
+ ("ServiceAccount", "test-basic-otel-collector"),
("ServiceAccount", "test-basic-triggerer"),
("ServiceAccount", "test-basic-worker"),
("Secret", "test-basic-api-secret-key"),
@@ -105,6 +110,7 @@ class TestBaseChartTest:
("Secret", "test-basic-redis-password"),
("ConfigMap", "test-basic-config"),
("ConfigMap", "test-basic-statsd"),
+ ("ConfigMap", "test-basic-otel-collector"),
("Role", "test-basic-pod-launcher-role"),
("Role", "test-basic-pod-log-reader-role"),
("RoleBinding", "test-basic-pod-launcher-rolebinding"),
@@ -114,12 +120,14 @@ class TestBaseChartTest:
("Service", "test-basic-postgresql"),
("Service", "test-basic-redis"),
("Service", "test-basic-statsd"),
+ ("Service", "test-basic-otel-collector"),
("Service", "test-basic-triggerer"),
("Service", "test-basic-worker"),
("Deployment", "test-basic-api-server"),
("Deployment", "test-basic-dag-processor"),
("Deployment", "test-basic-scheduler"),
("Deployment", "test-basic-statsd"),
+ ("Deployment", "test-basic-otel-collector"),
("StatefulSet", "test-basic-triggerer"),
("StatefulSet", "test-basic-postgresql"),
("StatefulSet", "test-basic-redis"),
@@ -191,6 +199,7 @@ class TestBaseChartTest:
("NetworkPolicy", "test-basic-pgbouncer-policy"),
("NetworkPolicy", "test-basic-scheduler-policy"),
("NetworkPolicy", "test-basic-statsd-policy"),
+ ("NetworkPolicy", "test-basic-otel-collector-policy"),
("NetworkPolicy", "test-basic-worker-policy"),
]
diff --git a/helm-tests/tests/helm_tests/otel_collector/__init__.py
b/helm-tests/tests/helm_tests/otel_collector/__init__.py
new file mode 100644
index 00000000000..13a83393a91
--- /dev/null
+++ b/helm-tests/tests/helm_tests/otel_collector/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git
a/helm-tests/tests/helm_tests/otel_collector/test_labels_deployment.py
b/helm-tests/tests/helm_tests/otel_collector/test_labels_deployment.py
new file mode 100644
index 00000000000..40d2e7b33d6
--- /dev/null
+++ b/helm-tests/tests/helm_tests/otel_collector/test_labels_deployment.py
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestOtelCollectorDeployment:
+ """Tests OTel Collector deployment labels."""
+
+ TEMPLATE_FILE = "templates/otel-collector/otel-collector-deployment.yaml"
+
+ def test_should_add_global_labels_to_metadata(self):
+ """Test adding only .Values.labels to metadata.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {"tracesEnabled": True},
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+
+ def test_should_add_global_labels_to_pod_template(self):
+ """Test adding only .Values.labels to spec.template.metadata.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {"tracesEnabled": True},
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in
jmespath.search("spec.template.metadata.labels", docs[0])
+ assert (
+ jmespath.search("spec.template.metadata.labels",
docs[0])["test_global_label"]
+ == "test_global_label_value"
+ )
+
+ def test_should_add_component_specific_labels_to_pod_template(self):
+ """Test adding only .Values.otelCollector.labels to
spec.template.metadata.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_component_label" in
jmespath.search("spec.template.metadata.labels", docs[0])
+ assert (
+ jmespath.search("spec.template.metadata.labels",
docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def
test_should_merge_global_and_component_specific_labels_in_pod_template(self):
+ """Test adding both .Values.labels and .Values.otelCollector.labels to
spec.template.metadata.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in
jmespath.search("spec.template.metadata.labels", docs[0])
+ assert (
+ jmespath.search("spec.template.metadata.labels",
docs[0])["test_global_label"]
+ == "test_global_label_value"
+ )
+ assert "test_component_label" in
jmespath.search("spec.template.metadata.labels", docs[0])
+ assert (
+ jmespath.search("spec.template.metadata.labels",
docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_component_specific_labels_should_override_global_labels(self):
+ """Test that component-specific labels take precedence over global
labels with the same key."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"common_label": "component_value"},
+ },
+ "labels": {"common_label": "global_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "common_label" in
jmespath.search("spec.template.metadata.labels", docs[0])
+ assert jmespath.search("spec.template.metadata.labels",
docs[0])["common_label"] == "component_value"
diff --git
a/helm-tests/tests/helm_tests/otel_collector/test_labels_networkpolicy.py
b/helm-tests/tests/helm_tests/otel_collector/test_labels_networkpolicy.py
new file mode 100644
index 00000000000..57595157cc1
--- /dev/null
+++ b/helm-tests/tests/helm_tests/otel_collector/test_labels_networkpolicy.py
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestOtelCollectorNetworkPolicy:
+ """Tests OTel Collector network policy labels."""
+
+ TEMPLATE_FILE =
"templates/otel-collector/otel-collector-networkpolicy.yaml"
+
+ def test_should_add_global_labels(self):
+ """Test adding only .Values.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {"tracesEnabled": True},
+ "networkPolicies": {"enabled": True},
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+
+ def test_should_add_component_specific_labels(self):
+ """Test adding only .Values.otelCollector.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ "networkPolicies": {"enabled": True},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_component_label" in jmespath.search("metadata.labels",
docs[0])
+ assert (
+ jmespath.search("metadata.labels", docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_should_merge_global_and_component_specific_labels(self):
+ """Test adding both .Values.labels and .Values.otelCollector.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ "networkPolicies": {"enabled": True},
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+ assert "test_component_label" in jmespath.search("metadata.labels",
docs[0])
+ assert (
+ jmespath.search("metadata.labels", docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_component_specific_labels_should_override_global_labels(self):
+ """Test that component-specific labels take precedence over global
labels with the same key."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"common_label": "component_value"},
+ },
+ "networkPolicies": {"enabled": True},
+ "labels": {"common_label": "global_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "common_label" in jmespath.search("metadata.labels", docs[0])
+ assert jmespath.search("metadata.labels", docs[0])["common_label"] ==
"component_value"
diff --git a/helm-tests/tests/helm_tests/otel_collector/test_labels_service.py
b/helm-tests/tests/helm_tests/otel_collector/test_labels_service.py
new file mode 100644
index 00000000000..f92abc10205
--- /dev/null
+++ b/helm-tests/tests/helm_tests/otel_collector/test_labels_service.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestOtelCollectorService:
+ """Tests OTel Collector service labels."""
+
+ TEMPLATE_FILE = "templates/otel-collector/otel-collector-service.yaml"
+
+ def test_should_add_global_labels(self):
+ """Test adding only .Values.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {"tracesEnabled": True},
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+
+ def test_should_add_component_specific_labels(self):
+ """Test adding only .Values.otelCollector.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_component_label" in jmespath.search("metadata.labels",
docs[0])
+ assert (
+ jmespath.search("metadata.labels", docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_should_merge_global_and_component_specific_labels(self):
+ """Test adding both .Values.labels and .Values.otelCollector.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+ assert "test_component_label" in jmespath.search("metadata.labels",
docs[0])
+ assert (
+ jmespath.search("metadata.labels", docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_component_specific_labels_should_override_global_labels(self):
+ """Test that component-specific labels take precedence over global
labels with the same key."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"common_label": "component_value"},
+ },
+ "labels": {"common_label": "global_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "common_label" in jmespath.search("metadata.labels", docs[0])
+ assert jmespath.search("metadata.labels", docs[0])["common_label"] ==
"component_value"
diff --git
a/helm-tests/tests/helm_tests/otel_collector/test_labels_serviceaccount.py
b/helm-tests/tests/helm_tests/otel_collector/test_labels_serviceaccount.py
new file mode 100644
index 00000000000..c899670766f
--- /dev/null
+++ b/helm-tests/tests/helm_tests/otel_collector/test_labels_serviceaccount.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+from chart_utils.helm_template_generator import render_chart
+
+
+class TestOtelCollectorServiceAccount:
+ """Tests OTel Collector service account labels."""
+
+ TEMPLATE_FILE =
"templates/otel-collector/otel-collector-serviceaccount.yaml"
+
+ def test_should_add_global_labels(self):
+ """Test adding only .Values.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {"tracesEnabled": True},
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+
+ def test_should_add_component_specific_labels(self):
+ """Test adding only .Values.otelCollector.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_component_label" in jmespath.search("metadata.labels",
docs[0])
+ assert (
+ jmespath.search("metadata.labels", docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_should_merge_global_and_component_specific_labels(self):
+ """Test adding both .Values.labels and .Values.otelCollector.labels."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"test_component_label":
"test_component_label_value"},
+ },
+ "labels": {"test_global_label": "test_global_label_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "test_global_label" in jmespath.search("metadata.labels",
docs[0])
+ assert jmespath.search("metadata.labels",
docs[0])["test_global_label"] == "test_global_label_value"
+ assert "test_component_label" in jmespath.search("metadata.labels",
docs[0])
+ assert (
+ jmespath.search("metadata.labels", docs[0])["test_component_label"]
+ == "test_component_label_value"
+ )
+
+ def test_component_specific_labels_should_override_global_labels(self):
+ """Test that component-specific labels take precedence over global
labels with the same key."""
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "tracesEnabled": True,
+ "labels": {"common_label": "component_value"},
+ },
+ "labels": {"common_label": "global_value"},
+ },
+ show_only=[self.TEMPLATE_FILE],
+ )
+
+ assert "common_label" in jmespath.search("metadata.labels", docs[0])
+ assert jmespath.search("metadata.labels", docs[0])["common_label"] ==
"component_value"
diff --git a/helm-tests/tests/helm_tests/otel_collector/test_otel_collector.py
b/helm-tests/tests/helm_tests/otel_collector/test_otel_collector.py
new file mode 100644
index 00000000000..70ef3aec2fd
--- /dev/null
+++ b/helm-tests/tests/helm_tests/otel_collector/test_otel_collector.py
@@ -0,0 +1,484 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import jmespath
+import pytest
+import yaml
+from chart_utils.helm_template_generator import render_chart
+
+OTEL_TEMPLATES = [
+ "templates/configmaps/otel-collector-configmap.yaml",
+ "templates/otel-collector/otel-collector-deployment.yaml",
+ "templates/otel-collector/otel-collector-service.yaml",
+ "templates/otel-collector/otel-collector-serviceaccount.yaml",
+]
+
+DEPLOYMENT_TEMPLATE = "templates/otel-collector/otel-collector-deployment.yaml"
+SERVICE_TEMPLATE = "templates/otel-collector/otel-collector-service.yaml"
+CONFIGMAP_TEMPLATE = "templates/configmaps/otel-collector-configmap.yaml"
+SERVICE_ACCOUNT_TEMPLATE =
"templates/otel-collector/otel-collector-serviceaccount.yaml"
+AIRFLOW_CONFIGMAP_TEMPLATE = "templates/configmaps/configmap.yaml"
+
+AIRFLOW_POD_TEMPLATES = [
+ "templates/api-server/api-server-deployment.yaml",
+ "templates/scheduler/scheduler-deployment.yaml",
+ "templates/workers/worker-deployment.yaml",
+ "templates/triggerer/triggerer-deployment.yaml",
+ "templates/dag-processor/dag-processor-deployment.yaml",
+]
+
+
+def _env_names(doc):
+ """Return the list of env var names from the first container of a pod
spec."""
+ env = jmespath.search("spec.template.spec.containers[0].env", doc) or []
+ return [e["name"] for e in env]
+
+
+def _env_value(doc, name):
+ env = jmespath.search("spec.template.spec.containers[0].env", doc) or []
+ for entry in env:
+ if entry["name"] == name:
+ return entry.get("value")
+ return None
+
+
+class TestOtelCollectorResourceGating:
+ """Resource emission depends on tracesEnabled / metricsEnabled."""
+
+ def test_default_renders_all_resources(self):
+ docs = render_chart(show_only=OTEL_TEMPLATES)
+ kinds = {d["kind"] for d in docs}
+ assert kinds == {"ConfigMap", "Deployment", "Service",
"ServiceAccount"}
+
+ def test_both_flags_disabled_renders_nothing(self):
+ docs = render_chart(
+ values={"otelCollector": {"tracesEnabled": False,
"metricsEnabled": False}},
+ show_only=OTEL_TEMPLATES,
+ )
+ assert docs == []
+
+ def test_metrics_only_renders_all_resources(self):
+ docs = render_chart(
+ values={"otelCollector": {"tracesEnabled": False,
"metricsEnabled": True}},
+ show_only=OTEL_TEMPLATES,
+ )
+ kinds = {d["kind"] for d in docs}
+ assert kinds == {"ConfigMap", "Deployment", "Service",
"ServiceAccount"}
+
+
+class TestOtelCollectorDefaultConfig:
+ """The chart-rendered config.yml has the default config for the
otel-collector."""
+
+ @staticmethod
+ def _get_config_yml(values=None):
+ docs = render_chart(values=values or {},
show_only=[CONFIGMAP_TEMPLATE])
+ config_yml = jmespath.search('data."config.yml"', docs[0])
+ return yaml.safe_load(config_yml)
+
+ def test_health_check_extension(self):
+ config = self._get_config_yml()
+ assert config["extensions"]["health_check"]["endpoint"] ==
"0.0.0.0:13133"
+
+ def test_otlp_receivers_on_default_ports(self):
+ config = self._get_config_yml()
+ protocols = config["receivers"]["otlp"]["protocols"]
+ assert protocols["grpc"]["endpoint"] == "0.0.0.0:4317"
+ assert protocols["http"]["endpoint"] == "0.0.0.0:4318"
+
+ def test_default_exporters_are_logging_only(self):
+ config = self._get_config_yml()
+ assert set(config["exporters"].keys()) == {"logging"}
+
+ def test_default_pipelines_only_export_to_logging(self):
+ config = self._get_config_yml()
+ pipelines = config["service"]["pipelines"]
+ assert pipelines["traces"]["exporters"] == ["logging"]
+ assert pipelines["metrics"]["exporters"] == ["logging"]
+
+
+class TestOtelCollectorConfigOverride:
+ """`otelCollector.config` replaces the default and supports `tpl`."""
+
+ def test_override_replaces_default(self):
+ override = (
+ "receivers:\n"
+ " otlp:\n"
+ " protocols:\n"
+ " http:\n"
+ " endpoint: 0.0.0.0:9999\n"
+ "exporters:\n"
+ " logging: {}\n"
+ "service:\n"
+ " pipelines:\n"
+ " traces:\n"
+ " receivers: [otlp]\n"
+ " exporters: [logging]\n"
+ )
+ docs = render_chart(
+ values={"otelCollector": {"config": override}},
+ show_only=[CONFIGMAP_TEMPLATE],
+ )
+ rendered = yaml.safe_load(jmespath.search('data."config.yml"',
docs[0]))
+ # The value is '0.0.0.0:9999' instead of the default '0.0.0.0:4318'.
+ assert rendered["receivers"]["otlp"]["protocols"]["http"]["endpoint"]
== "0.0.0.0:9999"
+ # default `health_check` extension is gone in the override
+ assert "extensions" not in rendered
+
+ def test_tpl_resolves_chart_values(self):
+ override = (
+ "receivers:\n"
+ " otlp:\n"
+ " protocols:\n"
+ " http:\n"
+ " endpoint: 0.0.0.0:{{ .Values.ports.otelCollectorOtlpHttp
}}\n"
+ )
+ docs = render_chart(
+ values={"otelCollector": {"config": override}},
+ show_only=[CONFIGMAP_TEMPLATE],
+ )
+ rendered = yaml.safe_load(jmespath.search('data."config.yml"',
docs[0]))
+ assert rendered["receivers"]["otlp"]["protocols"]["http"]["endpoint"]
== "0.0.0.0:4318"
+
+
+class TestOtelCollectorDeployment:
+ """Deployment-level configurability."""
+
+ def test_default_args(self):
+ docs = render_chart(show_only=[DEPLOYMENT_TEMPLATE])
+ assert jmespath.search("spec.template.spec.containers[0].args",
docs[0]) == [
+ "--config=/etc/otel-collector/config.yml"
+ ]
+
+ def test_args_override(self):
+ custom = ["--config=/etc/otel-collector/config.yml",
"--feature-gates=+foo"]
+ docs = render_chart(
+ values={"otelCollector": {"args": custom}},
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ assert jmespath.search("spec.template.spec.containers[0].args",
docs[0]) == custom
+
+ def test_default_probes(self):
+ docs = render_chart(show_only=[DEPLOYMENT_TEMPLATE])
+ liveness =
jmespath.search("spec.template.spec.containers[0].livenessProbe", docs[0])
+ readiness =
jmespath.search("spec.template.spec.containers[0].readinessProbe", docs[0])
+ assert liveness["initialDelaySeconds"] == 10
+ assert liveness["periodSeconds"] == 15
+ assert liveness["httpGet"] == {"path": "/", "port": 13133}
+ assert readiness["initialDelaySeconds"] == 10
+ assert readiness["periodSeconds"] == 15
+
+ def test_probe_overrides(self):
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "livenessProbe": {"initialDelaySeconds": 30,
"periodSeconds": 45},
+ "readinessProbe": {"initialDelaySeconds": 5,
"periodSeconds": 7},
+ }
+ },
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ liveness =
jmespath.search("spec.template.spec.containers[0].livenessProbe", docs[0])
+ readiness =
jmespath.search("spec.template.spec.containers[0].readinessProbe", docs[0])
+ assert liveness["initialDelaySeconds"] == 30
+ assert liveness["periodSeconds"] == 45
+ assert readiness["initialDelaySeconds"] == 5
+ assert readiness["periodSeconds"] == 7
+
+ def test_resources_default_empty(self):
+ docs = render_chart(show_only=[DEPLOYMENT_TEMPLATE])
+ assert jmespath.search("spec.template.spec.containers[0].resources",
docs[0]) == {}
+
+ def test_resources_configurable(self):
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "resources": {
+ "limits": {"cpu": "200m", "memory": "256Mi"},
+ "requests": {"cpu": "100m", "memory": "128Mi"},
+ }
+ }
+ },
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ resources =
jmespath.search("spec.template.spec.containers[0].resources", docs[0])
+ assert resources == {
+ "limits": {"cpu": "200m", "memory": "256Mi"},
+ "requests": {"cpu": "100m", "memory": "128Mi"},
+ }
+
+ @pytest.mark.parametrize(
+ ("values", "expected"),
+ [
+ ({}, 30),
+ ({"otelCollector": {"terminationGracePeriodSeconds": 1200}}, 1200),
+ ],
+ )
+ def test_termination_grace_period_seconds(self, values, expected):
+ docs = render_chart(values=values, show_only=[DEPLOYMENT_TEMPLATE])
+ assert
jmespath.search("spec.template.spec.terminationGracePeriodSeconds", docs[0]) ==
expected
+
+ @pytest.mark.parametrize(
+ ("component", "global_", "expected"),
+ [(8, 10, 8), (10, 8, 10), (8, None, 8), (None, 10, 10), (0, 10, 0),
(None, 0, 0)],
+ )
+ def test_revision_history_limit(self, component, global_, expected):
+ values = {"otelCollector": {}}
+ if component is not None:
+ values["otelCollector"]["revisionHistoryLimit"] = component
+ if global_ is not None:
+ values["revisionHistoryLimit"] = global_
+ docs = render_chart(values=values, show_only=[DEPLOYMENT_TEMPLATE])
+ assert jmespath.search("spec.revisionHistoryLimit", docs[0]) ==
expected
+
+ def test_scheduling_constraints(self):
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "nodeSelector": {"diskType": "ssd"},
+ "tolerations": [{"key": "k", "operator": "Equal", "value":
"v", "effect": "NoSchedule"}],
+ "affinity": {
+ "nodeAffinity": {
+ "requiredDuringSchedulingIgnoredDuringExecution": {
+ "nodeSelectorTerms": [
+ {
+ "matchExpressions": [
+ {"key": "foo", "operator": "In",
"values": ["true"]}
+ ]
+ }
+ ]
+ }
+ }
+ },
+ "topologySpreadConstraints": [
+ {"maxSkew": 1, "topologyKey": "zone",
"whenUnsatisfiable": "DoNotSchedule"}
+ ],
+ "priorityClassName": "high-priority",
+ }
+ },
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ spec = jmespath.search("spec.template.spec", docs[0])
+ assert spec["nodeSelector"] == {"diskType": "ssd"}
+ assert spec["tolerations"][0]["key"] == "k"
+ assert (
+
spec["affinity"]["nodeAffinity"]["requiredDuringSchedulingIgnoredDuringExecution"][
+ "nodeSelectorTerms"
+ ][0]["matchExpressions"][0]["key"]
+ == "foo"
+ )
+ assert spec["topologySpreadConstraints"][0]["topologyKey"] == "zone"
+ assert spec["priorityClassName"] == "high-priority"
+
+ def test_security_contexts(self):
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "securityContexts": {
+ "pod": {"runAsUser": 2000, "fsGroup": 1000},
+ "container": {"allowPrivilegeEscalation": False,
"readOnlyRootFilesystem": True},
+ }
+ }
+ },
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ assert jmespath.search("spec.template.spec.securityContext", docs[0])
== {
+ "runAsUser": 2000,
+ "fsGroup": 1000,
+ }
+ assert
jmespath.search("spec.template.spec.containers[0].securityContext", docs[0]) ==
{
+ "allowPrivilegeEscalation": False,
+ "readOnlyRootFilesystem": True,
+ }
+
+ def test_annotations_and_pod_annotations(self):
+ docs = render_chart(
+ values={
+ "otelCollector": {
+ "annotations": {"deploy_anno": "deploy_value"},
+ "podAnnotations": {"pod_anno": "pod_value"},
+ }
+ },
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ assert jmespath.search("metadata.annotations.deploy_anno", docs[0]) ==
"deploy_value"
+ assert jmespath.search("spec.template.metadata.annotations.pod_anno",
docs[0]) == "pod_value"
+
+ def test_image_override(self):
+ docs = render_chart(
+ values={
+ "images": {
+ "otelCollector": {
+ "repository": "example/custom-otel",
+ "tag": "1.2.3",
+ "pullPolicy": "Always",
+ }
+ }
+ },
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ assert (
+ jmespath.search("spec.template.spec.containers[0].image", docs[0])
== "example/custom-otel:1.2.3"
+ )
+ assert
jmespath.search("spec.template.spec.containers[0].imagePullPolicy", docs[0]) ==
"Always"
+
+
+class TestOtelCollectorService:
+ """Service-level configurability."""
+
+ def test_service_annotations(self):
+ docs = render_chart(
+ values={"otelCollector": {"service": {"annotations": {"some_anno":
"some_value"}}}},
+ show_only=[SERVICE_TEMPLATE],
+ )
+ assert jmespath.search("metadata.annotations.some_anno", docs[0]) ==
"some_value"
+
+ def test_service_no_annotations_block_when_unset(self):
+ docs = render_chart(show_only=[SERVICE_TEMPLATE])
+ assert "annotations" not in jmespath.search("metadata", docs[0])
+
+
+class TestOtelCollectorServiceAccount:
+ """ServiceAccount-level configurability."""
+
+ def test_default_create(self):
+ docs = render_chart(show_only=[SERVICE_ACCOUNT_TEMPLATE])
+ assert docs[0]["kind"] == "ServiceAccount"
+ assert docs[0]["automountServiceAccountToken"] is False
+
+ def test_create_false_suppresses_sa(self):
+ docs = render_chart(
+ values={"otelCollector": {"serviceAccount": {"create": False}}},
+ show_only=[SERVICE_ACCOUNT_TEMPLATE],
+ )
+ assert docs == []
+
+ def test_create_false_deployment_uses_default_sa(self):
+ docs = render_chart(
+ values={"otelCollector": {"serviceAccount": {"create": False}}},
+ show_only=[DEPLOYMENT_TEMPLATE],
+ )
+ assert jmespath.search("spec.template.spec.serviceAccountName",
docs[0]) == "default"
+
+ def test_custom_sa_name_propagates_to_deployment(self):
+ docs = render_chart(
+ values={"otelCollector": {"serviceAccount": {"name":
"my-custom-sa"}}},
+ show_only=[DEPLOYMENT_TEMPLATE, SERVICE_ACCOUNT_TEMPLATE],
+ )
+ sa = next(d for d in docs if d["kind"] == "ServiceAccount")
+ deployment = next(d for d in docs if d["kind"] == "Deployment")
+ assert sa["metadata"]["name"] == "my-custom-sa"
+ assert jmespath.search("spec.template.spec.serviceAccountName",
deployment) == "my-custom-sa"
+
+ def test_automount_token_override(self):
+ docs = render_chart(
+ values={"otelCollector": {"serviceAccount":
{"automountServiceAccountToken": True}}},
+ show_only=[SERVICE_ACCOUNT_TEMPLATE],
+ )
+ assert docs[0]["automountServiceAccountToken"] is True
+
+ def test_sa_annotations(self):
+ docs = render_chart(
+ values={"otelCollector": {"serviceAccount": {"annotations":
{"sa_anno": "sa_value"}}}},
+ show_only=[SERVICE_ACCOUNT_TEMPLATE],
+ )
+ assert jmespath.search("metadata.annotations.sa_anno", docs[0]) ==
"sa_value"
+
+
+class TestOtelCollectorAirflowEnvironment:
+ """The correct OTEL_* env vars can be found on Airflow pods based on the
trace/metrics flags."""
+
+ @pytest.mark.parametrize("template", AIRFLOW_POD_TEMPLATES)
+ def test_default_emits_traces_env_only(self, template):
+ docs = render_chart(show_only=[template])
+ names = _env_names(docs[0])
+ assert "OTEL_SERVICE_NAME" in names
+ assert "OTEL_EXPORTER_OTLP_PROTOCOL" in names
+ assert "OTEL_TRACES_EXPORTER" in names
+ assert "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" in names
+ # Metrics env vars aren't present.
+ assert "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" not in names
+ assert "OTEL_METRIC_EXPORT_INTERVAL" not in names
+
+ @pytest.mark.parametrize("template", AIRFLOW_POD_TEMPLATES)
+ def test_metrics_enabled_emits_metrics_env(self, template):
+ docs = render_chart(
+ values={"otelCollector": {"metricsEnabled": True}},
+ show_only=[template],
+ )
+ names = _env_names(docs[0])
+ assert "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT" in names
+ assert "OTEL_METRIC_EXPORT_INTERVAL" in names
+
+ @pytest.mark.parametrize("template", AIRFLOW_POD_TEMPLATES)
+ def test_both_flags_off_emits_no_otel_env(self, template):
+ docs = render_chart(
+ values={"otelCollector": {"tracesEnabled": False,
"metricsEnabled": False}},
+ show_only=[template],
+ )
+ names = _env_names(docs[0])
+ assert not any(n.startswith("OTEL_") for n in names)
+
+ def test_metric_export_interval_value(self):
+ docs = render_chart(
+ values={"otelCollector": {"metricsEnabled": True,
"metricExportIntervalMs": 12345}},
+ show_only=[AIRFLOW_POD_TEMPLATES[0]],
+ )
+ assert _env_value(docs[0], "OTEL_METRIC_EXPORT_INTERVAL") == "12345"
+
+
+class TestOtelCollectorAirflowConfig:
+ """Renders the correct values in airflow.cfg."""
+
+ @staticmethod
+ def _get_conf_section(cfg: str, name: str) -> str:
+ """Return the body of the specified section (everything until the next
'[')."""
+ return cfg.split(f"[{name}]")[1].split("[")[0]
+
+ @staticmethod
+ def _get_airflow_conf(values=None):
+ docs = render_chart(values=values or {},
show_only=[AIRFLOW_CONFIGMAP_TEMPLATE])
+ return jmespath.search('data."airflow.cfg"', docs[0])
+
+ def test_default_traces_section_has_otel_on_true(self):
+ traces = self._get_conf_section(self._get_airflow_conf(), "traces")
+ assert "otel_on = True" in traces
+
+ def test_default_metrics_section_has_otel_off_and_statsd_on(self):
+ metrics = self._get_conf_section(self._get_airflow_conf(), "metrics")
+ assert "otel_on = False" in metrics
+ assert "statsd_on = True" in metrics
+
+ def test_traces_disabled_otel_off(self):
+ traces = self._get_conf_section(
+ self._get_airflow_conf({"otelCollector": {"tracesEnabled":
False}}), "traces"
+ )
+ assert "otel_on = False" in traces
+
+ def test_metrics_enabled_disables_statsd_and_enables_otel(self):
+ metrics = self._get_conf_section(
+ self._get_airflow_conf({"otelCollector": {"metricsEnabled":
True}}), "metrics"
+ )
+ assert "statsd_on = False" in metrics
+ assert "otel_on = True" in metrics
+
+ def test_metrics_enabled_with_statsd_disabled_still_disables_statsd(self):
+ cfg = self._get_airflow_conf(
+ {"statsd": {"enabled": False}, "otelCollector": {"metricsEnabled":
True}}
+ )
+ metrics_section = cfg.split("[metrics]")[1].split("[")[0]
+ assert "statsd_on = False" in metrics_section
diff --git a/helm-tests/tests/helm_tests/security/test_rbac.py
b/helm-tests/tests/helm_tests/security/test_rbac.py
index 09e44f5ae2f..4a4a7785b09 100644
--- a/helm-tests/tests/helm_tests/security/test_rbac.py
+++ b/helm-tests/tests/helm_tests/security/test_rbac.py
@@ -29,10 +29,12 @@ DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES = [
("Secret", "test-rbac-pgbouncer-stats"),
("ConfigMap", "test-rbac-config"),
("ConfigMap", "test-rbac-statsd"),
+ ("ConfigMap", "test-rbac-otel-collector"),
("Service", "test-rbac-api-server"),
("Service", "test-rbac-postgresql-hl"),
("Service", "test-rbac-postgresql"),
("Service", "test-rbac-statsd"),
+ ("Service", "test-rbac-otel-collector"),
("Service", "test-rbac-flower"),
("Service", "test-rbac-pgbouncer"),
("Service", "test-rbac-redis"),
@@ -42,6 +44,7 @@ DEPLOYMENT_NO_RBAC_NO_SA_KIND_NAME_TUPLES = [
("Deployment", "test-rbac-dag-processor"),
("Deployment", "test-rbac-scheduler"),
("Deployment", "test-rbac-statsd"),
+ ("Deployment", "test-rbac-otel-collector"),
("Deployment", "test-rbac-flower"),
("Deployment", "test-rbac-pgbouncer"),
("StatefulSet", "test-rbac-postgresql"),
@@ -79,6 +82,7 @@ SERVICE_ACCOUNT_NAME_TUPLES = [
("ServiceAccount", "test-rbac-database-cleanup"),
("ServiceAccount", "test-rbac-flower"),
("ServiceAccount", "test-rbac-statsd"),
+ ("ServiceAccount", "test-rbac-otel-collector"),
("ServiceAccount", "test-rbac-create-user-job"),
("ServiceAccount", "test-rbac-migrate-database-job"),
("ServiceAccount", "test-rbac-redis"),
@@ -95,6 +99,7 @@ CUSTOM_SERVICE_ACCOUNT_NAMES = (
(CUSTOM_FLOWER_NAME := "TestFlower"),
(CUSTOM_PGBOUNCER_NAME := "TestPGBouncer"),
(CUSTOM_STATSD_NAME := "TestStatsd"),
+ (CUSTOM_OTEL_COLLECTOR_NAME := "TestOtelCollector"),
(CUSTOM_CREATE_USER_JOBS_NAME := "TestCreateUserJob"),
(CUSTOM_MIGRATE_DATABASE_JOBS_NAME := "TestMigrateDatabaseJob"),
(CUSTOM_REDIS_NAME := "TestRedis"),
@@ -145,6 +150,7 @@ class TestRBAC:
"workers": {"serviceAccount": {"create": False}},
"triggerer": {"serviceAccount": {"create": False}},
"statsd": {"serviceAccount": {"create": False}},
+ "otelCollector": {"serviceAccount": {"create": False}},
"createUserJob": {"serviceAccount": {"create": False}},
"migrateDatabaseJob": {"serviceAccount": {"create": False}},
"flower": {"enabled": True, "serviceAccount": {"create":
False}},
@@ -199,6 +205,7 @@ class TestRBAC:
"triggerer": {"serviceAccount": {"create": False}},
"flower": {"enabled": True, "serviceAccount": {"create":
False}},
"statsd": {"serviceAccount": {"create": False}},
+ "otelCollector": {"serviceAccount": {"create": False}},
"redis": {"serviceAccount": {"create": False}},
"pgbouncer": {
"enabled": True,
@@ -281,6 +288,7 @@ class TestRBAC:
"triggerer": {"serviceAccount": {"name":
CUSTOM_TRIGGERER_NAME}},
"flower": {"enabled": True, "serviceAccount": {"name":
CUSTOM_FLOWER_NAME}},
"statsd": {"serviceAccount": {"name": CUSTOM_STATSD_NAME}},
+ "otelCollector": {"serviceAccount": {"name":
CUSTOM_OTEL_COLLECTOR_NAME}},
"redis": {"serviceAccount": {"name": CUSTOM_REDIS_NAME}},
"postgresql": {"serviceAccount": {"create": True, "name":
CUSTOM_POSTGRESQL_NAME}},
"pgbouncer": {
@@ -325,6 +333,7 @@ class TestRBAC:
"triggerer": {"serviceAccount": {"name":
CUSTOM_TRIGGERER_NAME}},
"flower": {"enabled": True, "serviceAccount": {"name":
CUSTOM_FLOWER_NAME}},
"statsd": {"serviceAccount": {"name": CUSTOM_STATSD_NAME}},
+ "otelCollector": {"serviceAccount": {"name":
CUSTOM_OTEL_COLLECTOR_NAME}},
"redis": {"serviceAccount": {"name": CUSTOM_REDIS_NAME}},
"postgresql": {"serviceAccount": {"name":
CUSTOM_POSTGRESQL_NAME}},
"pgbouncer": {
@@ -363,6 +372,7 @@ class TestRBAC:
"redis": {"enabled": False},
"flower": {"enabled": False},
"statsd": {"enabled": False},
+ "otelCollector": {"tracesEnabled": False, "metricsEnabled":
False},
},
)
list_of_sa_names = [
