o-nikolas commented on code in PR #30873: URL: https://github.com/apache/airflow/pull/30873#discussion_r1180856122
########## tests/core/test_otel_logger.py: ########## @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest +from opentelemetry.metrics import MeterProvider + +from airflow.metrics.otel_logger import ( + METRIC_NAME_PREFIX, + OTEL_NAME_MAX_LENGTH, + UP_DOWN_COUNTERS, + SafeOtelLogger, + _is_up_down_counter, +) + + +def full_name(name: str): + return f"{METRIC_NAME_PREFIX}{name}" + + [email protected] +def name(): + return "test_stats_run" + + +class TestOtelMetrics: + def setup_method(self): + self.meter = mock.Mock(MeterProvider) + self.stats = SafeOtelLogger(otel_provider=self.meter) + self.map = self.stats.metrics_map.map + + def test_is_up_down_counter_positive(self): + udc = next(iter(UP_DOWN_COUNTERS)) + + assert _is_up_down_counter(udc) + + def test_is_up_down_counter_negative(self): + assert not _is_up_down_counter("this_is_not_a_udc") + + def test_stat_name_must_be_a_string(self): + self.stats.incr(42) + + self.meter.assert_not_called() + + def test_stat_name_must_not_exceed_max_length(self): + self.stats.incr("X" * OTEL_NAME_MAX_LENGTH) + + self.meter.assert_not_called() + + def test_stat_name_must_only_include_allowed_characters(self): + self.stats.incr("test/$tats") + + self.meter.assert_not_called() + + def test_incr_new_metric(self, name): + self.stats.incr(name) + + self.meter.get_meter().create_counter.assert_called_once_with(name=full_name(name)) + + def test_incr_new_metric_with_tags(self, name): + tags = {"hello": "world"} + + self.stats.incr(name, tags=tags) + + self.meter.get_meter().create_counter.assert_called_once_with(name=full_name(name)) + self.map[full_name(name)].add.assert_called_once_with(1, attributes=tags) + + def test_incr_existing_metric(self, name): + # Create the metric and set value to 1 + self.stats.incr(name) + # Increment value to 2 + self.stats.incr(name) + + assert self.map[full_name(name)].add.call_count == 2 + + def test_incr_with_back_compat_name(self, name): + back_compat_name = f"back_compat_{name}" + expected_calls = [ + mock.call(name=full_name(back_compat_name)), + mock.call(name=full_name(name)), + ] + + self.stats.incr(name, back_compat_name=back_compat_name) + + self.meter.get_meter().create_counter.assert_has_calls(expected_calls, any_order=True) + assert self.meter.get_meter().create_counter.call_count == len(expected_calls) + + @mock.patch("random.random", side_effect=[0.1, 0.9]) Review Comment: Nice use of multiple side effects :+1: ########## tests/core/test_otel_logger.py: ########## @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest +from opentelemetry.metrics import MeterProvider + +from airflow.metrics.otel_logger import ( + METRIC_NAME_PREFIX, + OTEL_NAME_MAX_LENGTH, + UP_DOWN_COUNTERS, + SafeOtelLogger, + _is_up_down_counter, +) + + +def full_name(name: str): + return f"{METRIC_NAME_PREFIX}{name}" + + [email protected] +def name(): + return "test_stats_run" + + +class TestOtelMetrics: + def setup_method(self): + self.meter = mock.Mock(MeterProvider) + self.stats = SafeOtelLogger(otel_provider=self.meter) + self.map = self.stats.metrics_map.map + + def test_is_up_down_counter_positive(self): + udc = next(iter(UP_DOWN_COUNTERS)) + + assert _is_up_down_counter(udc) + + def test_is_up_down_counter_negative(self): + assert not _is_up_down_counter("this_is_not_a_udc") + + def test_stat_name_must_be_a_string(self): + self.stats.incr(42) Review Comment: Shouldn't this and the other incorrect name cases throw an exception? ########## airflow/metrics/otel_logger.py: ########## @@ -0,0 +1,261 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +import random +import warnings +from typing import Callable + +from opentelemetry import metrics +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics._internal.export import ConsoleMetricExporter, PeriodicExportingMetricReader +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.util.types import Attributes + +from airflow.configuration import conf +from airflow.metrics.protocols import DeltaType, Timer, TimerProtocol +from airflow.metrics.validators import AllowListValidator, validate_stat + +log = logging.getLogger(__name__) + + +# This is currently the only UDC used. If more are added, we should add a better system for this. +UP_DOWN_COUNTERS = {"airflow.dag_processing.processes"} +OTEL_NAME_MAX_LENGTH = 63 +METRIC_NAME_PREFIX = "airflow." + + +def _is_up_down_counter(name): + return name in UP_DOWN_COUNTERS + + +def _generate_key_name(name: str, attributes: Attributes = None): + return f"{name}{'_' + str(attributes) if attributes else ''}" + + +class SafeOtelLogger: + """Otel Logger""" + + def __init__(self, otel_provider, prefix: str = "airflow", allow_list_validator=AllowListValidator()): + self.otel: Callable = otel_provider + self.prefix: str = prefix + self.metrics_validator = allow_list_validator + self.meter = otel_provider.get_meter(__name__) + self.metrics_map = MetricsMap(self.meter) + + @validate_stat + def incr( + self, + stat: str, + count: int = 1, + rate: float = 1, + tags: Attributes = None, + *, + back_compat_name: str = "", + ): + """ + Increment stat by count. + + :param stat: The name of the stat to increment. + :param count: A positive integer to add to the current value of stat. + :param rate: value between 0 and 1 that represents the sampled rate at + which the metric is going to be emitted. + :param tags: Tags to append to the stat. + :param back_compat_name: If possible, the metric will also be emitted + under this name for backward compatibility. + """ + if (count < 0) or (rate < 0): + raise ValueError("count and rate must both be positive values.") + if rate < 1 and random.random() > rate: + return + + if back_compat_name and self.metrics_validator.test(back_compat_name): Review Comment: Does the metrics_validator do the assertion of the name length for otel's short naming? Otherwise it's going to blow up when it hits the hotel code. ########## airflow/metrics/otel_logger.py: ########## @@ -0,0 +1,261 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +import random +import warnings +from typing import Callable + +from opentelemetry import metrics +from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics._internal.export import ConsoleMetricExporter, PeriodicExportingMetricReader +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.util.types import Attributes + +from airflow.configuration import conf +from airflow.metrics.protocols import DeltaType, Timer, TimerProtocol +from airflow.metrics.validators import AllowListValidator, validate_stat + +log = logging.getLogger(__name__) + + +# This is currently the only UDC used. If more are added, we should add a better system for this. +UP_DOWN_COUNTERS = {"airflow.dag_processing.processes"} +OTEL_NAME_MAX_LENGTH = 63 +METRIC_NAME_PREFIX = "airflow." + + +def _is_up_down_counter(name): + return name in UP_DOWN_COUNTERS + + +def _generate_key_name(name: str, attributes: Attributes = None): + return f"{name}{'_' + str(attributes) if attributes else ''}" + + +class SafeOtelLogger: + """Otel Logger""" + + def __init__(self, otel_provider, prefix: str = "airflow", allow_list_validator=AllowListValidator()): + self.otel: Callable = otel_provider + self.prefix: str = prefix + self.metrics_validator = allow_list_validator + self.meter = otel_provider.get_meter(__name__) + self.metrics_map = MetricsMap(self.meter) + + @validate_stat + def incr( + self, + stat: str, + count: int = 1, + rate: float = 1, + tags: Attributes = None, + *, + back_compat_name: str = "", + ): + """ + Increment stat by count. + + :param stat: The name of the stat to increment. + :param count: A positive integer to add to the current value of stat. + :param rate: value between 0 and 1 that represents the sampled rate at + which the metric is going to be emitted. + :param tags: Tags to append to the stat. + :param back_compat_name: If possible, the metric will also be emitted Review Comment: When would it not be possible to use the back compat name? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
