This is an automated email from the ASF dual-hosted git repository.

hussein-awala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 4f459eab707 Report Datadog timer metrics in milliseconds (#74848)
4f459eab707 is described below

commit 4f459eab7073566b783fdc0fff08f2ca291548e4
Author: Timothé Barbe <[email protected]>
AuthorDate: Thu Nov 19 16:04:04 2026 +0100

    Report Datadog timer metrics in milliseconds (#74848)
    
    * Report Datadog timer metrics in milliseconds
    
    Stats.timer() metrics are documented in milliseconds, but the Datadog
    backend relied on DogStatsd.timed(), which reports seconds unless the
    client enables use_ms, and then rounds to whole milliseconds. Datadog
    users saw timers such as scheduler.scheduler_loop_duration 1000x smaller
    than on StatsD or OpenTelemetry, and sub-millisecond timers reported as 0.
    
    * Add newsfragment for Datadog timer units
    
    * Fail fast on unsupported Datadog timer arguments
    
    Arguments that DogStatsd.timing() does not accept used to surface as a
    TypeError when the timer stopped, from the with block's __exit__, which
    could hide an exception raised by the timed code. Stopping a timer that
    was never started also raised AttributeError. The Datadog timer change
    breaks existing dashboards and monitors, so it belongs in the upgrade notes.
    
    * Share the timer that emits its duration between metric backends
    
    The Datadog and OpenTelemetry timers duplicated the logic deciding when a
    measured duration is sent, so a fix to one could easily miss the other.
---
 airflow-core/newsfragments/74848.significant.rst   | 18 +++++++++++
 .../observability/metrics/datadog_logger.py        | 12 +++++--
 .../observability/metrics/otel_logger.py           | 37 ++++++----------------
 .../observability/metrics/protocols.py             | 19 +++++++++--
 .../tests/observability/metrics/test_stats.py      | 32 +++++++++++++++----
 5 files changed, 78 insertions(+), 40 deletions(-)

diff --git a/airflow-core/newsfragments/74848.significant.rst 
b/airflow-core/newsfragments/74848.significant.rst
new file mode 100644
index 00000000000..07ef185c2b4
--- /dev/null
+++ b/airflow-core/newsfragments/74848.significant.rst
@@ -0,0 +1,18 @@
+Datadog timer metrics are now reported in milliseconds
+
+With the Datadog (DogStatsD) metrics backend, ``Stats.timer()`` metrics such as
+``scheduler.scheduler_loop_duration`` were sent in seconds, while they are 
documented in milliseconds
+and reported in milliseconds by the StatsD and OpenTelemetry backends. They 
are now sent in
+milliseconds, so their values become 1000x larger after upgrading. Update 
Datadog dashboards, monitors
+and SLOs that use these timers accordingly.
+
+* Types of change
+
+  * [ ] Dag changes
+  * [ ] Config changes
+  * [ ] API changes
+  * [ ] CLI changes
+  * [x] Behaviour changes
+  * [ ] Plugin changes
+  * [ ] Dependency changes
+  * [ ] Code interface changes
diff --git 
a/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
 
b/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
index efe621b4104..3d35bcd34e1 100644
--- 
a/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
+++ 
b/shared/observability/src/airflow_shared/observability/metrics/datadog_logger.py
@@ -22,7 +22,7 @@ import logging
 from collections.abc import Callable
 from typing import TYPE_CHECKING, Any
 
-from .protocols import Timer
+from .protocols import DurationEmittingTimer, Timer
 from .validators import (
     PatternAllowListValidator,
     PatternBlockListValidator,
@@ -134,12 +134,18 @@ class SafeDogStatsdLogger:
         self,
         stat: str | None = None,
         tags: dict[str, str] | None = None,
-        **kwargs,
+        sample_rate: float | None = None,
     ) -> Timer:
         """Timer metric that can be cancelled."""
         tags_list = self._build_tags_list(tags)
         if stat and self.metrics_validator.test(stat):
-            return Timer(self.dogstatsd.timed(stat, tags=tags_list, **kwargs))
+            # DogStatsd.timed() reports seconds unless the client sets use_ms, 
and rounds to whole
+            # milliseconds when it does, so the duration measured by Timer is 
sent instead.
+            return DurationEmittingTimer(
+                lambda duration: self.dogstatsd.timing(
+                    metric=stat, value=duration, tags=tags_list, 
sample_rate=sample_rate
+                )
+            )
         return Timer()
 
 
diff --git 
a/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py 
b/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
index bc8b819f144..e3c32562153 100644
--- 
a/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
+++ 
b/shared/observability/src/airflow_shared/observability/metrics/otel_logger.py
@@ -37,7 +37,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME, Resource
 from ..common import get_otel_data_exporter
 from ..exceptions import InvalidStatsNameException
 from ..otel_env_config import load_metrics_env_config
-from .protocols import Timer
+from .protocols import DurationEmittingTimer, Timer
 from .validators import (
     OTEL_NAME_MAX_LENGTH,
     ListValidator,
@@ -154,31 +154,6 @@ def _skip_due_to_rate(rate: float) -> bool:
     return rate < 1 and random.random() > rate
 
 
-class _OtelTimer(Timer):
-    """
-    An implementation of stats.Timer() which records the result in the OTel 
Metrics Map.
-
-    OpenTelemetry does not have a native timer; values are stored as a 
Histogram so that
-    all observations (count, sum, bucket distribution) are preserved across 
multiple recordings.
-
-    :param name: The name of the timer.
-    :param tags: Tags to append to the timer.
-    """
-
-    def __init__(self, otel_logger: SafeOtelLogger, name: str | None, tags: 
Attributes):
-        super().__init__()
-        self.otel_logger = otel_logger
-        self.name = name
-        self.tags = tags
-
-    def stop(self, send: bool = True) -> None:
-        super().stop(send)
-        if self.name and send and self.duration is not None:
-            self.otel_logger.metrics_map.record_histogram_value(
-                full_name(prefix=self.otel_logger.prefix, name=self.name), 
self.duration, self.tags
-            )
-
-
 class SafeOtelLogger:
     """Otel Logger."""
 
@@ -317,8 +292,14 @@ class SafeOtelLogger:
         **kwargs,
     ) -> Timer:
         """Timer context manager returns the duration and can be cancelled."""
-        safe_stat = stat if self._is_recordable(stat) else None
-        return _OtelTimer(self, safe_stat, tags)
+        if not (stat and self._is_recordable(stat)):
+            return Timer()
+        # OpenTelemetry has no native timer; a Histogram preserves count, sum 
and bucket distribution.
+        return DurationEmittingTimer(
+            lambda duration: self.metrics_map.record_histogram_value(
+                full_name(prefix=self.prefix, name=stat), duration, tags
+            )
+        )
 
 
 class InternalGauge:
diff --git 
a/shared/observability/src/airflow_shared/observability/metrics/protocols.py 
b/shared/observability/src/airflow_shared/observability/metrics/protocols.py
index fad6e6a570b..9b71b9f2fff 100644
--- a/shared/observability/src/airflow_shared/observability/metrics/protocols.py
+++ b/shared/observability/src/airflow_shared/observability/metrics/protocols.py
@@ -22,6 +22,8 @@ import time
 from typing import TYPE_CHECKING, Protocol
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from .. import Self
 
 DeltaType = int | float | datetime.timedelta
@@ -105,8 +107,8 @@ class Timer(TimerProtocol):
     # pystatsd and dogstatsd both have a timer class, but present different API
     # so we can't use this as a mixin on those, instead this class contains 
the "real" timer
 
-    _start_time: float | None
-    duration: float | None
+    _start_time: float | None = None
+    duration: float | None = None
 
     def __init__(self, real_timer: BackendTimerProtocol | None = None) -> None:
         self.real_timer = real_timer
@@ -130,3 +132,16 @@ class Timer(TimerProtocol):
             self.duration = 1000.0 * (time.perf_counter() - self._start_time)  
# Convert to milliseconds.
         if send and self.real_timer:
             self.real_timer.stop()
+
+
+class DurationEmittingTimer(Timer):
+    """Timer that passes its measured duration, in milliseconds, to ``emit`` 
when stopped."""
+
+    def __init__(self, emit: Callable[[float], object]) -> None:
+        super().__init__()
+        self._emit = emit
+
+    def stop(self, send: bool = True) -> None:
+        super().stop(send)
+        if send and self.duration is not None:
+            self._emit(self.duration)
diff --git a/shared/observability/tests/observability/metrics/test_stats.py 
b/shared/observability/tests/observability/metrics/test_stats.py
index c2a4a931021..d9fe1ee4504 100644
--- a/shared/observability/tests/observability/metrics/test_stats.py
+++ b/shared/observability/tests/observability/metrics/test_stats.py
@@ -253,19 +253,37 @@ class TestDogStats:
             metric="empty_key", sample_rate=1, tags=[], value=1
         )
 
-    @mock.patch.object(time, "perf_counter", side_effect=[0.0, 100.0])
-    def test_timer(self, time_mock):
+    @mock.patch.object(time, "perf_counter", side_effect=[0.0, 0.0004])
+    def test_timer_sends_duration_in_milliseconds(self, time_mock):
         with self.dogstatsd.timer("empty_timer") as timer:
             pass
-        self.dogstatsd_client.timed.assert_called_once_with("empty_timer", 
tags=[])
-        expected_duration = 1000.0 * 100.0
-        assert expected_duration == timer.duration
-        assert time_mock.call_count == 2
+        assert timer.duration == pytest.approx(0.4)
+        self.dogstatsd_client.timing.assert_called_once_with(
+            metric="empty_timer", value=pytest.approx(0.4), tags=[], 
sample_rate=None
+        )
+        self.dogstatsd_client.timed.assert_not_called()
+
+    @mock.patch.object(time, "perf_counter", side_effect=[0.0, 1.0])
+    def test_timer_stop_without_send(self, time_mock):
+        timer = self.dogstatsd.timer("empty_timer").start()
+        timer.stop(send=False)
+        assert timer.duration == 1000.0
+        self.dogstatsd_client.timing.assert_not_called()
+
+    def test_timer_stop_without_start(self):
+        timer = self.dogstatsd.timer("empty_timer")
+        timer.stop()
+        assert timer.duration is None
+        self.dogstatsd_client.timing.assert_not_called()
+
+    def test_timer_rejects_unsupported_arguments_on_creation(self):
+        with pytest.raises(TypeError):
+            self.dogstatsd.timer("empty_timer", use_ms=True)
 
     def test_empty_timer(self):
         with self.dogstatsd.timer():
             pass
-        self.dogstatsd_client.timed.assert_not_called()
+        self.dogstatsd_client.timing.assert_not_called()
 
     def test_timing(self):
         import datetime

Reply via email to