This is an automated email from the ASF dual-hosted git repository.
pierrejeambrun pushed a commit to branch v3-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-1-test by this push:
new 11a1eaa9cb5 perf(api): optimize /ui/dags endpoint serialization
(#61483) (#63001)
11a1eaa9cb5 is described below
commit 11a1eaa9cb5d8c8833b7a1cc5b5491f2d44b2821
Author: Pierre Jeambrun <[email protected]>
AuthorDate: Mon Mar 9 10:58:36 2026 +0100
perf(api): optimize /ui/dags endpoint serialization (#61483) (#63001)
This PR addresses a significant performance issue in the /ui/dags endpoint
where page load times scaled poorly with the number of DAGs (12-16 seconds
for just 25 DAGs in our testing).
Two optimizations are implemented:
1. Cache URLSafeSerializer for file_token generation
- Previously, a new URLSafeSerializer was instantiated and
conf.get_mandatory_value() was called for every DAG
- Now uses @lru_cache to create the serializer once and reuse it
2. Eliminate redundant Pydantic validation in response construction
- The original pattern used model_validate -> model_dump ->
model_validate
which caused triple serialization overhead per DAG
- Now validates once with DAGResponse.model_validate(), then uses
model_construct() to build DAGWithLatestDagRunsResponse
Together, these changes reduced page load time from 12-16 seconds to
~130ms in our dev environment.
(cherry picked from commit a915216434cf583b0b54eee129cc9a93b96492b2)
Co-authored-by: john-rodriguez-mgni
<[email protected]>
Co-authored-by: Cursor <[email protected]>
---
.../airflow/api_fastapi/core_api/datamodels/dags.py | 17 +++++++++++++++--
.../airflow/api_fastapi/core_api/routes/ui/dags.py | 19 +++++++++++++------
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
index 4cd3ed93817..294092c6639 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
@@ -20,6 +20,7 @@ from __future__ import annotations
import inspect
from collections import abc
from datetime import datetime, timedelta
+from functools import cache
from typing import Any
from itsdangerous import URLSafeSerializer
@@ -38,6 +39,19 @@ from airflow.api_fastapi.core_api.datamodels.dag_versions
import DagVersionRespo
from airflow.configuration import conf
from airflow.models.dag_version import DagVersion
+
+@cache
+def _get_file_token_serializer() -> URLSafeSerializer:
+ """
+ Return a cached URLSafeSerializer instance.
+
+ Uses @cache for lazy initialization - the serializer is created on first
+ call rather than at module import time. This avoids issues if the module
+ is imported before configuration is fully loaded.
+ """
+ return URLSafeSerializer(conf.get_mandatory_value("api", "secret_key"))
+
+
DAG_ALIAS_MAPPING: dict[str, str] = {
# The keys are the names in the response, the values are the original
names in the model
# This is used to map the names in the response to the names in the model
@@ -113,12 +127,11 @@ class DAGResponse(BaseModel):
@property
def file_token(self) -> str:
"""Return file token."""
- serializer = URLSafeSerializer(conf.get_mandatory_value("api",
"secret_key"))
payload = {
"bundle_name": self.bundle_name,
"relative_fileloc": self.relative_fileloc,
}
- return serializer.dumps(payload)
+ return _get_file_token_serializer().dumps(payload)
class DAGPatchBody(StrictBaseModel):
diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
index e1af93e746e..b404534cc7f 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
@@ -52,7 +52,7 @@ from airflow.api_fastapi.common.parameters import (
filter_param_factory,
)
from airflow.api_fastapi.common.router import AirflowRouter
-from airflow.api_fastapi.core_api.datamodels.dags import DAGResponse
+from airflow.api_fastapi.core_api.datamodels.dags import DAG_ALIAS_MAPPING,
DAGResponse
from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import
DAGRunLightResponse
from airflow.api_fastapi.core_api.datamodels.ui.dags import (
DAGWithLatestDagRunsCollectionResponse,
@@ -234,18 +234,25 @@ def get_dags(
pending_actions_by_dag_id[dag_id].append(hitl_detail)
# aggregate rows by dag_id
- dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = {
- dag.dag_id: DAGWithLatestDagRunsResponse.model_validate(
+ # Build the dict dynamically from DAGResponse.model_fields so that new
fields
+ # added to DAGResponse are picked up automatically without code changes
here.
+ dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = {}
+ for dag in dags:
+ dag_data = {
+ DAG_ALIAS_MAPPING.get(field_name, field_name): getattr(
+ dag, DAG_ALIAS_MAPPING.get(field_name, field_name)
+ )
+ for field_name in DAGResponse.model_fields
+ }
+ dag_data.update(
{
- **DAGResponse.model_validate(dag).model_dump(),
"asset_expression": dag.asset_expression,
"latest_dag_runs": [],
"pending_actions": pending_actions_by_dag_id[dag.dag_id],
"is_favorite": dag.dag_id in favorite_dag_ids,
}
)
- for dag in dags
- }
+ dag_runs_by_dag_id[dag.dag_id] =
DAGWithLatestDagRunsResponse.model_validate(dag_data)
for row in recent_dag_runs:
dag_run_response = DAGRunLightResponse.model_validate(row)
