This is an automated email from the ASF dual-hosted git repository. rahulvats pushed a commit to branch v3-1-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b3672eae00bcc66bfbc0fb6e2aab579e67d75719 Author: Pierre Jeambrun <[email protected]> AuthorDate: Mon Mar 9 10:58:36 2026 +0100 perf(api): optimize /ui/dags endpoint serialization (#61483) (#63001) This PR addresses a significant performance issue in the /ui/dags endpoint where page load times scaled poorly with the number of DAGs (12-16 seconds for just 25 DAGs in our testing). Two optimizations are implemented: 1. Cache URLSafeSerializer for file_token generation - Previously, a new URLSafeSerializer was instantiated and conf.get_mandatory_value() was called for every DAG - Now uses @lru_cache to create the serializer once and reuse it 2. Eliminate redundant Pydantic validation in response construction - The original pattern used model_validate -> model_dump -> model_validate which caused triple serialization overhead per DAG - Now validates once with DAGResponse.model_validate(), then uses model_construct() to build DAGWithLatestDagRunsResponse Together, these changes reduced page load time from 12-16 seconds to ~130ms in our dev environment. (cherry picked from commit a915216434cf583b0b54eee129cc9a93b96492b2) Co-authored-by: john-rodriguez-mgni <[email protected]> Co-authored-by: Cursor <[email protected]> --- .../airflow/api_fastapi/core_api/datamodels/dags.py | 17 +++++++++++++++-- .../airflow/api_fastapi/core_api/routes/ui/dags.py | 19 +++++++++++++------ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py index 4cd3ed93817..294092c6639 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py @@ -20,6 +20,7 @@ from __future__ import annotations import inspect from collections import abc from datetime import datetime, timedelta +from functools import cache from typing import Any from itsdangerous import URLSafeSerializer @@ -38,6 +39,19 @@ from airflow.api_fastapi.core_api.datamodels.dag_versions import DagVersionRespo from airflow.configuration import conf from airflow.models.dag_version import DagVersion + +@cache +def _get_file_token_serializer() -> URLSafeSerializer: + """ + Return a cached URLSafeSerializer instance. + + Uses @cache for lazy initialization - the serializer is created on first + call rather than at module import time. This avoids issues if the module + is imported before configuration is fully loaded. + """ + return URLSafeSerializer(conf.get_mandatory_value("api", "secret_key")) + + DAG_ALIAS_MAPPING: dict[str, str] = { # The keys are the names in the response, the values are the original names in the model # This is used to map the names in the response to the names in the model @@ -113,12 +127,11 @@ class DAGResponse(BaseModel): @property def file_token(self) -> str: """Return file token.""" - serializer = URLSafeSerializer(conf.get_mandatory_value("api", "secret_key")) payload = { "bundle_name": self.bundle_name, "relative_fileloc": self.relative_fileloc, } - return serializer.dumps(payload) + return _get_file_token_serializer().dumps(payload) class DAGPatchBody(StrictBaseModel): diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py index e1af93e746e..b404534cc7f 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py @@ -52,7 +52,7 @@ from airflow.api_fastapi.common.parameters import ( filter_param_factory, ) from airflow.api_fastapi.common.router import AirflowRouter -from airflow.api_fastapi.core_api.datamodels.dags import DAGResponse +from airflow.api_fastapi.core_api.datamodels.dags import DAG_ALIAS_MAPPING, DAGResponse from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import DAGRunLightResponse from airflow.api_fastapi.core_api.datamodels.ui.dags import ( DAGWithLatestDagRunsCollectionResponse, @@ -234,18 +234,25 @@ def get_dags( pending_actions_by_dag_id[dag_id].append(hitl_detail) # aggregate rows by dag_id - dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = { - dag.dag_id: DAGWithLatestDagRunsResponse.model_validate( + # Build the dict dynamically from DAGResponse.model_fields so that new fields + # added to DAGResponse are picked up automatically without code changes here. + dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = {} + for dag in dags: + dag_data = { + DAG_ALIAS_MAPPING.get(field_name, field_name): getattr( + dag, DAG_ALIAS_MAPPING.get(field_name, field_name) + ) + for field_name in DAGResponse.model_fields + } + dag_data.update( { - **DAGResponse.model_validate(dag).model_dump(), "asset_expression": dag.asset_expression, "latest_dag_runs": [], "pending_actions": pending_actions_by_dag_id[dag.dag_id], "is_favorite": dag.dag_id in favorite_dag_ids, } ) - for dag in dags - } + dag_runs_by_dag_id[dag.dag_id] = DAGWithLatestDagRunsResponse.model_validate(dag_data) for row in recent_dag_runs: dag_run_response = DAGRunLightResponse.model_validate(row)
