This is an automated email from the ASF dual-hosted git repository.
uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 8809478f44 Refactor existence-checking SQL to helper (#32790)
8809478f44 is described below
commit 8809478f444da794254a52b19c4090b2d5726102
Author: Tzu-ping Chung <[email protected]>
AuthorDate: Wed Jul 26 09:48:56 2023 +0800
Refactor existence-checking SQL to helper (#32790)
---
airflow/models/abstractoperator.py | 17 +++++-------
airflow/models/pool.py | 10 +++----
airflow/models/xcom_arg.py | 6 ++--
airflow/ti_deps/deps/prev_dagrun_dep.py | 49 ++++++++++++---------------------
airflow/utils/db.py | 30 ++++++++++++++++++--
5 files changed, 62 insertions(+), 50 deletions(-)
diff --git a/airflow/models/abstractoperator.py
b/airflow/models/abstractoperator.py
index 97e3941f0f..908cb32580 100644
--- a/airflow/models/abstractoperator.py
+++ b/airflow/models/abstractoperator.py
@@ -31,6 +31,7 @@ from airflow.models.expandinput import NotFullyPopulated
from airflow.models.taskmixin import DAGNode, DependencyMixin
from airflow.template.templater import Templater
from airflow.utils.context import Context
+from airflow.utils.db import exists_query
from airflow.utils.log.secrets_masker import redact
from airflow.utils.session import NEW_SESSION, provide_session
from airflow.utils.sqlalchemy import skip_locked, with_row_locks
@@ -593,16 +594,12 @@ class AbstractOperator(Templater, DAGNode):
)
unmapped_ti.state = TaskInstanceState.SKIPPED
else:
- zero_index_ti_exists = (
- session.scalar(
- select(func.count(TaskInstance.task_id)).where(
- TaskInstance.dag_id == self.dag_id,
- TaskInstance.task_id == self.task_id,
- TaskInstance.run_id == run_id,
- TaskInstance.map_index == 0,
- )
- )
- > 0
+ zero_index_ti_exists = exists_query(
+ TaskInstance.dag_id == self.dag_id,
+ TaskInstance.task_id == self.task_id,
+ TaskInstance.run_id == run_id,
+ TaskInstance.map_index == 0,
+ session=session,
)
if not zero_index_ti_exists:
# Otherwise convert this into the first mapped index, and
create
diff --git a/airflow/models/pool.py b/airflow/models/pool.py
index 0aab6fb634..3b5708b121 100644
--- a/airflow/models/pool.py
+++ b/airflow/models/pool.py
@@ -26,6 +26,7 @@ from airflow.exceptions import AirflowException, PoolNotFound
from airflow.models.base import Base
from airflow.ti_deps.dependencies_states import EXECUTION_STATES
from airflow.typing_compat import TypedDict
+from airflow.utils.db import exists_query
from airflow.utils.session import NEW_SESSION, provide_session
from airflow.utils.sqlalchemy import nowait, with_row_locks
from airflow.utils.state import TaskInstanceState
@@ -95,11 +96,10 @@ class Pool(Base):
:param session: SQLAlchemy ORM Session
:return: True if id is default_pool, otherwise False
"""
- return (
- session.scalar(
- select(func.count(Pool.id)).where(Pool.id == id, Pool.pool ==
Pool.DEFAULT_POOL_NAME)
- )
- > 0
+ return exists_query(
+ Pool.id == id,
+ Pool.pool == Pool.DEFAULT_POOL_NAME,
+ session=session,
)
@staticmethod
diff --git a/airflow/models/xcom_arg.py b/airflow/models/xcom_arg.py
index 154878a4db..2383861483 100644
--- a/airflow/models/xcom_arg.py
+++ b/airflow/models/xcom_arg.py
@@ -30,6 +30,7 @@ from airflow.models.baseoperator import BaseOperator
from airflow.models.mappedoperator import MappedOperator
from airflow.models.taskmixin import DAGNode, DependencyMixin
from airflow.utils.context import Context
+from airflow.utils.db import exists_query
from airflow.utils.edgemodifier import EdgeModifier
from airflow.utils.mixins import ResolveMixin
from airflow.utils.session import NEW_SESSION, provide_session
@@ -377,7 +378,7 @@ class PlainXComArg(XComArg):
task = self.operator
if isinstance(task, MappedOperator):
- unfinished_ti_count_query =
session.query(func.count(TaskInstance.map_index)).filter(
+ unfinished_ti_exists = exists_query(
TaskInstance.dag_id == task.dag_id,
TaskInstance.run_id == run_id,
TaskInstance.task_id == task.task_id,
@@ -388,8 +389,9 @@ class PlainXComArg(XComArg):
TaskInstance.state.is_(None),
TaskInstance.state.in_(s.value for s in State.unfinished
if s is not None),
),
+ session=session,
)
- if unfinished_ti_count_query.scalar():
+ if unfinished_ti_exists:
return None # Not all of the expanded tis are done yet.
query = session.query(func.count(XCom.map_index)).filter(
XCom.dag_id == task.dag_id,
diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py
b/airflow/ti_deps/deps/prev_dagrun_dep.py
index a21944d410..0ea192c708 100644
--- a/airflow/ti_deps/deps/prev_dagrun_dep.py
+++ b/airflow/ti_deps/deps/prev_dagrun_dep.py
@@ -19,11 +19,12 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from sqlalchemy import func, literal, or_, select
+from sqlalchemy import func, or_, select
from sqlalchemy.orm import Session
from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
+from airflow.utils.db import exists_query
from airflow.utils.session import provide_session
from airflow.utils.state import TaskInstanceState
@@ -57,13 +58,11 @@ class PrevDagrunDep(BaseTIDep):
This function exists for easy mocking in tests.
"""
- return (
- session.scalar(
- select(literal(True))
- .where(TI.dag_id == dagrun.dag_id, TI.task_id == task_id,
TI.run_id == dagrun.run_id)
- .limit(1)
- )
- is not None
+ return exists_query(
+ TI.dag_id == dagrun.dag_id,
+ TI.task_id == task_id,
+ TI.run_id == dagrun.run_id,
+ session=session,
)
@staticmethod
@@ -72,17 +71,11 @@ class PrevDagrunDep(BaseTIDep):
This function exists for easy mocking in tests.
"""
- return (
- session.scalar(
- select(literal(True))
- .where(
- TI.dag_id == ti.dag_id,
- TI.task_id == ti.task_id,
- TI.execution_date < ti.execution_date,
- )
- .limit(1)
- )
- is not None
+ return exists_query(
+ TI.dag_id == ti.dag_id,
+ TI.task_id == ti.task_id,
+ TI.execution_date < ti.execution_date,
+ session=session,
)
@staticmethod
@@ -116,18 +109,12 @@ class PrevDagrunDep(BaseTIDep):
"""
if not task.downstream_task_ids:
return False
- return (
- session.scalar(
- select(literal(True))
- .where(
- TI.dag_id == dagrun.dag_id,
- TI.task_id.in_(task.downstream_task_ids),
- TI.run_id == dagrun.run_id,
- or_(TI.state.is_(None),
TI.state.not_in(_SUCCESSFUL_STATES)),
- )
- .limit(1)
- )
- is not None
+ return exists_query(
+ TI.dag_id == dagrun.dag_id,
+ TI.task_id.in_(task.downstream_task_ids),
+ TI.run_id == dagrun.run_id,
+ or_(TI.state.is_(None), TI.state.not_in(_SUCCESSFUL_STATES)),
+ session=session,
)
@provide_session
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 8e8c4b7fa9..9ef504e739 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -29,7 +29,21 @@ from dataclasses import dataclass
from tempfile import gettempdir
from typing import TYPE_CHECKING, Callable, Generator, Iterable
-from sqlalchemy import Table, and_, column, delete, exc, func, inspect, or_,
select, table, text, tuple_
+from sqlalchemy import (
+ Table,
+ and_,
+ column,
+ delete,
+ exc,
+ func,
+ inspect,
+ literal,
+ or_,
+ select,
+ table,
+ text,
+ tuple_,
+)
import airflow
from airflow import settings
@@ -45,6 +59,7 @@ if TYPE_CHECKING:
from alembic.runtime.environment import EnvironmentContext
from alembic.script import ScriptDirectory
from sqlalchemy.orm import Query, Session
+ from sqlalchemy.sql.elements import ClauseElement
from sqlalchemy.sql.selectable import Select
from airflow.models.base import Base
@@ -1875,7 +1890,7 @@ def get_sqla_model_classes():
return Base._decl_class_registry.values()
-def get_query_count(query_stmt: Select, session: Session) -> int:
+def get_query_count(query_stmt: Select, *, session: Session) -> int:
"""Get count of query.
A SELECT COUNT() FROM is issued against the subquery built from the
@@ -1887,3 +1902,14 @@ def get_query_count(query_stmt: Select, session:
Session) -> int:
"""
count_stmt =
select(func.count()).select_from(query_stmt.order_by(None).subquery())
return session.scalar(count_stmt)
+
+
+def exists_query(*where: ClauseElement, session: Session) -> bool:
+ """Check whether there is at least one row matching given clause.
+
+ This does a SELECT 1 WHERE ... LIMIT 1 and check the result.
+
+ :meta private:
+ """
+ stmt = select(literal(True)).where(*where).limit(1)
+ return session.scalar(stmt) is not None