This is an automated email from the ASF dual-hosted git repository.

uranusjr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 8809478f44 Refactor existence-checking SQL to helper (#32790)
8809478f44 is described below

commit 8809478f444da794254a52b19c4090b2d5726102
Author: Tzu-ping Chung <[email protected]>
AuthorDate: Wed Jul 26 09:48:56 2023 +0800

    Refactor existence-checking SQL to helper (#32790)
---
 airflow/models/abstractoperator.py      | 17 +++++-------
 airflow/models/pool.py                  | 10 +++----
 airflow/models/xcom_arg.py              |  6 ++--
 airflow/ti_deps/deps/prev_dagrun_dep.py | 49 ++++++++++++---------------------
 airflow/utils/db.py                     | 30 ++++++++++++++++++--
 5 files changed, 62 insertions(+), 50 deletions(-)

diff --git a/airflow/models/abstractoperator.py 
b/airflow/models/abstractoperator.py
index 97e3941f0f..908cb32580 100644
--- a/airflow/models/abstractoperator.py
+++ b/airflow/models/abstractoperator.py
@@ -31,6 +31,7 @@ from airflow.models.expandinput import NotFullyPopulated
 from airflow.models.taskmixin import DAGNode, DependencyMixin
 from airflow.template.templater import Templater
 from airflow.utils.context import Context
+from airflow.utils.db import exists_query
 from airflow.utils.log.secrets_masker import redact
 from airflow.utils.session import NEW_SESSION, provide_session
 from airflow.utils.sqlalchemy import skip_locked, with_row_locks
@@ -593,16 +594,12 @@ class AbstractOperator(Templater, DAGNode):
                 )
                 unmapped_ti.state = TaskInstanceState.SKIPPED
             else:
-                zero_index_ti_exists = (
-                    session.scalar(
-                        select(func.count(TaskInstance.task_id)).where(
-                            TaskInstance.dag_id == self.dag_id,
-                            TaskInstance.task_id == self.task_id,
-                            TaskInstance.run_id == run_id,
-                            TaskInstance.map_index == 0,
-                        )
-                    )
-                    > 0
+                zero_index_ti_exists = exists_query(
+                    TaskInstance.dag_id == self.dag_id,
+                    TaskInstance.task_id == self.task_id,
+                    TaskInstance.run_id == run_id,
+                    TaskInstance.map_index == 0,
+                    session=session,
                 )
                 if not zero_index_ti_exists:
                     # Otherwise convert this into the first mapped index, and 
create
diff --git a/airflow/models/pool.py b/airflow/models/pool.py
index 0aab6fb634..3b5708b121 100644
--- a/airflow/models/pool.py
+++ b/airflow/models/pool.py
@@ -26,6 +26,7 @@ from airflow.exceptions import AirflowException, PoolNotFound
 from airflow.models.base import Base
 from airflow.ti_deps.dependencies_states import EXECUTION_STATES
 from airflow.typing_compat import TypedDict
+from airflow.utils.db import exists_query
 from airflow.utils.session import NEW_SESSION, provide_session
 from airflow.utils.sqlalchemy import nowait, with_row_locks
 from airflow.utils.state import TaskInstanceState
@@ -95,11 +96,10 @@ class Pool(Base):
         :param session: SQLAlchemy ORM Session
         :return: True if id is default_pool, otherwise False
         """
-        return (
-            session.scalar(
-                select(func.count(Pool.id)).where(Pool.id == id, Pool.pool == 
Pool.DEFAULT_POOL_NAME)
-            )
-            > 0
+        return exists_query(
+            Pool.id == id,
+            Pool.pool == Pool.DEFAULT_POOL_NAME,
+            session=session,
         )
 
     @staticmethod
diff --git a/airflow/models/xcom_arg.py b/airflow/models/xcom_arg.py
index 154878a4db..2383861483 100644
--- a/airflow/models/xcom_arg.py
+++ b/airflow/models/xcom_arg.py
@@ -30,6 +30,7 @@ from airflow.models.baseoperator import BaseOperator
 from airflow.models.mappedoperator import MappedOperator
 from airflow.models.taskmixin import DAGNode, DependencyMixin
 from airflow.utils.context import Context
+from airflow.utils.db import exists_query
 from airflow.utils.edgemodifier import EdgeModifier
 from airflow.utils.mixins import ResolveMixin
 from airflow.utils.session import NEW_SESSION, provide_session
@@ -377,7 +378,7 @@ class PlainXComArg(XComArg):
 
         task = self.operator
         if isinstance(task, MappedOperator):
-            unfinished_ti_count_query = 
session.query(func.count(TaskInstance.map_index)).filter(
+            unfinished_ti_exists = exists_query(
                 TaskInstance.dag_id == task.dag_id,
                 TaskInstance.run_id == run_id,
                 TaskInstance.task_id == task.task_id,
@@ -388,8 +389,9 @@ class PlainXComArg(XComArg):
                     TaskInstance.state.is_(None),
                     TaskInstance.state.in_(s.value for s in State.unfinished 
if s is not None),
                 ),
+                session=session,
             )
-            if unfinished_ti_count_query.scalar():
+            if unfinished_ti_exists:
                 return None  # Not all of the expanded tis are done yet.
             query = session.query(func.count(XCom.map_index)).filter(
                 XCom.dag_id == task.dag_id,
diff --git a/airflow/ti_deps/deps/prev_dagrun_dep.py 
b/airflow/ti_deps/deps/prev_dagrun_dep.py
index a21944d410..0ea192c708 100644
--- a/airflow/ti_deps/deps/prev_dagrun_dep.py
+++ b/airflow/ti_deps/deps/prev_dagrun_dep.py
@@ -19,11 +19,12 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from sqlalchemy import func, literal, or_, select
+from sqlalchemy import func, or_, select
 from sqlalchemy.orm import Session
 
 from airflow.models.taskinstance import PAST_DEPENDS_MET, TaskInstance as TI
 from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
+from airflow.utils.db import exists_query
 from airflow.utils.session import provide_session
 from airflow.utils.state import TaskInstanceState
 
@@ -57,13 +58,11 @@ class PrevDagrunDep(BaseTIDep):
 
         This function exists for easy mocking in tests.
         """
-        return (
-            session.scalar(
-                select(literal(True))
-                .where(TI.dag_id == dagrun.dag_id, TI.task_id == task_id, 
TI.run_id == dagrun.run_id)
-                .limit(1)
-            )
-            is not None
+        return exists_query(
+            TI.dag_id == dagrun.dag_id,
+            TI.task_id == task_id,
+            TI.run_id == dagrun.run_id,
+            session=session,
         )
 
     @staticmethod
@@ -72,17 +71,11 @@ class PrevDagrunDep(BaseTIDep):
 
         This function exists for easy mocking in tests.
         """
-        return (
-            session.scalar(
-                select(literal(True))
-                .where(
-                    TI.dag_id == ti.dag_id,
-                    TI.task_id == ti.task_id,
-                    TI.execution_date < ti.execution_date,
-                )
-                .limit(1)
-            )
-            is not None
+        return exists_query(
+            TI.dag_id == ti.dag_id,
+            TI.task_id == ti.task_id,
+            TI.execution_date < ti.execution_date,
+            session=session,
         )
 
     @staticmethod
@@ -116,18 +109,12 @@ class PrevDagrunDep(BaseTIDep):
         """
         if not task.downstream_task_ids:
             return False
-        return (
-            session.scalar(
-                select(literal(True))
-                .where(
-                    TI.dag_id == dagrun.dag_id,
-                    TI.task_id.in_(task.downstream_task_ids),
-                    TI.run_id == dagrun.run_id,
-                    or_(TI.state.is_(None), 
TI.state.not_in(_SUCCESSFUL_STATES)),
-                )
-                .limit(1)
-            )
-            is not None
+        return exists_query(
+            TI.dag_id == dagrun.dag_id,
+            TI.task_id.in_(task.downstream_task_ids),
+            TI.run_id == dagrun.run_id,
+            or_(TI.state.is_(None), TI.state.not_in(_SUCCESSFUL_STATES)),
+            session=session,
         )
 
     @provide_session
diff --git a/airflow/utils/db.py b/airflow/utils/db.py
index 8e8c4b7fa9..9ef504e739 100644
--- a/airflow/utils/db.py
+++ b/airflow/utils/db.py
@@ -29,7 +29,21 @@ from dataclasses import dataclass
 from tempfile import gettempdir
 from typing import TYPE_CHECKING, Callable, Generator, Iterable
 
-from sqlalchemy import Table, and_, column, delete, exc, func, inspect, or_, 
select, table, text, tuple_
+from sqlalchemy import (
+    Table,
+    and_,
+    column,
+    delete,
+    exc,
+    func,
+    inspect,
+    literal,
+    or_,
+    select,
+    table,
+    text,
+    tuple_,
+)
 
 import airflow
 from airflow import settings
@@ -45,6 +59,7 @@ if TYPE_CHECKING:
     from alembic.runtime.environment import EnvironmentContext
     from alembic.script import ScriptDirectory
     from sqlalchemy.orm import Query, Session
+    from sqlalchemy.sql.elements import ClauseElement
     from sqlalchemy.sql.selectable import Select
 
     from airflow.models.base import Base
@@ -1875,7 +1890,7 @@ def get_sqla_model_classes():
         return Base._decl_class_registry.values()
 
 
-def get_query_count(query_stmt: Select, session: Session) -> int:
+def get_query_count(query_stmt: Select, *, session: Session) -> int:
     """Get count of query.
 
     A SELECT COUNT() FROM is issued against the subquery built from the
@@ -1887,3 +1902,14 @@ def get_query_count(query_stmt: Select, session: 
Session) -> int:
     """
     count_stmt = 
select(func.count()).select_from(query_stmt.order_by(None).subquery())
     return session.scalar(count_stmt)
+
+
+def exists_query(*where: ClauseElement, session: Session) -> bool:
+    """Check whether there is at least one row matching given clause.
+
+    This does a SELECT 1 WHERE ... LIMIT 1 and check the result.
+
+    :meta private:
+    """
+    stmt = select(literal(True)).where(*where).limit(1)
+    return session.scalar(stmt) is not None

Reply via email to