This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9855515bec Experimental: Support custom weight_rule implementation to
calculate the TI priority_weight (#38222)
9855515bec is described below
commit 9855515becfb85a06dc9a64e50b0c5b4e5a6597c
Author: Hussein Awala <[email protected]>
AuthorDate: Mon Mar 18 01:49:14 2024 +0100
Experimental: Support custom weight_rule implementation to calculate the TI
priority_weight (#38222)
---
airflow/api_connexion/schemas/common_schema.py | 13 +-
.../plugins/decreasing_priority_weight_strategy.py | 41 ++++++
airflow/executors/base_executor.py | 2 +-
airflow/executors/debug_executor.py | 2 +-
airflow/models/abstractoperator.py | 15 ++-
airflow/models/baseoperator.py | 21 +--
airflow/models/mappedoperator.py | 11 +-
airflow/models/taskinstance.py | 12 +-
airflow/plugins_manager.py | 36 +++++-
airflow/serialization/serialized_objects.py | 65 ++++++++++
airflow/task/priority_strategy.py | 143 +++++++++++++++++++++
.../priority-weight.rst | 34 +++++
tests/cli/commands/test_plugins_command.py | 1 +
tests/models/test_baseoperator.py | 6 +-
tests/models/test_dag.py | 36 ++++++
tests/plugins/priority_weight_strategy.py | 68 ++++++++++
tests/plugins/test_plugin.py | 7 +
tests/plugins/test_plugins_manager.py | 2 +-
tests/serialization/test_dag_serialization.py | 5 +-
19 files changed, 488 insertions(+), 32 deletions(-)
diff --git a/airflow/api_connexion/schemas/common_schema.py
b/airflow/api_connexion/schemas/common_schema.py
index 16740fc4c0..aaaf0812c6 100644
--- a/airflow/api_connexion/schemas/common_schema.py
+++ b/airflow/api_connexion/schemas/common_schema.py
@@ -28,7 +28,6 @@ from marshmallow_oneofschema import OneOfSchema
from airflow.models.mappedoperator import MappedOperator
from airflow.serialization.serialized_objects import SerializedBaseOperator
-from airflow.utils.weight_rule import WeightRule
class CronExpression(typing.NamedTuple):
@@ -138,9 +137,15 @@ class ColorField(fields.String):
class WeightRuleField(fields.String):
"""Schema for WeightRule."""
- def __init__(self, **metadata):
- super().__init__(**metadata)
- self.validators = [validate.OneOf(WeightRule.all_weight_rules()),
*self.validators]
+ def _serialize(self, value, attr, obj, **kwargs):
+ from airflow.serialization.serialized_objects import
encode_priority_weight_strategy
+
+ return encode_priority_weight_strategy(value)
+
+ def _deserialize(self, value, attr, data, **kwargs):
+ from airflow.serialization.serialized_objects import
decode_priority_weight_strategy
+
+ return decode_priority_weight_strategy(value)
class TimezoneField(fields.String):
diff --git
a/airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
b/airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
new file mode 100644
index 0000000000..44e6bad432
--- /dev/null
+++ b/airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.task.priority_strategy import PriorityWeightStrategy
+
+if TYPE_CHECKING:
+ from airflow.models import TaskInstance
+
+
+# [START custom_priority_weight_strategy]
+class DecreasingPriorityStrategy(PriorityWeightStrategy):
+ """A priority weight strategy that decreases the priority weight with each
attempt of the DAG task."""
+
+ def get_weight(self, ti: TaskInstance):
+ return max(3 - ti._try_number + 1, 1)
+
+
+class DecreasingPriorityWeightStrategyPlugin(AirflowPlugin):
+ name = "decreasing_priority_weight_strategy_plugin"
+ priority_weight_strategies = [DecreasingPriorityStrategy]
+
+
+# [END custom_priority_weight_strategy]
diff --git a/airflow/executors/base_executor.py
b/airflow/executors/base_executor.py
index 2c275a86c0..276c6f127c 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -185,7 +185,7 @@ class BaseExecutor(LoggingMixin):
self.queue_command(
task_instance,
command_list_to_run,
- priority=task_instance.task.priority_weight_total,
+ priority=task_instance.priority_weight,
queue=task_instance.task.queue,
)
diff --git a/airflow/executors/debug_executor.py
b/airflow/executors/debug_executor.py
index 9b376cdb01..2096ca06e0 100644
--- a/airflow/executors/debug_executor.py
+++ b/airflow/executors/debug_executor.py
@@ -109,7 +109,7 @@ class DebugExecutor(BaseExecutor):
self.queue_command(
task_instance,
[str(task_instance)], # Just for better logging, it's not used
anywhere
- priority=task_instance.task.priority_weight_total,
+ priority=task_instance.priority_weight,
queue=task_instance.task.queue,
)
# Save params for TaskInstance._run_raw_task
diff --git a/airflow/models/abstractoperator.py
b/airflow/models/abstractoperator.py
index b97f88695c..f2d179f01b 100644
--- a/airflow/models/abstractoperator.py
+++ b/airflow/models/abstractoperator.py
@@ -53,6 +53,7 @@ if TYPE_CHECKING:
from airflow.models.mappedoperator import MappedOperator
from airflow.models.operator import Operator
from airflow.models.taskinstance import TaskInstance
+ from airflow.task.priority_strategy import PriorityWeightStrategy
from airflow.utils.task_group import TaskGroup
DEFAULT_OWNER: str = conf.get_mandatory_value("operators", "default_owner")
@@ -97,7 +98,7 @@ class AbstractOperator(Templater, DAGNode):
operator_class: type[BaseOperator] | dict[str, Any]
- weight_rule: str
+ weight_rule: PriorityWeightStrategy
priority_weight: int
# Defines the operator level extra links.
@@ -397,11 +398,17 @@ class AbstractOperator(Templater, DAGNode):
- WeightRule.DOWNSTREAM - adds priority weight of all downstream tasks
- WeightRule.UPSTREAM - adds priority weight of all upstream tasks
"""
- if self.weight_rule == WeightRule.ABSOLUTE:
+ from airflow.task.priority_strategy import (
+ _AbsolutePriorityWeightStrategy,
+ _DownstreamPriorityWeightStrategy,
+ _UpstreamPriorityWeightStrategy,
+ )
+
+ if type(self.weight_rule) == _AbsolutePriorityWeightStrategy:
return self.priority_weight
- elif self.weight_rule == WeightRule.DOWNSTREAM:
+ elif type(self.weight_rule) == _DownstreamPriorityWeightStrategy:
upstream = False
- elif self.weight_rule == WeightRule.UPSTREAM:
+ elif type(self.weight_rule) == _UpstreamPriorityWeightStrategy:
upstream = True
else:
upstream = False
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index c563b0e63f..052f248950 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -80,6 +80,7 @@ from airflow.models.pool import Pool
from airflow.models.taskinstance import TaskInstance, clear_task_instances
from airflow.models.taskmixin import DependencyMixin
from airflow.serialization.enums import DagAttributeTypes
+from airflow.task.priority_strategy import PriorityWeightStrategy,
validate_and_load_priority_weight_strategy
from airflow.ti_deps.deps.not_in_retry_period_dep import NotInRetryPeriodDep
from airflow.ti_deps.deps.not_previously_skipped_dep import
NotPreviouslySkippedDep
from airflow.ti_deps.deps.prev_dagrun_dep import PrevDagrunDep
@@ -94,7 +95,6 @@ from airflow.utils.session import NEW_SESSION, provide_session
from airflow.utils.setup_teardown import SetupTeardownContext
from airflow.utils.trigger_rule import TriggerRule
from airflow.utils.types import NOTSET
-from airflow.utils.weight_rule import WeightRule
from airflow.utils.xcom import XCOM_RETURN_KEY
if TYPE_CHECKING:
@@ -244,7 +244,7 @@ def partial(
retry_delay: timedelta | float | ArgNotSet = NOTSET,
retry_exponential_backoff: bool | ArgNotSet = NOTSET,
priority_weight: int | ArgNotSet = NOTSET,
- weight_rule: str | ArgNotSet = NOTSET,
+ weight_rule: str | PriorityWeightStrategy | ArgNotSet = NOTSET,
sla: timedelta | None | ArgNotSet = NOTSET,
map_index_template: str | None | ArgNotSet = NOTSET,
max_active_tis_per_dag: int | None | ArgNotSet = NOTSET,
@@ -575,6 +575,13 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
significantly speeding up the task creation process as for very large
DAGs. Options can be set as string or using the constants defined in
the static class ``airflow.utils.WeightRule``
+ |experimental|
+ Since 2.9.0, Airflow allows to define custom priority weight strategy,
+ by creating a subclass of
+ ``airflow.task.priority_strategy.PriorityWeightStrategy`` and
registering
+ in a plugin, then providing the class path or the class instance via
+ ``weight_rule`` parameter. The custom priority weight strategy will be
+ used to calculate the effective total priority weight of the task
instance.
:param queue: which queue to target when running this job. Not
all executors implement queue management, the CeleryExecutor
does support targeting specific queues.
@@ -767,7 +774,7 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
params: collections.abc.MutableMapping | None = None,
default_args: dict | None = None,
priority_weight: int = DEFAULT_PRIORITY_WEIGHT,
- weight_rule: str = DEFAULT_WEIGHT_RULE,
+ weight_rule: str | PriorityWeightStrategy = DEFAULT_WEIGHT_RULE,
queue: str = DEFAULT_QUEUE,
pool: str | None = None,
pool_slots: int = DEFAULT_POOL_SLOTS,
@@ -918,13 +925,7 @@ class BaseOperator(AbstractOperator,
metaclass=BaseOperatorMeta):
f"received '{type(priority_weight)}'."
)
self.priority_weight = priority_weight
- if not WeightRule.is_valid(weight_rule):
- raise AirflowException(
- f"The weight_rule must be one of "
- f"{WeightRule.all_weight_rules},'{dag.dag_id if dag else
''}.{task_id}'; "
- f"received '{weight_rule}'."
- )
- self.weight_rule = weight_rule
+ self.weight_rule =
validate_and_load_priority_weight_strategy(weight_rule)
self.resources = coerce_resources(resources)
if task_concurrency and not max_active_tis_per_dag:
# TODO: Remove in Airflow 3.0
diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py
index f22ba6dbc9..c1d75fc11e 100644
--- a/airflow/models/mappedoperator.py
+++ b/airflow/models/mappedoperator.py
@@ -48,6 +48,7 @@ from airflow.models.expandinput import (
)
from airflow.models.pool import Pool
from airflow.serialization.enums import DagAttributeTypes
+from airflow.task.priority_strategy import PriorityWeightStrategy,
validate_and_load_priority_weight_strategy
from airflow.ti_deps.deps.mapped_task_expanded import MappedTaskIsExpanded
from airflow.typing_compat import Literal
from airflow.utils.context import context_update_for_unmapped
@@ -534,12 +535,14 @@ class MappedOperator(AbstractOperator):
self.partial_kwargs["priority_weight"] = value
@property
- def weight_rule(self) -> str: # type: ignore[override]
- return self.partial_kwargs.get("weight_rule", DEFAULT_WEIGHT_RULE)
+ def weight_rule(self) -> PriorityWeightStrategy: # type: ignore[override]
+ return validate_and_load_priority_weight_strategy(
+ self.partial_kwargs.get("weight_rule", DEFAULT_WEIGHT_RULE)
+ )
@weight_rule.setter
- def weight_rule(self, value: str) -> None:
- self.partial_kwargs["weight_rule"] = value
+ def weight_rule(self, value: str | PriorityWeightStrategy) -> None:
+ self.partial_kwargs["weight_rule"] =
validate_and_load_priority_weight_strategy(value)
@property
def sla(self) -> datetime.timedelta | None:
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index 433f454f1a..0de73f92c1 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -917,7 +917,11 @@ def _refresh_from_task(
task_instance.queue = task.queue
task_instance.pool = pool_override or task.pool
task_instance.pool_slots = task.pool_slots
- task_instance.priority_weight = task.priority_weight_total
+ with contextlib.suppress(Exception):
+ # This method is called from the different places, and sometimes the
TI is not fully initialized
+ task_instance.priority_weight =
task_instance.task.weight_rule.get_weight(
+ task_instance # type: ignore[arg-type]
+ )
task_instance.run_as_user = task.run_as_user
# Do not set max_tries to task.retries here because max_tries is a
cumulative
# value that needs to be stored in the db.
@@ -1421,6 +1425,10 @@ class TaskInstance(Base, LoggingMixin):
:meta private:
"""
+ priority_weight = task.weight_rule.get_weight(
+ TaskInstance(task=task, run_id=run_id, map_index=map_index)
+ )
+
return {
"dag_id": task.dag_id,
"task_id": task.task_id,
@@ -1431,7 +1439,7 @@ class TaskInstance(Base, LoggingMixin):
"queue": task.queue,
"pool": task.pool,
"pool_slots": task.pool_slots,
- "priority_weight": task.priority_weight_total,
+ "priority_weight": priority_weight,
"run_as_user": task.run_as_user,
"max_tries": task.retries,
"executor_config": task.executor_config,
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 6514409ef4..d334eb4465 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -30,6 +30,10 @@ from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterable
from airflow import settings
+from airflow.task.priority_strategy import (
+ PriorityWeightStrategy,
+ airflow_priority_weight_strategies,
+)
from airflow.utils.entry_points import entry_points_with_dist
from airflow.utils.file import find_path_from_directory
from airflow.utils.module_loading import import_string, qualname
@@ -68,6 +72,7 @@ operator_extra_links: list[Any] | None = None
registered_operator_link_classes: dict[str, type] | None = None
registered_ti_dep_classes: dict[str, type] | None = None
timetable_classes: dict[str, type[Timetable]] | None = None
+priority_weight_strategy_classes: dict[str, type[PriorityWeightStrategy]] |
None = None
"""
Mapping of class names to class of OperatorLinks registered by plugins.
@@ -89,6 +94,7 @@ PLUGINS_ATTRIBUTES_TO_DUMP = {
"ti_deps",
"timetables",
"listeners",
+ "priority_weight_strategies",
}
@@ -169,6 +175,9 @@ class AirflowPlugin:
listeners: list[ModuleType | object] = []
+ # A list of priority weight strategy classes that can be used for
calculating tasks weight priority.
+ priority_weight_strategies: list[type[PriorityWeightStrategy]] = []
+
@classmethod
def validate(cls):
"""Validate if plugin has a name."""
@@ -556,7 +565,7 @@ def get_plugin_info(attrs_to_dump: Iterable[str] | None =
None) -> list[dict[str
for attr in attrs_to_dump:
if attr in ("global_operator_extra_links",
"operator_extra_links"):
info[attr] = [f"<{qualname(d.__class__)} object>" for d in
getattr(plugin, attr)]
- elif attr in ("macros", "timetables", "hooks", "executors"):
+ elif attr in ("macros", "timetables", "hooks", "executors",
"priority_weight_strategies"):
info[attr] = [qualname(d) for d in getattr(plugin, attr)]
elif attr == "listeners":
# listeners may be modules or class instances
@@ -577,3 +586,28 @@ def get_plugin_info(attrs_to_dump: Iterable[str] | None =
None) -> list[dict[str
info[attr] = getattr(plugin, attr)
plugins_info.append(info)
return plugins_info
+
+
+def initialize_priority_weight_strategy_plugins():
+ """Collect priority weight strategy classes registered by plugins."""
+ global priority_weight_strategy_classes
+
+ if priority_weight_strategy_classes is not None:
+ return
+
+ ensure_plugins_loaded()
+
+ if plugins is None:
+ raise AirflowPluginException("Can't load plugins.")
+
+ log.debug("Initialize extra priority weight strategy plugins")
+
+ plugins_priority_weight_strategy_classes = {
+ qualname(priority_weight_strategy_class):
priority_weight_strategy_class
+ for plugin in plugins
+ for priority_weight_strategy_class in plugin.priority_weight_strategies
+ }
+ priority_weight_strategy_classes = {
+ **airflow_priority_weight_strategies,
+ **plugins_priority_weight_strategy_classes,
+ }
diff --git a/airflow/serialization/serialized_objects.py
b/airflow/serialization/serialized_objects.py
index a27772ee10..a917557a4b 100644
--- a/airflow/serialization/serialized_objects.py
+++ b/airflow/serialization/serialized_objects.py
@@ -60,6 +60,11 @@ from airflow.serialization.pydantic.job import JobPydantic
from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic
from airflow.serialization.pydantic.tasklog import LogTemplatePydantic
from airflow.settings import _ENABLE_AIP_44, DAGS_FOLDER, json
+from airflow.task.priority_strategy import (
+ PriorityWeightStrategy,
+ airflow_priority_weight_strategies,
+ airflow_priority_weight_strategies_classes,
+)
from airflow.utils.code_utils import get_python_source
from airflow.utils.docs import get_docs_url
from airflow.utils.module_loading import import_string, qualname
@@ -184,6 +189,18 @@ def _get_registered_timetable(importable_string: str) ->
type[Timetable] | None:
return None
+def _get_registered_priority_weight_strategy(importable_string: str) ->
type[PriorityWeightStrategy] | None:
+ from airflow import plugins_manager
+
+ if importable_string in airflow_priority_weight_strategies:
+ return airflow_priority_weight_strategies[importable_string]
+ plugins_manager.initialize_priority_weight_strategy_plugins()
+ if plugins_manager.priority_weight_strategy_classes:
+ return
plugins_manager.priority_weight_strategy_classes.get(importable_string)
+ else:
+ return None
+
+
class _TimetableNotRegistered(ValueError):
def __init__(self, type_string: str) -> None:
self.type_string = type_string
@@ -196,6 +213,18 @@ class _TimetableNotRegistered(ValueError):
)
+class _PriorityWeightStrategyNotRegistered(AirflowException):
+ def __init__(self, type_string: str) -> None:
+ self.type_string = type_string
+
+ def __str__(self) -> str:
+ return (
+ f"Priority weight strategy class {self.type_string!r} is not
registered or "
+ "you have a top level database access that disrupted the session. "
+ "Please check the airflow best practices documentation."
+ )
+
+
def encode_timetable(var: Timetable) -> dict[str, Any]:
"""
Encode a timetable instance.
@@ -228,6 +257,36 @@ def decode_timetable(var: dict[str, Any]) -> Timetable:
return timetable_class.deserialize(var[Encoding.VAR])
+def encode_priority_weight_strategy(var: PriorityWeightStrategy) -> str:
+ """
+ Encode a priority weight strategy instance.
+
+ In this version, we only store the importable string, so the class should
not wait
+ for any parameters to be passed to it. If you need to store the
parameters, you
+ should store them in the class itself.
+ """
+ priority_weight_strategy_class = type(var)
+ if priority_weight_strategy_class in
airflow_priority_weight_strategies_classes:
+ return
airflow_priority_weight_strategies_classes[priority_weight_strategy_class]
+ importable_string = qualname(priority_weight_strategy_class)
+ if _get_registered_priority_weight_strategy(importable_string) is None:
+ raise _PriorityWeightStrategyNotRegistered(importable_string)
+ return importable_string
+
+
+def decode_priority_weight_strategy(var: str) -> PriorityWeightStrategy:
+ """
+ Decode a previously serialized priority weight strategy.
+
+ In this version, we only store the importable string, so we just need to
get the class
+ from the dictionary of registered classes and instantiate it with no
parameters.
+ """
+ priority_weight_strategy_class =
_get_registered_priority_weight_strategy(var)
+ if priority_weight_strategy_class is None:
+ raise _PriorityWeightStrategyNotRegistered(var)
+ return priority_weight_strategy_class()
+
+
class _XComRef(NamedTuple):
"""
Store info needed to create XComArg.
@@ -422,6 +481,8 @@ class BaseSerialization:
serialized_object[key] = cls.serialize(value)
elif key == "timetable" and value is not None:
serialized_object[key] = encode_timetable(value)
+ elif key == "weight_rule" and value is not None:
+ serialized_object[key] = encode_priority_weight_strategy(value)
elif key == "dataset_triggers":
serialized_object[key] = cls.serialize(value)
else:
@@ -1064,6 +1125,8 @@ class SerializedBaseOperator(BaseOperator,
BaseSerialization):
v = cls.deserialize(v)
elif k == "on_failure_fail_dagrun":
k = "_on_failure_fail_dagrun"
+ elif k == "weight_rule":
+ v = decode_priority_weight_strategy(v)
# else use v as it is
setattr(op, k, v)
@@ -1411,6 +1474,8 @@ class SerializedDAG(DAG, BaseSerialization):
pass
elif k == "timetable":
v = decode_timetable(v)
+ elif k == "weight_rule":
+ v = decode_priority_weight_strategy(v)
elif k in cls._decorated_fields:
v = cls.deserialize(v)
elif k == "params":
diff --git a/airflow/task/priority_strategy.py
b/airflow/task/priority_strategy.py
new file mode 100644
index 0000000000..556539e220
--- /dev/null
+++ b/airflow/task/priority_strategy.py
@@ -0,0 +1,143 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Priority weight strategies for task scheduling."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from airflow.exceptions import AirflowException
+
+if TYPE_CHECKING:
+ from airflow.models.taskinstance import TaskInstance
+
+
+class PriorityWeightStrategy(ABC):
+ """Priority weight strategy interface.
+
+ This feature is experimental and subject to change at any time.
+
+ Currently, we don't serialize the priority weight strategy parameters.
This means that
+ the priority weight strategy must be stateless, but you can add class
attributes, and
+ create multiple subclasses with different attributes values if you need to
create
+ different versions of the same strategy.
+ """
+
+ @abstractmethod
+ def get_weight(self, ti: TaskInstance):
+ """Get the priority weight of a task."""
+ ...
+
+ @classmethod
+ def deserialize(cls, data: dict[str, Any]) -> PriorityWeightStrategy:
+ """Deserialize a priority weight strategy from data.
+
+ This is called when a serialized DAG is deserialized. ``data`` will be
whatever
+ was returned by ``serialize`` during DAG serialization. The default
+ implementation constructs the priority weight strategy without any
arguments.
+ """
+ return cls(**data) # type: ignore[call-arg]
+
+ def serialize(self) -> dict[str, Any]:
+ """Serialize the priority weight strategy for JSON encoding.
+
+ This is called during DAG serialization to store priority weight
strategy information
+ in the database. This should return a JSON-serializable dict that will
be fed into
+ ``deserialize`` when the DAG is deserialized. The default
implementation returns
+ an empty dict.
+ """
+ return {}
+
+ def __eq__(self, other: object) -> bool:
+ """Equality comparison."""
+ if not isinstance(other, type(self)):
+ return False
+ return self.serialize() == other.serialize()
+
+
+class _AbsolutePriorityWeightStrategy(PriorityWeightStrategy):
+ """Priority weight strategy that uses the task's priority weight
directly."""
+
+ def get_weight(self, ti: TaskInstance):
+ return ti.task.priority_weight
+
+
+class _DownstreamPriorityWeightStrategy(PriorityWeightStrategy):
+ """Priority weight strategy that uses the sum of the priority weights of
all downstream tasks."""
+
+ def get_weight(self, ti: TaskInstance) -> int:
+ dag = ti.task.get_dag()
+ if dag is None:
+ return ti.task.priority_weight
+ return ti.task.priority_weight + sum(
+ dag.task_dict[task_id].priority_weight
+ for task_id in ti.task.get_flat_relative_ids(upstream=False)
+ )
+
+
+class _UpstreamPriorityWeightStrategy(PriorityWeightStrategy):
+ """Priority weight strategy that uses the sum of the priority weights of
all upstream tasks."""
+
+ def get_weight(self, ti: TaskInstance):
+ dag = ti.task.get_dag()
+ if dag is None:
+ return ti.task.priority_weight
+ return ti.task.priority_weight + sum(
+ dag.task_dict[task_id].priority_weight for task_id in
ti.task.get_flat_relative_ids(upstream=True)
+ )
+
+
+airflow_priority_weight_strategies: dict[str, type[PriorityWeightStrategy]] = {
+ "absolute": _AbsolutePriorityWeightStrategy,
+ "downstream": _DownstreamPriorityWeightStrategy,
+ "upstream": _UpstreamPriorityWeightStrategy,
+}
+
+
+airflow_priority_weight_strategies_classes = {
+ cls: name for name, cls in airflow_priority_weight_strategies.items()
+}
+
+
+def validate_and_load_priority_weight_strategy(
+ priority_weight_strategy: str | PriorityWeightStrategy | None,
+) -> PriorityWeightStrategy:
+ """Validate and load a priority weight strategy.
+
+ Returns the priority weight strategy if it is valid, otherwise raises an
exception.
+
+ :param priority_weight_strategy: The priority weight strategy to validate
and load.
+
+ :meta private:
+ """
+ from airflow.serialization.serialized_objects import
_get_registered_priority_weight_strategy
+ from airflow.utils.module_loading import qualname
+
+ if priority_weight_strategy is None:
+ return _AbsolutePriorityWeightStrategy()
+
+ if isinstance(priority_weight_strategy, str):
+ if priority_weight_strategy in airflow_priority_weight_strategies:
+ return
airflow_priority_weight_strategies[priority_weight_strategy]()
+ priority_weight_strategy_class = priority_weight_strategy
+ else:
+ priority_weight_strategy_class = qualname(priority_weight_strategy)
+ loaded_priority_weight_strategy =
_get_registered_priority_weight_strategy(priority_weight_strategy_class)
+ if loaded_priority_weight_strategy is None:
+ raise AirflowException(f"Unknown priority strategy
{priority_weight_strategy_class}")
+ return loaded_priority_weight_strategy()
diff --git
a/docs/apache-airflow/administration-and-deployment/priority-weight.rst
b/docs/apache-airflow/administration-and-deployment/priority-weight.rst
index 3807b3ee5d..dd61d25fcd 100644
--- a/docs/apache-airflow/administration-and-deployment/priority-weight.rst
+++ b/docs/apache-airflow/administration-and-deployment/priority-weight.rst
@@ -62,3 +62,37 @@ Below are the weighting methods. By default, Airflow's
weighting method is ``dow
The ``priority_weight`` parameter can be used in conjunction with
:ref:`concepts:pool`.
+
+Custom Weight Rule
+------------------
+
+.. versionadded:: 2.9.0
+
+You can implement your own custom weighting method by extending the
``PriorityWeightStrategy`` class and
+registering it in a plugin.
+
+.. exampleinclude::
/../../airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
+ :language: python
+ :dedent: 0
+ :start-after: [START custom_priority_weight_strategy]
+ :end-before: [END custom_priority_weight_strategy]
+
+
+Then to use it, you can create an instance of the custom class and provide it
in the ``weight_rule`` parameter
+of the task or provide the path of the custom class:
+
+.. code-block:: python
+
+ from custom_weight_rule_module import CustomPriorityWeightStrategy
+
+ # provide the class instance
+ task1 = BashOperator(task_id="task", bash_command="echo 1",
weight_rule=CustomPriorityWeightStrategy())
+
+ # or provide the path of the class
+ task1 = BashOperator(
+ task_id="task",
+ bash_command="echo 1",
+ weight_rule="custom_weight_rule_module.CustomPriorityWeightStrategy",
+ )
+
+|experimental|
diff --git a/tests/cli/commands/test_plugins_command.py
b/tests/cli/commands/test_plugins_command.py
index cbf6afeab9..71d5ce5f72 100644
--- a/tests/cli/commands/test_plugins_command.py
+++ b/tests/cli/commands/test_plugins_command.py
@@ -101,6 +101,7 @@ class TestPluginsCommand:
},
],
"ti_deps": ["<TIDep(CustomTestTriggerRule)>"],
+ "priority_weight_strategies":
["tests.plugins.test_plugin.CustomPriorityWeightStrategy"],
}
]
get_listener_manager().clear()
diff --git a/tests/models/test_baseoperator.py
b/tests/models/test_baseoperator.py
index d58a52224b..959bea7209 100644
--- a/tests/models/test_baseoperator.py
+++ b/tests/models/test_baseoperator.py
@@ -41,12 +41,12 @@ from airflow.models.baseoperator import (
from airflow.models.dag import DAG
from airflow.models.dagrun import DagRun
from airflow.models.taskinstance import TaskInstance
+from airflow.task.priority_strategy import _DownstreamPriorityWeightStrategy,
_UpstreamPriorityWeightStrategy
from airflow.utils.edgemodifier import Label
from airflow.utils.task_group import TaskGroup
from airflow.utils.template import literal
from airflow.utils.trigger_rule import TriggerRule
from airflow.utils.types import DagRunType
-from airflow.utils.weight_rule import WeightRule
from tests.models import DEFAULT_DATE
from tests.test_utils.config import conf_vars
from tests.test_utils.mock_operators import DeprecatedOperator, MockOperator
@@ -775,11 +775,11 @@ class TestBaseOperator:
def test_weight_rule_default(self):
op = BaseOperator(task_id="test_task")
- assert WeightRule.DOWNSTREAM == op.weight_rule
+ assert _DownstreamPriorityWeightStrategy() == op.weight_rule
def test_weight_rule_override(self):
op = BaseOperator(task_id="test_task", weight_rule="upstream")
- assert WeightRule.UPSTREAM == op.weight_rule
+ assert _UpstreamPriorityWeightStrategy() == op.weight_rule
# ensure the default logging config is used for this test, no matter what
ran before
@pytest.mark.usefixtures("reset_logging_config")
diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py
index 4dd29ecbe8..2836fb827f 100644
--- a/tests/models/test_dag.py
+++ b/tests/models/test_dag.py
@@ -90,10 +90,17 @@ from airflow.utils.trigger_rule import TriggerRule
from airflow.utils.types import DagRunType
from airflow.utils.weight_rule import WeightRule
from tests.models import DEFAULT_DATE
+from tests.plugins.priority_weight_strategy import (
+ FactorPriorityWeightStrategy,
+ NotRegisteredPriorityWeightStrategy,
+ StaticTestPriorityWeightStrategy,
+ TestPriorityWeightStrategyPlugin,
+)
from tests.test_utils.asserts import assert_queries_count
from tests.test_utils.config import conf_vars
from tests.test_utils.db import clear_db_dags, clear_db_datasets,
clear_db_runs, clear_db_serialized_dags
from tests.test_utils.mapping import expand_mapped_task
+from tests.test_utils.mock_plugins import mock_plugin_manager
from tests.test_utils.timetables import cron_timetable, delta_timetable
pytestmark = pytest.mark.db_test
@@ -433,6 +440,35 @@ class TestDag:
with pytest.raises(AirflowException):
EmptyOperator(task_id="should_fail", weight_rule="no rule")
+ @pytest.mark.parametrize(
+ "cls, expected",
+ [
+ (StaticTestPriorityWeightStrategy, 99),
+ (FactorPriorityWeightStrategy, 3),
+ ],
+ )
+ def test_dag_task_custom_weight_strategy(self, cls, expected):
+ with mock_plugin_manager(plugins=[TestPriorityWeightStrategyPlugin]),
DAG(
+ "dag", start_date=DEFAULT_DATE, default_args={"owner": "owner1"}
+ ) as dag:
+ task = EmptyOperator(
+ task_id="empty_task",
+ weight_rule=cls(),
+ )
+ dr = dag.create_dagrun(state=None, run_id="test",
execution_date=DEFAULT_DATE)
+ ti = dr.get_task_instance(task.task_id)
+ assert ti.priority_weight == expected
+
+ def test_dag_task_not_registered_weight_strategy(self):
+ with mock_plugin_manager(plugins=[TestPriorityWeightStrategyPlugin]),
DAG(
+ "dag", start_date=DEFAULT_DATE, default_args={"owner": "owner1"}
+ ):
+ with pytest.raises(AirflowException, match="Unknown priority
strategy"):
+ EmptyOperator(
+ task_id="empty_task",
+ weight_rule=NotRegisteredPriorityWeightStrategy(),
+ )
+
def test_get_num_task_instances(self):
test_dag_id = "test_get_num_task_instances_dag"
test_task_id = "task_1"
diff --git a/tests/plugins/priority_weight_strategy.py
b/tests/plugins/priority_weight_strategy.py
new file mode 100644
index 0000000000..c56ae7364a
--- /dev/null
+++ b/tests/plugins/priority_weight_strategy.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.task.priority_strategy import PriorityWeightStrategy
+
+if TYPE_CHECKING:
+ from airflow.models import TaskInstance
+
+
+class StaticTestPriorityWeightStrategy(PriorityWeightStrategy):
+ def get_weight(self, ti: TaskInstance):
+ return 99
+
+
+class FactorPriorityWeightStrategy(PriorityWeightStrategy):
+ factor: int = 3
+
+ def serialize(self) -> dict[str, Any]:
+ return {"factor": self.factor}
+
+ def get_weight(self, ti: TaskInstance):
+ return max(ti.map_index, 1) * self.factor
+
+
+class DecreasingPriorityStrategy(PriorityWeightStrategy):
+ """A priority weight strategy that decreases the priority weight with each
attempt."""
+
+ def get_weight(self, ti: TaskInstance):
+ return max(3 - ti._try_number + 1, 1)
+
+
+class TestPriorityWeightStrategyPlugin(AirflowPlugin):
+ # Without this import, the qualname method will not use the correct
classes names
+ from tests.plugins.priority_weight_strategy import (
+ DecreasingPriorityStrategy,
+ FactorPriorityWeightStrategy,
+ StaticTestPriorityWeightStrategy,
+ )
+
+ name = "priority_weight_strategy_plugin"
+ priority_weight_strategies = [
+ StaticTestPriorityWeightStrategy,
+ FactorPriorityWeightStrategy,
+ DecreasingPriorityStrategy,
+ ]
+
+
+class NotRegisteredPriorityWeightStrategy(PriorityWeightStrategy):
+ def get_weight(self, ti: TaskInstance):
+ return 99
diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py
index e207fd12da..717dab1a60 100644
--- a/tests/plugins/test_plugin.py
+++ b/tests/plugins/test_plugin.py
@@ -29,6 +29,7 @@ from airflow.models.baseoperator import BaseOperator
# This is the class you derive to create a plugin
from airflow.plugins_manager import AirflowPlugin
from airflow.sensors.base import BaseSensorOperator
+from airflow.task.priority_strategy import PriorityWeightStrategy
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
from airflow.timetables.interval import CronDataIntervalTimetable
from tests.listeners import empty_listener
@@ -113,6 +114,11 @@ class CustomTestTriggerRule(BaseTIDep):
pass
+class CustomPriorityWeightStrategy(PriorityWeightStrategy):
+ def get_weight(self, ti):
+ return 1
+
+
# Defining the plugin class
class AirflowTestPlugin(AirflowPlugin):
name = "test_plugin"
@@ -132,6 +138,7 @@ class AirflowTestPlugin(AirflowPlugin):
timetables = [CustomCronDataIntervalTimetable]
listeners = [empty_listener, ClassBasedListener()]
ti_deps = [CustomTestTriggerRule()]
+ priority_weight_strategies = [CustomPriorityWeightStrategy]
class MockPluginA(AirflowPlugin):
diff --git a/tests/plugins/test_plugins_manager.py
b/tests/plugins/test_plugins_manager.py
index 50501e4cbe..35058a9afd 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -209,7 +209,7 @@ class TestPluginsManager:
with mock.patch("airflow.plugins_manager.plugins", []):
plugins_manager.load_plugins_from_plugin_directory()
- assert 6 == len(plugins_manager.plugins)
+ assert 7 == len(plugins_manager.plugins)
for plugin in plugins_manager.plugins:
if "AirflowTestOnLoadPlugin" in str(plugin):
assert "postload" == plugin.name
diff --git a/tests/serialization/test_dag_serialization.py
b/tests/serialization/test_dag_serialization.py
index e99f4641d5..a9fe733688 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -69,6 +69,7 @@ from airflow.serialization.serialized_objects import (
SerializedBaseOperator,
SerializedDAG,
)
+from airflow.task.priority_strategy import _DownstreamPriorityWeightStrategy
from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
from airflow.timetables.simple import NullTimetable, OnceTimetable
from airflow.utils import timezone
@@ -190,6 +191,7 @@ serialized_simple_dag_ground_truth = {
},
"doc_md": "### Task Tutorial Documentation",
"_log_config_logger_name": "airflow.task.operators",
+ "weight_rule": "downstream",
},
{
"task_id": "custom_task",
@@ -213,6 +215,7 @@ serialized_simple_dag_ground_truth = {
"is_teardown": False,
"on_failure_fail_dagrun": False,
"_log_config_logger_name": "airflow.task.operators",
+ "weight_rule": "downstream",
},
],
"schedule_interval": {"__type": "timedelta", "__var": 86400.0},
@@ -1289,7 +1292,7 @@ class TestStringifiedDAGs:
"trigger_rule": "all_success",
"wait_for_downstream": False,
"wait_for_past_depends_before_skipping": False,
- "weight_rule": "downstream",
+ "weight_rule": _DownstreamPriorityWeightStrategy(),
"multiple_outputs": False,
}, """
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!