This is an automated email from the ASF dual-hosted git repository.

husseinawala pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 9855515bec Experimental: Support custom weight_rule implementation to 
calculate the TI priority_weight (#38222)
9855515bec is described below

commit 9855515becfb85a06dc9a64e50b0c5b4e5a6597c
Author: Hussein Awala <[email protected]>
AuthorDate: Mon Mar 18 01:49:14 2024 +0100

    Experimental: Support custom weight_rule implementation to calculate the TI 
priority_weight (#38222)
---
 airflow/api_connexion/schemas/common_schema.py     |  13 +-
 .../plugins/decreasing_priority_weight_strategy.py |  41 ++++++
 airflow/executors/base_executor.py                 |   2 +-
 airflow/executors/debug_executor.py                |   2 +-
 airflow/models/abstractoperator.py                 |  15 ++-
 airflow/models/baseoperator.py                     |  21 +--
 airflow/models/mappedoperator.py                   |  11 +-
 airflow/models/taskinstance.py                     |  12 +-
 airflow/plugins_manager.py                         |  36 +++++-
 airflow/serialization/serialized_objects.py        |  65 ++++++++++
 airflow/task/priority_strategy.py                  | 143 +++++++++++++++++++++
 .../priority-weight.rst                            |  34 +++++
 tests/cli/commands/test_plugins_command.py         |   1 +
 tests/models/test_baseoperator.py                  |   6 +-
 tests/models/test_dag.py                           |  36 ++++++
 tests/plugins/priority_weight_strategy.py          |  68 ++++++++++
 tests/plugins/test_plugin.py                       |   7 +
 tests/plugins/test_plugins_manager.py              |   2 +-
 tests/serialization/test_dag_serialization.py      |   5 +-
 19 files changed, 488 insertions(+), 32 deletions(-)

diff --git a/airflow/api_connexion/schemas/common_schema.py 
b/airflow/api_connexion/schemas/common_schema.py
index 16740fc4c0..aaaf0812c6 100644
--- a/airflow/api_connexion/schemas/common_schema.py
+++ b/airflow/api_connexion/schemas/common_schema.py
@@ -28,7 +28,6 @@ from marshmallow_oneofschema import OneOfSchema
 
 from airflow.models.mappedoperator import MappedOperator
 from airflow.serialization.serialized_objects import SerializedBaseOperator
-from airflow.utils.weight_rule import WeightRule
 
 
 class CronExpression(typing.NamedTuple):
@@ -138,9 +137,15 @@ class ColorField(fields.String):
 class WeightRuleField(fields.String):
     """Schema for WeightRule."""
 
-    def __init__(self, **metadata):
-        super().__init__(**metadata)
-        self.validators = [validate.OneOf(WeightRule.all_weight_rules()), 
*self.validators]
+    def _serialize(self, value, attr, obj, **kwargs):
+        from airflow.serialization.serialized_objects import 
encode_priority_weight_strategy
+
+        return encode_priority_weight_strategy(value)
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        from airflow.serialization.serialized_objects import 
decode_priority_weight_strategy
+
+        return decode_priority_weight_strategy(value)
 
 
 class TimezoneField(fields.String):
diff --git 
a/airflow/example_dags/plugins/decreasing_priority_weight_strategy.py 
b/airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
new file mode 100644
index 0000000000..44e6bad432
--- /dev/null
+++ b/airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.task.priority_strategy import PriorityWeightStrategy
+
+if TYPE_CHECKING:
+    from airflow.models import TaskInstance
+
+
+# [START custom_priority_weight_strategy]
+class DecreasingPriorityStrategy(PriorityWeightStrategy):
+    """A priority weight strategy that decreases the priority weight with each 
attempt of the DAG task."""
+
+    def get_weight(self, ti: TaskInstance):
+        return max(3 - ti._try_number + 1, 1)
+
+
+class DecreasingPriorityWeightStrategyPlugin(AirflowPlugin):
+    name = "decreasing_priority_weight_strategy_plugin"
+    priority_weight_strategies = [DecreasingPriorityStrategy]
+
+
+# [END custom_priority_weight_strategy]
diff --git a/airflow/executors/base_executor.py 
b/airflow/executors/base_executor.py
index 2c275a86c0..276c6f127c 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -185,7 +185,7 @@ class BaseExecutor(LoggingMixin):
         self.queue_command(
             task_instance,
             command_list_to_run,
-            priority=task_instance.task.priority_weight_total,
+            priority=task_instance.priority_weight,
             queue=task_instance.task.queue,
         )
 
diff --git a/airflow/executors/debug_executor.py 
b/airflow/executors/debug_executor.py
index 9b376cdb01..2096ca06e0 100644
--- a/airflow/executors/debug_executor.py
+++ b/airflow/executors/debug_executor.py
@@ -109,7 +109,7 @@ class DebugExecutor(BaseExecutor):
         self.queue_command(
             task_instance,
             [str(task_instance)],  # Just for better logging, it's not used 
anywhere
-            priority=task_instance.task.priority_weight_total,
+            priority=task_instance.priority_weight,
             queue=task_instance.task.queue,
         )
         # Save params for TaskInstance._run_raw_task
diff --git a/airflow/models/abstractoperator.py 
b/airflow/models/abstractoperator.py
index b97f88695c..f2d179f01b 100644
--- a/airflow/models/abstractoperator.py
+++ b/airflow/models/abstractoperator.py
@@ -53,6 +53,7 @@ if TYPE_CHECKING:
     from airflow.models.mappedoperator import MappedOperator
     from airflow.models.operator import Operator
     from airflow.models.taskinstance import TaskInstance
+    from airflow.task.priority_strategy import PriorityWeightStrategy
     from airflow.utils.task_group import TaskGroup
 
 DEFAULT_OWNER: str = conf.get_mandatory_value("operators", "default_owner")
@@ -97,7 +98,7 @@ class AbstractOperator(Templater, DAGNode):
 
     operator_class: type[BaseOperator] | dict[str, Any]
 
-    weight_rule: str
+    weight_rule: PriorityWeightStrategy
     priority_weight: int
 
     # Defines the operator level extra links.
@@ -397,11 +398,17 @@ class AbstractOperator(Templater, DAGNode):
         - WeightRule.DOWNSTREAM - adds priority weight of all downstream tasks
         - WeightRule.UPSTREAM - adds priority weight of all upstream tasks
         """
-        if self.weight_rule == WeightRule.ABSOLUTE:
+        from airflow.task.priority_strategy import (
+            _AbsolutePriorityWeightStrategy,
+            _DownstreamPriorityWeightStrategy,
+            _UpstreamPriorityWeightStrategy,
+        )
+
+        if type(self.weight_rule) == _AbsolutePriorityWeightStrategy:
             return self.priority_weight
-        elif self.weight_rule == WeightRule.DOWNSTREAM:
+        elif type(self.weight_rule) == _DownstreamPriorityWeightStrategy:
             upstream = False
-        elif self.weight_rule == WeightRule.UPSTREAM:
+        elif type(self.weight_rule) == _UpstreamPriorityWeightStrategy:
             upstream = True
         else:
             upstream = False
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
index c563b0e63f..052f248950 100644
--- a/airflow/models/baseoperator.py
+++ b/airflow/models/baseoperator.py
@@ -80,6 +80,7 @@ from airflow.models.pool import Pool
 from airflow.models.taskinstance import TaskInstance, clear_task_instances
 from airflow.models.taskmixin import DependencyMixin
 from airflow.serialization.enums import DagAttributeTypes
+from airflow.task.priority_strategy import PriorityWeightStrategy, 
validate_and_load_priority_weight_strategy
 from airflow.ti_deps.deps.not_in_retry_period_dep import NotInRetryPeriodDep
 from airflow.ti_deps.deps.not_previously_skipped_dep import 
NotPreviouslySkippedDep
 from airflow.ti_deps.deps.prev_dagrun_dep import PrevDagrunDep
@@ -94,7 +95,6 @@ from airflow.utils.session import NEW_SESSION, provide_session
 from airflow.utils.setup_teardown import SetupTeardownContext
 from airflow.utils.trigger_rule import TriggerRule
 from airflow.utils.types import NOTSET
-from airflow.utils.weight_rule import WeightRule
 from airflow.utils.xcom import XCOM_RETURN_KEY
 
 if TYPE_CHECKING:
@@ -244,7 +244,7 @@ def partial(
     retry_delay: timedelta | float | ArgNotSet = NOTSET,
     retry_exponential_backoff: bool | ArgNotSet = NOTSET,
     priority_weight: int | ArgNotSet = NOTSET,
-    weight_rule: str | ArgNotSet = NOTSET,
+    weight_rule: str | PriorityWeightStrategy | ArgNotSet = NOTSET,
     sla: timedelta | None | ArgNotSet = NOTSET,
     map_index_template: str | None | ArgNotSet = NOTSET,
     max_active_tis_per_dag: int | None | ArgNotSet = NOTSET,
@@ -575,6 +575,13 @@ class BaseOperator(AbstractOperator, 
metaclass=BaseOperatorMeta):
         significantly speeding up the task creation process as for very large
         DAGs. Options can be set as string or using the constants defined in
         the static class ``airflow.utils.WeightRule``
+        |experimental|
+        Since 2.9.0, Airflow allows to define custom priority weight strategy,
+        by creating a subclass of
+        ``airflow.task.priority_strategy.PriorityWeightStrategy`` and 
registering
+        in a plugin, then providing the class path or the class instance via
+        ``weight_rule`` parameter. The custom priority weight strategy will be
+        used to calculate the effective total priority weight of the task 
instance.
     :param queue: which queue to target when running this job. Not
         all executors implement queue management, the CeleryExecutor
         does support targeting specific queues.
@@ -767,7 +774,7 @@ class BaseOperator(AbstractOperator, 
metaclass=BaseOperatorMeta):
         params: collections.abc.MutableMapping | None = None,
         default_args: dict | None = None,
         priority_weight: int = DEFAULT_PRIORITY_WEIGHT,
-        weight_rule: str = DEFAULT_WEIGHT_RULE,
+        weight_rule: str | PriorityWeightStrategy = DEFAULT_WEIGHT_RULE,
         queue: str = DEFAULT_QUEUE,
         pool: str | None = None,
         pool_slots: int = DEFAULT_POOL_SLOTS,
@@ -918,13 +925,7 @@ class BaseOperator(AbstractOperator, 
metaclass=BaseOperatorMeta):
                 f"received '{type(priority_weight)}'."
             )
         self.priority_weight = priority_weight
-        if not WeightRule.is_valid(weight_rule):
-            raise AirflowException(
-                f"The weight_rule must be one of "
-                f"{WeightRule.all_weight_rules},'{dag.dag_id if dag else 
''}.{task_id}'; "
-                f"received '{weight_rule}'."
-            )
-        self.weight_rule = weight_rule
+        self.weight_rule = 
validate_and_load_priority_weight_strategy(weight_rule)
         self.resources = coerce_resources(resources)
         if task_concurrency and not max_active_tis_per_dag:
             # TODO: Remove in Airflow 3.0
diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py
index f22ba6dbc9..c1d75fc11e 100644
--- a/airflow/models/mappedoperator.py
+++ b/airflow/models/mappedoperator.py
@@ -48,6 +48,7 @@ from airflow.models.expandinput import (
 )
 from airflow.models.pool import Pool
 from airflow.serialization.enums import DagAttributeTypes
+from airflow.task.priority_strategy import PriorityWeightStrategy, 
validate_and_load_priority_weight_strategy
 from airflow.ti_deps.deps.mapped_task_expanded import MappedTaskIsExpanded
 from airflow.typing_compat import Literal
 from airflow.utils.context import context_update_for_unmapped
@@ -534,12 +535,14 @@ class MappedOperator(AbstractOperator):
         self.partial_kwargs["priority_weight"] = value
 
     @property
-    def weight_rule(self) -> str:  # type: ignore[override]
-        return self.partial_kwargs.get("weight_rule", DEFAULT_WEIGHT_RULE)
+    def weight_rule(self) -> PriorityWeightStrategy:  # type: ignore[override]
+        return validate_and_load_priority_weight_strategy(
+            self.partial_kwargs.get("weight_rule", DEFAULT_WEIGHT_RULE)
+        )
 
     @weight_rule.setter
-    def weight_rule(self, value: str) -> None:
-        self.partial_kwargs["weight_rule"] = value
+    def weight_rule(self, value: str | PriorityWeightStrategy) -> None:
+        self.partial_kwargs["weight_rule"] = 
validate_and_load_priority_weight_strategy(value)
 
     @property
     def sla(self) -> datetime.timedelta | None:
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
index 433f454f1a..0de73f92c1 100644
--- a/airflow/models/taskinstance.py
+++ b/airflow/models/taskinstance.py
@@ -917,7 +917,11 @@ def _refresh_from_task(
     task_instance.queue = task.queue
     task_instance.pool = pool_override or task.pool
     task_instance.pool_slots = task.pool_slots
-    task_instance.priority_weight = task.priority_weight_total
+    with contextlib.suppress(Exception):
+        # This method is called from the different places, and sometimes the 
TI is not fully initialized
+        task_instance.priority_weight = 
task_instance.task.weight_rule.get_weight(
+            task_instance  # type: ignore[arg-type]
+        )
     task_instance.run_as_user = task.run_as_user
     # Do not set max_tries to task.retries here because max_tries is a 
cumulative
     # value that needs to be stored in the db.
@@ -1421,6 +1425,10 @@ class TaskInstance(Base, LoggingMixin):
 
         :meta private:
         """
+        priority_weight = task.weight_rule.get_weight(
+            TaskInstance(task=task, run_id=run_id, map_index=map_index)
+        )
+
         return {
             "dag_id": task.dag_id,
             "task_id": task.task_id,
@@ -1431,7 +1439,7 @@ class TaskInstance(Base, LoggingMixin):
             "queue": task.queue,
             "pool": task.pool,
             "pool_slots": task.pool_slots,
-            "priority_weight": task.priority_weight_total,
+            "priority_weight": priority_weight,
             "run_as_user": task.run_as_user,
             "max_tries": task.retries,
             "executor_config": task.executor_config,
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 6514409ef4..d334eb4465 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -30,6 +30,10 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any, Iterable
 
 from airflow import settings
+from airflow.task.priority_strategy import (
+    PriorityWeightStrategy,
+    airflow_priority_weight_strategies,
+)
 from airflow.utils.entry_points import entry_points_with_dist
 from airflow.utils.file import find_path_from_directory
 from airflow.utils.module_loading import import_string, qualname
@@ -68,6 +72,7 @@ operator_extra_links: list[Any] | None = None
 registered_operator_link_classes: dict[str, type] | None = None
 registered_ti_dep_classes: dict[str, type] | None = None
 timetable_classes: dict[str, type[Timetable]] | None = None
+priority_weight_strategy_classes: dict[str, type[PriorityWeightStrategy]] | 
None = None
 """
 Mapping of class names to class of OperatorLinks registered by plugins.
 
@@ -89,6 +94,7 @@ PLUGINS_ATTRIBUTES_TO_DUMP = {
     "ti_deps",
     "timetables",
     "listeners",
+    "priority_weight_strategies",
 }
 
 
@@ -169,6 +175,9 @@ class AirflowPlugin:
 
     listeners: list[ModuleType | object] = []
 
+    # A list of priority weight strategy classes that can be used for 
calculating tasks weight priority.
+    priority_weight_strategies: list[type[PriorityWeightStrategy]] = []
+
     @classmethod
     def validate(cls):
         """Validate if plugin has a name."""
@@ -556,7 +565,7 @@ def get_plugin_info(attrs_to_dump: Iterable[str] | None = 
None) -> list[dict[str
             for attr in attrs_to_dump:
                 if attr in ("global_operator_extra_links", 
"operator_extra_links"):
                     info[attr] = [f"<{qualname(d.__class__)} object>" for d in 
getattr(plugin, attr)]
-                elif attr in ("macros", "timetables", "hooks", "executors"):
+                elif attr in ("macros", "timetables", "hooks", "executors", 
"priority_weight_strategies"):
                     info[attr] = [qualname(d) for d in getattr(plugin, attr)]
                 elif attr == "listeners":
                     # listeners may be modules or class instances
@@ -577,3 +586,28 @@ def get_plugin_info(attrs_to_dump: Iterable[str] | None = 
None) -> list[dict[str
                     info[attr] = getattr(plugin, attr)
             plugins_info.append(info)
     return plugins_info
+
+
+def initialize_priority_weight_strategy_plugins():
+    """Collect priority weight strategy classes registered by plugins."""
+    global priority_weight_strategy_classes
+
+    if priority_weight_strategy_classes is not None:
+        return
+
+    ensure_plugins_loaded()
+
+    if plugins is None:
+        raise AirflowPluginException("Can't load plugins.")
+
+    log.debug("Initialize extra priority weight strategy plugins")
+
+    plugins_priority_weight_strategy_classes = {
+        qualname(priority_weight_strategy_class): 
priority_weight_strategy_class
+        for plugin in plugins
+        for priority_weight_strategy_class in plugin.priority_weight_strategies
+    }
+    priority_weight_strategy_classes = {
+        **airflow_priority_weight_strategies,
+        **plugins_priority_weight_strategy_classes,
+    }
diff --git a/airflow/serialization/serialized_objects.py 
b/airflow/serialization/serialized_objects.py
index a27772ee10..a917557a4b 100644
--- a/airflow/serialization/serialized_objects.py
+++ b/airflow/serialization/serialized_objects.py
@@ -60,6 +60,11 @@ from airflow.serialization.pydantic.job import JobPydantic
 from airflow.serialization.pydantic.taskinstance import TaskInstancePydantic
 from airflow.serialization.pydantic.tasklog import LogTemplatePydantic
 from airflow.settings import _ENABLE_AIP_44, DAGS_FOLDER, json
+from airflow.task.priority_strategy import (
+    PriorityWeightStrategy,
+    airflow_priority_weight_strategies,
+    airflow_priority_weight_strategies_classes,
+)
 from airflow.utils.code_utils import get_python_source
 from airflow.utils.docs import get_docs_url
 from airflow.utils.module_loading import import_string, qualname
@@ -184,6 +189,18 @@ def _get_registered_timetable(importable_string: str) -> 
type[Timetable] | None:
         return None
 
 
+def _get_registered_priority_weight_strategy(importable_string: str) -> 
type[PriorityWeightStrategy] | None:
+    from airflow import plugins_manager
+
+    if importable_string in airflow_priority_weight_strategies:
+        return airflow_priority_weight_strategies[importable_string]
+    plugins_manager.initialize_priority_weight_strategy_plugins()
+    if plugins_manager.priority_weight_strategy_classes:
+        return 
plugins_manager.priority_weight_strategy_classes.get(importable_string)
+    else:
+        return None
+
+
 class _TimetableNotRegistered(ValueError):
     def __init__(self, type_string: str) -> None:
         self.type_string = type_string
@@ -196,6 +213,18 @@ class _TimetableNotRegistered(ValueError):
         )
 
 
+class _PriorityWeightStrategyNotRegistered(AirflowException):
+    def __init__(self, type_string: str) -> None:
+        self.type_string = type_string
+
+    def __str__(self) -> str:
+        return (
+            f"Priority weight strategy class {self.type_string!r} is not 
registered or "
+            "you have a top level database access that disrupted the session. "
+            "Please check the airflow best practices documentation."
+        )
+
+
 def encode_timetable(var: Timetable) -> dict[str, Any]:
     """
     Encode a timetable instance.
@@ -228,6 +257,36 @@ def decode_timetable(var: dict[str, Any]) -> Timetable:
     return timetable_class.deserialize(var[Encoding.VAR])
 
 
+def encode_priority_weight_strategy(var: PriorityWeightStrategy) -> str:
+    """
+    Encode a priority weight strategy instance.
+
+    In this version, we only store the importable string, so the class should 
not wait
+    for any parameters to be passed to it. If you need to store the 
parameters, you
+    should store them in the class itself.
+    """
+    priority_weight_strategy_class = type(var)
+    if priority_weight_strategy_class in 
airflow_priority_weight_strategies_classes:
+        return 
airflow_priority_weight_strategies_classes[priority_weight_strategy_class]
+    importable_string = qualname(priority_weight_strategy_class)
+    if _get_registered_priority_weight_strategy(importable_string) is None:
+        raise _PriorityWeightStrategyNotRegistered(importable_string)
+    return importable_string
+
+
+def decode_priority_weight_strategy(var: str) -> PriorityWeightStrategy:
+    """
+    Decode a previously serialized priority weight strategy.
+
+    In this version, we only store the importable string, so we just need to 
get the class
+    from the dictionary of registered classes and instantiate it with no 
parameters.
+    """
+    priority_weight_strategy_class = 
_get_registered_priority_weight_strategy(var)
+    if priority_weight_strategy_class is None:
+        raise _PriorityWeightStrategyNotRegistered(var)
+    return priority_weight_strategy_class()
+
+
 class _XComRef(NamedTuple):
     """
     Store info needed to create XComArg.
@@ -422,6 +481,8 @@ class BaseSerialization:
                 serialized_object[key] = cls.serialize(value)
             elif key == "timetable" and value is not None:
                 serialized_object[key] = encode_timetable(value)
+            elif key == "weight_rule" and value is not None:
+                serialized_object[key] = encode_priority_weight_strategy(value)
             elif key == "dataset_triggers":
                 serialized_object[key] = cls.serialize(value)
             else:
@@ -1064,6 +1125,8 @@ class SerializedBaseOperator(BaseOperator, 
BaseSerialization):
                 v = cls.deserialize(v)
             elif k == "on_failure_fail_dagrun":
                 k = "_on_failure_fail_dagrun"
+            elif k == "weight_rule":
+                v = decode_priority_weight_strategy(v)
             # else use v as it is
 
             setattr(op, k, v)
@@ -1411,6 +1474,8 @@ class SerializedDAG(DAG, BaseSerialization):
                 pass
             elif k == "timetable":
                 v = decode_timetable(v)
+            elif k == "weight_rule":
+                v = decode_priority_weight_strategy(v)
             elif k in cls._decorated_fields:
                 v = cls.deserialize(v)
             elif k == "params":
diff --git a/airflow/task/priority_strategy.py 
b/airflow/task/priority_strategy.py
new file mode 100644
index 0000000000..556539e220
--- /dev/null
+++ b/airflow/task/priority_strategy.py
@@ -0,0 +1,143 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Priority weight strategies for task scheduling."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+
+from airflow.exceptions import AirflowException
+
+if TYPE_CHECKING:
+    from airflow.models.taskinstance import TaskInstance
+
+
+class PriorityWeightStrategy(ABC):
+    """Priority weight strategy interface.
+
+    This feature is experimental and subject to change at any time.
+
+    Currently, we don't serialize the priority weight strategy parameters. 
This means that
+    the priority weight strategy must be stateless, but you can add class 
attributes, and
+    create multiple subclasses with different attributes values if you need to 
create
+    different versions of the same strategy.
+    """
+
+    @abstractmethod
+    def get_weight(self, ti: TaskInstance):
+        """Get the priority weight of a task."""
+        ...
+
+    @classmethod
+    def deserialize(cls, data: dict[str, Any]) -> PriorityWeightStrategy:
+        """Deserialize a priority weight strategy from data.
+
+        This is called when a serialized DAG is deserialized. ``data`` will be 
whatever
+        was returned by ``serialize`` during DAG serialization. The default
+        implementation constructs the priority weight strategy without any 
arguments.
+        """
+        return cls(**data)  # type: ignore[call-arg]
+
+    def serialize(self) -> dict[str, Any]:
+        """Serialize the priority weight strategy for JSON encoding.
+
+        This is called during DAG serialization to store priority weight 
strategy information
+        in the database. This should return a JSON-serializable dict that will 
be fed into
+        ``deserialize`` when the DAG is deserialized. The default 
implementation returns
+        an empty dict.
+        """
+        return {}
+
+    def __eq__(self, other: object) -> bool:
+        """Equality comparison."""
+        if not isinstance(other, type(self)):
+            return False
+        return self.serialize() == other.serialize()
+
+
+class _AbsolutePriorityWeightStrategy(PriorityWeightStrategy):
+    """Priority weight strategy that uses the task's priority weight 
directly."""
+
+    def get_weight(self, ti: TaskInstance):
+        return ti.task.priority_weight
+
+
+class _DownstreamPriorityWeightStrategy(PriorityWeightStrategy):
+    """Priority weight strategy that uses the sum of the priority weights of 
all downstream tasks."""
+
+    def get_weight(self, ti: TaskInstance) -> int:
+        dag = ti.task.get_dag()
+        if dag is None:
+            return ti.task.priority_weight
+        return ti.task.priority_weight + sum(
+            dag.task_dict[task_id].priority_weight
+            for task_id in ti.task.get_flat_relative_ids(upstream=False)
+        )
+
+
+class _UpstreamPriorityWeightStrategy(PriorityWeightStrategy):
+    """Priority weight strategy that uses the sum of the priority weights of 
all upstream tasks."""
+
+    def get_weight(self, ti: TaskInstance):
+        dag = ti.task.get_dag()
+        if dag is None:
+            return ti.task.priority_weight
+        return ti.task.priority_weight + sum(
+            dag.task_dict[task_id].priority_weight for task_id in 
ti.task.get_flat_relative_ids(upstream=True)
+        )
+
+
+airflow_priority_weight_strategies: dict[str, type[PriorityWeightStrategy]] = {
+    "absolute": _AbsolutePriorityWeightStrategy,
+    "downstream": _DownstreamPriorityWeightStrategy,
+    "upstream": _UpstreamPriorityWeightStrategy,
+}
+
+
+airflow_priority_weight_strategies_classes = {
+    cls: name for name, cls in airflow_priority_weight_strategies.items()
+}
+
+
+def validate_and_load_priority_weight_strategy(
+    priority_weight_strategy: str | PriorityWeightStrategy | None,
+) -> PriorityWeightStrategy:
+    """Validate and load a priority weight strategy.
+
+    Returns the priority weight strategy if it is valid, otherwise raises an 
exception.
+
+    :param priority_weight_strategy: The priority weight strategy to validate 
and load.
+
+    :meta private:
+    """
+    from airflow.serialization.serialized_objects import 
_get_registered_priority_weight_strategy
+    from airflow.utils.module_loading import qualname
+
+    if priority_weight_strategy is None:
+        return _AbsolutePriorityWeightStrategy()
+
+    if isinstance(priority_weight_strategy, str):
+        if priority_weight_strategy in airflow_priority_weight_strategies:
+            return 
airflow_priority_weight_strategies[priority_weight_strategy]()
+        priority_weight_strategy_class = priority_weight_strategy
+    else:
+        priority_weight_strategy_class = qualname(priority_weight_strategy)
+    loaded_priority_weight_strategy = 
_get_registered_priority_weight_strategy(priority_weight_strategy_class)
+    if loaded_priority_weight_strategy is None:
+        raise AirflowException(f"Unknown priority strategy 
{priority_weight_strategy_class}")
+    return loaded_priority_weight_strategy()
diff --git 
a/docs/apache-airflow/administration-and-deployment/priority-weight.rst 
b/docs/apache-airflow/administration-and-deployment/priority-weight.rst
index 3807b3ee5d..dd61d25fcd 100644
--- a/docs/apache-airflow/administration-and-deployment/priority-weight.rst
+++ b/docs/apache-airflow/administration-and-deployment/priority-weight.rst
@@ -62,3 +62,37 @@ Below are the weighting methods. By default, Airflow's 
weighting method is ``dow
 
 
 The ``priority_weight`` parameter can be used in conjunction with 
:ref:`concepts:pool`.
+
+Custom Weight Rule
+------------------
+
+.. versionadded:: 2.9.0
+
+You can implement your own custom weighting method by extending the 
``PriorityWeightStrategy`` class and
+registering it in a plugin.
+
+.. exampleinclude:: 
/../../airflow/example_dags/plugins/decreasing_priority_weight_strategy.py
+    :language: python
+    :dedent: 0
+    :start-after: [START custom_priority_weight_strategy]
+    :end-before: [END custom_priority_weight_strategy]
+
+
+Then to use it, you can create an instance of the custom class and provide it 
in the ``weight_rule`` parameter
+of the task or provide the path of the custom class:
+
+.. code-block:: python
+
+    from custom_weight_rule_module import CustomPriorityWeightStrategy
+
+    # provide the class instance
+    task1 = BashOperator(task_id="task", bash_command="echo 1", 
weight_rule=CustomPriorityWeightStrategy())
+
+    # or provide the path of the class
+    task1 = BashOperator(
+        task_id="task",
+        bash_command="echo 1",
+        weight_rule="custom_weight_rule_module.CustomPriorityWeightStrategy",
+    )
+
+|experimental|
diff --git a/tests/cli/commands/test_plugins_command.py 
b/tests/cli/commands/test_plugins_command.py
index cbf6afeab9..71d5ce5f72 100644
--- a/tests/cli/commands/test_plugins_command.py
+++ b/tests/cli/commands/test_plugins_command.py
@@ -101,6 +101,7 @@ class TestPluginsCommand:
                     },
                 ],
                 "ti_deps": ["<TIDep(CustomTestTriggerRule)>"],
+                "priority_weight_strategies": 
["tests.plugins.test_plugin.CustomPriorityWeightStrategy"],
             }
         ]
         get_listener_manager().clear()
diff --git a/tests/models/test_baseoperator.py 
b/tests/models/test_baseoperator.py
index d58a52224b..959bea7209 100644
--- a/tests/models/test_baseoperator.py
+++ b/tests/models/test_baseoperator.py
@@ -41,12 +41,12 @@ from airflow.models.baseoperator import (
 from airflow.models.dag import DAG
 from airflow.models.dagrun import DagRun
 from airflow.models.taskinstance import TaskInstance
+from airflow.task.priority_strategy import _DownstreamPriorityWeightStrategy, 
_UpstreamPriorityWeightStrategy
 from airflow.utils.edgemodifier import Label
 from airflow.utils.task_group import TaskGroup
 from airflow.utils.template import literal
 from airflow.utils.trigger_rule import TriggerRule
 from airflow.utils.types import DagRunType
-from airflow.utils.weight_rule import WeightRule
 from tests.models import DEFAULT_DATE
 from tests.test_utils.config import conf_vars
 from tests.test_utils.mock_operators import DeprecatedOperator, MockOperator
@@ -775,11 +775,11 @@ class TestBaseOperator:
 
     def test_weight_rule_default(self):
         op = BaseOperator(task_id="test_task")
-        assert WeightRule.DOWNSTREAM == op.weight_rule
+        assert _DownstreamPriorityWeightStrategy() == op.weight_rule
 
     def test_weight_rule_override(self):
         op = BaseOperator(task_id="test_task", weight_rule="upstream")
-        assert WeightRule.UPSTREAM == op.weight_rule
+        assert _UpstreamPriorityWeightStrategy() == op.weight_rule
 
     # ensure the default logging config is used for this test, no matter what 
ran before
     @pytest.mark.usefixtures("reset_logging_config")
diff --git a/tests/models/test_dag.py b/tests/models/test_dag.py
index 4dd29ecbe8..2836fb827f 100644
--- a/tests/models/test_dag.py
+++ b/tests/models/test_dag.py
@@ -90,10 +90,17 @@ from airflow.utils.trigger_rule import TriggerRule
 from airflow.utils.types import DagRunType
 from airflow.utils.weight_rule import WeightRule
 from tests.models import DEFAULT_DATE
+from tests.plugins.priority_weight_strategy import (
+    FactorPriorityWeightStrategy,
+    NotRegisteredPriorityWeightStrategy,
+    StaticTestPriorityWeightStrategy,
+    TestPriorityWeightStrategyPlugin,
+)
 from tests.test_utils.asserts import assert_queries_count
 from tests.test_utils.config import conf_vars
 from tests.test_utils.db import clear_db_dags, clear_db_datasets, 
clear_db_runs, clear_db_serialized_dags
 from tests.test_utils.mapping import expand_mapped_task
+from tests.test_utils.mock_plugins import mock_plugin_manager
 from tests.test_utils.timetables import cron_timetable, delta_timetable
 
 pytestmark = pytest.mark.db_test
@@ -433,6 +440,35 @@ class TestDag:
             with pytest.raises(AirflowException):
                 EmptyOperator(task_id="should_fail", weight_rule="no rule")
 
+    @pytest.mark.parametrize(
+        "cls, expected",
+        [
+            (StaticTestPriorityWeightStrategy, 99),
+            (FactorPriorityWeightStrategy, 3),
+        ],
+    )
+    def test_dag_task_custom_weight_strategy(self, cls, expected):
+        with mock_plugin_manager(plugins=[TestPriorityWeightStrategyPlugin]), 
DAG(
+            "dag", start_date=DEFAULT_DATE, default_args={"owner": "owner1"}
+        ) as dag:
+            task = EmptyOperator(
+                task_id="empty_task",
+                weight_rule=cls(),
+            )
+        dr = dag.create_dagrun(state=None, run_id="test", 
execution_date=DEFAULT_DATE)
+        ti = dr.get_task_instance(task.task_id)
+        assert ti.priority_weight == expected
+
+    def test_dag_task_not_registered_weight_strategy(self):
+        with mock_plugin_manager(plugins=[TestPriorityWeightStrategyPlugin]), 
DAG(
+            "dag", start_date=DEFAULT_DATE, default_args={"owner": "owner1"}
+        ):
+            with pytest.raises(AirflowException, match="Unknown priority 
strategy"):
+                EmptyOperator(
+                    task_id="empty_task",
+                    weight_rule=NotRegisteredPriorityWeightStrategy(),
+                )
+
     def test_get_num_task_instances(self):
         test_dag_id = "test_get_num_task_instances_dag"
         test_task_id = "task_1"
diff --git a/tests/plugins/priority_weight_strategy.py 
b/tests/plugins/priority_weight_strategy.py
new file mode 100644
index 0000000000..c56ae7364a
--- /dev/null
+++ b/tests/plugins/priority_weight_strategy.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from airflow.plugins_manager import AirflowPlugin
+from airflow.task.priority_strategy import PriorityWeightStrategy
+
+if TYPE_CHECKING:
+    from airflow.models import TaskInstance
+
+
+class StaticTestPriorityWeightStrategy(PriorityWeightStrategy):
+    def get_weight(self, ti: TaskInstance):
+        return 99
+
+
+class FactorPriorityWeightStrategy(PriorityWeightStrategy):
+    factor: int = 3
+
+    def serialize(self) -> dict[str, Any]:
+        return {"factor": self.factor}
+
+    def get_weight(self, ti: TaskInstance):
+        return max(ti.map_index, 1) * self.factor
+
+
+class DecreasingPriorityStrategy(PriorityWeightStrategy):
+    """A priority weight strategy that decreases the priority weight with each 
attempt."""
+
+    def get_weight(self, ti: TaskInstance):
+        return max(3 - ti._try_number + 1, 1)
+
+
+class TestPriorityWeightStrategyPlugin(AirflowPlugin):
+    # Without this import, the qualname method will not use the correct 
classes names
+    from tests.plugins.priority_weight_strategy import (
+        DecreasingPriorityStrategy,
+        FactorPriorityWeightStrategy,
+        StaticTestPriorityWeightStrategy,
+    )
+
+    name = "priority_weight_strategy_plugin"
+    priority_weight_strategies = [
+        StaticTestPriorityWeightStrategy,
+        FactorPriorityWeightStrategy,
+        DecreasingPriorityStrategy,
+    ]
+
+
+class NotRegisteredPriorityWeightStrategy(PriorityWeightStrategy):
+    def get_weight(self, ti: TaskInstance):
+        return 99
diff --git a/tests/plugins/test_plugin.py b/tests/plugins/test_plugin.py
index e207fd12da..717dab1a60 100644
--- a/tests/plugins/test_plugin.py
+++ b/tests/plugins/test_plugin.py
@@ -29,6 +29,7 @@ from airflow.models.baseoperator import BaseOperator
 # This is the class you derive to create a plugin
 from airflow.plugins_manager import AirflowPlugin
 from airflow.sensors.base import BaseSensorOperator
+from airflow.task.priority_strategy import PriorityWeightStrategy
 from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
 from airflow.timetables.interval import CronDataIntervalTimetable
 from tests.listeners import empty_listener
@@ -113,6 +114,11 @@ class CustomTestTriggerRule(BaseTIDep):
     pass
 
 
+class CustomPriorityWeightStrategy(PriorityWeightStrategy):
+    def get_weight(self, ti):
+        return 1
+
+
 # Defining the plugin class
 class AirflowTestPlugin(AirflowPlugin):
     name = "test_plugin"
@@ -132,6 +138,7 @@ class AirflowTestPlugin(AirflowPlugin):
     timetables = [CustomCronDataIntervalTimetable]
     listeners = [empty_listener, ClassBasedListener()]
     ti_deps = [CustomTestTriggerRule()]
+    priority_weight_strategies = [CustomPriorityWeightStrategy]
 
 
 class MockPluginA(AirflowPlugin):
diff --git a/tests/plugins/test_plugins_manager.py 
b/tests/plugins/test_plugins_manager.py
index 50501e4cbe..35058a9afd 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -209,7 +209,7 @@ class TestPluginsManager:
         with mock.patch("airflow.plugins_manager.plugins", []):
             plugins_manager.load_plugins_from_plugin_directory()
 
-            assert 6 == len(plugins_manager.plugins)
+            assert 7 == len(plugins_manager.plugins)
             for plugin in plugins_manager.plugins:
                 if "AirflowTestOnLoadPlugin" in str(plugin):
                     assert "postload" == plugin.name
diff --git a/tests/serialization/test_dag_serialization.py 
b/tests/serialization/test_dag_serialization.py
index e99f4641d5..a9fe733688 100644
--- a/tests/serialization/test_dag_serialization.py
+++ b/tests/serialization/test_dag_serialization.py
@@ -69,6 +69,7 @@ from airflow.serialization.serialized_objects import (
     SerializedBaseOperator,
     SerializedDAG,
 )
+from airflow.task.priority_strategy import _DownstreamPriorityWeightStrategy
 from airflow.ti_deps.deps.base_ti_dep import BaseTIDep
 from airflow.timetables.simple import NullTimetable, OnceTimetable
 from airflow.utils import timezone
@@ -190,6 +191,7 @@ serialized_simple_dag_ground_truth = {
                 },
                 "doc_md": "### Task Tutorial Documentation",
                 "_log_config_logger_name": "airflow.task.operators",
+                "weight_rule": "downstream",
             },
             {
                 "task_id": "custom_task",
@@ -213,6 +215,7 @@ serialized_simple_dag_ground_truth = {
                 "is_teardown": False,
                 "on_failure_fail_dagrun": False,
                 "_log_config_logger_name": "airflow.task.operators",
+                "weight_rule": "downstream",
             },
         ],
         "schedule_interval": {"__type": "timedelta", "__var": 86400.0},
@@ -1289,7 +1292,7 @@ class TestStringifiedDAGs:
             "trigger_rule": "all_success",
             "wait_for_downstream": False,
             "wait_for_past_depends_before_skipping": False,
-            "weight_rule": "downstream",
+            "weight_rule": _DownstreamPriorityWeightStrategy(),
             "multiple_outputs": False,
         }, """
 
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!


Reply via email to