kaxil commented on code in PR #76999:
URL: https://github.com/apache/airflow/pull/76999#discussion_r4606191196
##########
providers/common/ai/tests/unit/common/ai/policies/test_retry.py:
##########
@@ -955,6 +956,158 @@ def test_long_traceback_keeps_its_tail(self,
mock_hook_cls, truncated):
assert prompt == f"{PROMPT_HEADER}{expected}"
+class _RateLimited(Exception):
+ """A connection answered 429."""
+
+
+class _BadRequest(Exception):
+ """A connection answered 400."""
+
+
+def _exhausted_chain(*members: Exception, replayed=None, recorded=None) ->
FallbackChainExhaustedError:
+ return FallbackChainExhaustedError.from_group(
+ FallbackExceptionGroup("All models from FallbackModel failed",
list(members)),
+ replayed_steps=replayed,
+ recorded_steps=recorded,
+ )
+
+
+class TestFallbackChain:
+ """A failed chain reaches the policy as one error; the policy still sees
the members and the progress."""
+
+ @pytest.mark.parametrize("policy_cls", POLICY_CLASSES)
+ @patch(HOOK, autospec=True)
+ def test_prompt_names_every_member_error_in_chain_order(self,
mock_hook_cls, policy_cls):
+ # Without the member lines in ``_prompt`` the model sees only the
group's message.
+ error = _exhausted_chain(_RateLimited("primary throttled"),
_BadRequest("secondary rejected"))
+
+ prompt = _prompt_for(mock_hook_cls, policy_cls, error)
+
+ assert prompt.index("chain member 1 of 2: _RateLimited: primary
throttled") < prompt.index(
+ "chain member 2 of 2: _BadRequest: secondary rejected"
+ )
+
+ @pytest.mark.enable_redact
+ @pytest.mark.parametrize("policy_cls", POLICY_CLASSES)
+ @patch(HOOK, autospec=True)
+ def test_prompt_redacts_a_secret_in_a_member_error(self, mock_hook_cls,
policy_cls):
+ # Dropping the member lines from the redacted ``text`` leaks the
secret into the prompt.
+ reset_secrets_masker()
+ secret_value = "member-secret-token"
+ mask_secret(secret_value)
+ error = _exhausted_chain(_BadRequest(f"bad token {secret_value}"),
_RateLimited("slow down"))
+
+ prompt = _prompt_for(mock_hook_cls, policy_cls, error)
+
+ assert "chain member 1 of 2: _BadRequest: bad token ***" in prompt
+ assert secret_value not in prompt
+
+ @pytest.mark.parametrize("policy_cls", POLICY_CLASSES)
+ @patch(HOOK, autospec=True)
+ def
test_a_long_earlier_member_does_not_push_the_last_one_out_of_the_prompt(
+ self, mock_hook_cls, policy_cls
+ ):
+ # Truncating the joined text instead of each member keeps the head and
drops the last member.
+ error = _exhausted_chain(_RateLimited("x" * 500), _BadRequest("final
rejection"))
+
+ prompt = _prompt_for(mock_hook_cls, policy_cls, error,
max_exception_length=100)
+
+ assert "chain member 1 of 2: _RateLimited: " + "x" * 100 + "...
(truncated)" in prompt
+ assert "chain member 2 of 2: _BadRequest: final rejection" in prompt
+
+ @pytest.mark.parametrize("policy_cls", POLICY_CLASSES)
+ @patch(HOOK, autospec=True)
+ def test_prompt_says_how_far_the_durable_attempt_got(self, mock_hook_cls,
policy_cls):
+ error = _exhausted_chain(_RateLimited("a"), _RateLimited("b"),
replayed=2, recorded=1)
+
+ prompt = _prompt_for(mock_hook_cls, policy_cls, error)
+
+ assert "replayed 2 and recorded 1 durable journal steps" in prompt
+
+ @pytest.mark.parametrize("policy_cls", POLICY_CLASSES)
+ @patch(HOOK, autospec=True)
+ def test_prompt_has_no_progress_line_without_a_journal(self,
mock_hook_cls, policy_cls):
+ error = _exhausted_chain(_RateLimited("a"), _RateLimited("b"))
+
+ prompt = _prompt_for(mock_hook_cls, policy_cls, error)
+
+ assert "journal" not in prompt
+
+ @patch(HOOK, autospec=True)
+ def test_traceback_prompt_is_left_to_the_exception_group_formatter(self,
mock_hook_cls):
+ error = _exhausted_chain(_RateLimited("a"), _BadRequest("b"))
+
+ prompt = _prompt_for(mock_hook_cls, ClassifierRetryPolicy, error,
include_traceback=True)
+
+ assert "chain member" not in prompt
+ assert "_BadRequest: b" in prompt
+
+ @pytest.fixture
+ def rules(self):
+ return [
+ RetryRule(exception=_RateLimited, action=RetryAction.RETRY,
retry_delay=timedelta(seconds=60)),
+ RetryRule(exception=_BadRequest, action=RetryAction.FAIL,
reason="bad request"),
+ ]
+
+ @pytest.mark.parametrize("policy_cls", POLICY_CLASSES)
+ def test_rules_match_the_members_when_all_are_rate_limited(self, rules,
policy_cls):
+ # Without the last-member target, only the group is compared and no
rule matches.
+ policy = policy_cls(llm_conn_id="nonexistent", fallback_rules=rules)
+
+ decision = policy.evaluate(
+ _exhausted_chain(_RateLimited("a"), _RateLimited("b")),
try_number=1, max_tries=3
+ )
+
+ assert (decision.action, decision.retry_delay) == (RetryAction.RETRY,
timedelta(seconds=60))
+ assert decision.reason.endswith("(last chain member)")
+
+ @pytest.mark.parametrize(
+ ("members", "expected"),
+ [
+ pytest.param((_RateLimited("a"), _BadRequest("b")),
RetryAction.FAIL, id="last-is-400"),
+ pytest.param((_BadRequest("a"), _RateLimited("b")),
RetryAction.RETRY, id="last-is-429"),
+ ],
+ )
+ def test_the_last_member_decides_when_the_members_disagree(self, rules,
members, expected):
+ policy = LLMRetryPolicy(llm_conn_id="nonexistent",
fallback_rules=rules)
+
+ decision = policy.evaluate(_exhausted_chain(*members), try_number=1,
max_tries=3)
+
+ assert decision.action == expected
+
+ def test_a_rule_on_the_chain_error_itself_keeps_matching(self):
+ policy = LLMRetryPolicy(
+ llm_conn_id="nonexistent",
+ fallback_rules=[RetryRule(exception=FallbackExceptionGroup,
action=RetryAction.FAIL)],
+ )
+
+ decision = policy.evaluate(_exhausted_chain(_RateLimited("a")),
try_number=1, max_tries=3)
+
+ assert decision.action == RetryAction.FAIL
+ assert not decision.reason.endswith("(last chain member)")
+
+ def
test_rules_apply_in_the_order_they_are_listed_across_both_targets(self):
+ # A catch-all on the group listed first still wins over a member rule
listed after it.
+ policy = LLMRetryPolicy(
+ llm_conn_id="nonexistent",
+ fallback_rules=[
+ RetryRule(exception=FallbackExceptionGroup,
action=RetryAction.FAIL, reason="group"),
Review Comment:
With the group catch-all listed first, this passes whether `_rules_decision`
goes rule by rule or target by target (group, then last member), so swapping
its two `for` loops keeps the test green. Listing the `_RateLimited` rule first
and the group catch-all second would pin the documented order: that should give
`RETRY` with a reason ending in `(last chain member)`.
##########
providers/common/ai/docs/provider_fallback.rst:
##########
@@ -140,9 +140,15 @@ vendor can answer the same prompt now; a bad prompt, an
exhausted quota on every
a permanent data error still has to be decided by the retry policy.
Adding a chain changes what the retry layer sees. When every connection in the
chain
-fails, the exception the task raises is
``pydantic_ai.exceptions.FallbackExceptionGroup``,
-not the last provider's own exception, so retry rules matched against a
provider-specific
-exception type stop matching. Before adding a chain to a connection that Dags
already use,
+fails, the exception the task raises is
+:class:`~airflow.providers.common.ai.fallback_exceptions.FallbackChainExhaustedError`,
a subclass of
Review Comment:
This reads as true for every chain, but only `run_agent_sync` in
`CancellableAgentRunMixin` builds the wrapper. A `@task` that builds the agent
from `PydanticAIHook` (the "Configure it in code" example just above) still
raises the plain `FallbackExceptionGroup`, so `except
FallbackChainExhaustedError` never matches there and there are no journal
counts. Could this say it is the operators and `@task.llm` / `@task.agent` that
raise it, and what the hook path raises? `recovery.rst` and
`retry_policies.rst` have the same unscoped wording.
##########
providers/common/ai/docs/retry_policies.rst:
##########
@@ -379,15 +379,31 @@ retry behaviour with no change to the Dag. Two things
follow:
Separately, and regardless of this policy: if the connection **the task
itself** uses to call
the LLM (for example ``llm_conn_id`` on ``LLMOperator`` or ``AgentOperator``)
carries a fallback
chain, the exception the task raises once that chain is exhausted is
-``pydantic_ai.exceptions.FallbackExceptionGroup``, not the last provider's own
exception.
-``RetryRule`` matches with ``isinstance``, so a rule written as
-``RetryRule(exception=ModelHTTPError, ...)`` -- in ``fallback_rules`` here or
in a plain
-``ExceptionRetryPolicy`` -- stops matching. Match
``pydantic_ai.exceptions.FallbackExceptionGroup``
-explicitly as well; its only common ancestor with ``ModelAPIError`` is
``Exception``, too
-broad to write a rule against. The original per-model exceptions are still
available on
-``FallbackExceptionGroup.exceptions``, but ``RetryRule`` only compares the
top-level
-exception type, so a rule set that told 429s apart from 400s collapses into
one rule once
-the chain is in play.
+:class:`~airflow.providers.common.ai.fallback_exceptions.FallbackChainExhaustedError`.
It subclasses
+``pydantic_ai.exceptions.FallbackExceptionGroup``, so a rule or ``except``
clause written for the
+group keeps matching, and it is raised ``from`` the group the chain raised.
The error each
+connection raised is in ``member_errors``, in chain order.
+
+``RetryRule`` still compares one exception type, so the policies here read the
members for it:
+in ``fallback_rules``, each rule is tried against the chain's own error and
then against the
+error of the **last** connection, rule by rule in the order listed. A rule
written as
+``RetryRule(exception=ModelHTTPError, ...)`` therefore matches again, and a
rule set that tells
+failures apart by exception type, such as retrying a rate-limit error but
failing a bad-request
Review Comment:
This example needs a newer pydantic-ai than the `pydantic-ai-slim>=2.46.0`
floor. Before 2.59.0 there is no `ModelRateLimitError`, and before 2.62.0 no
`ModelRequestRejectedError`, so on older releases a 429 and a 400 both arrive
as `ModelHTTPError` and a rule can't tell them apart. Can we name the two
classes here and say they need 2.62.0+? The bad-request line in `recovery.rst`
could point at the same version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]