guan404ming commented on code in PR #74388:
URL: https://github.com/apache/airflow/pull/74388#discussion_r4396686359
##########
providers/common/ai/src/airflow/providers/common/ai/operators/llm.py:
##########
@@ -359,9 +360,6 @@ def _create_agent(self, *, output_type: Any, instructions:
str) -> Agent[object,
)
def execute(self, context: Context) -> Any:
- if self._may_review:
Review Comment:
Restored the early `validate_approval_prompt()` call in all four `execute`
methods, as on `main`, and added a test that an override raising there stops
the run before the hook is built, for `require_approval` and a below-threshold
`decision_policy`.
##########
providers/common/ai/tests/unit/common/ai/operators/test_llm_schema_compare.py:
##########
@@ -827,13 +827,35 @@ def
test_execute_complete_rejects_invalid_modified_output(self, modified):
with pytest.raises(ReviewedOutputValidationError, match="output_type
SchemaCompareResult"):
op.execute_complete({}, generated_output=result.model_dump_json(),
event=event)
- def test_execute_rejects_sequence_prompt_with_require_approval(self):
+ @mock.patch(
+
"airflow.providers.common.ai.operators.llm_schema_compare.LLMSchemaCompareOperator._build_schema_context"
+ )
+ @mock.patch(
+
"airflow.providers.common.ai.operators.llm_schema_compare.LLMSchemaCompareOperator._build_system_prompt"
+ )
+ @mock.patch("airflow.providers.standard.triggers.hitl.HITLTrigger",
autospec=True)
+ @mock.patch("airflow.sdk.execution_time.hitl.upsert_hitl_detail")
+ def test_execute_with_approval_reviews_a_sequence_prompt(
+ self,
+ mock_upsert,
+ mock_trigger_cls,
+ mock_build_system_prompt,
+ mock_build_schema_context,
+ make_mock_run_result,
+ ):
+ mock_build_schema_context.return_value = "schema_context"
+ mock_build_system_prompt.return_value = "system_prompt"
op = LLMSchemaCompareOperator(
- task_id="test_task",
- prompt=["describe", b"bytes"],
- llm_conn_id="llm_conn",
+ **{**_BASE_KWARGS, "prompt": ["Compare",
BinaryContent(data=b"\x89PNG", media_type="image/png")]},
**self._APPROVAL_KWARGS,
)
+ mock_agent = mock.Mock()
Review Comment:
The agent mock now uses `MagicMock(spec=["run_sync", "instrument"])` and the
hook `mock.Mock(spec=PydanticAIHook)`.
##########
providers/common/ai/tests/unit/common/ai/operators/test_llm.py:
##########
@@ -1647,25 +1633,44 @@ def
test_execute_complete_returns_an_unedited_output_that_does_not_validate_back
@pytest.mark.skipif(
not AIRFLOW_V_3_1_PLUS, reason="Human in the loop is only compatible with
Airflow >= 3.1.0"
)
-class TestLLMOperatorMultimodalPromptGuard:
- """LLMOperator.execute raises before agent.run_sync when require_approval
is True
- and self.prompt is not a string -- covering direct-operator construction
and the
- native template rendering escape (where a string template renders to a
Sequence)."""
-
+class TestLLMOperatorMultimodalPromptApproval:
+ @pytest.mark.parametrize(
+ "review_kwargs",
+ [
+ pytest.param({"require_approval": True}, id="require-approval"),
+ pytest.param({"decision_policy":
DecisionPolicy(min_confidence=0.7)}, id="below-threshold"),
+ ],
+ )
+ @patch("airflow.providers.standard.triggers.hitl.HITLTrigger",
autospec=True)
+ @patch("airflow.sdk.execution_time.hitl.upsert_hitl_detail")
@patch("airflow.providers.common.ai.operators.llm.PydanticAIHook",
autospec=True)
- def test_execute_rejects_sequence_prompt_with_require_approval(self,
mock_hook_cls):
- mock_agent = MagicMock(spec=["run_sync"])
+ def test_execute_with_approval_reviews_a_sequence_prompt(
+ self, mock_hook_cls, mock_upsert, mock_trigger_cls,
make_mock_run_result, review_kwargs
+ ):
+ result = make_mock_run_result(Assessment(category="billing",
score=0.3))
+ result.response = ModelResponse(
Review Comment:
Moved `_result` to module level and reused it in the multimodal test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
