This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13000-a3743d7f52e9200a7733857ca813a0836bf66bba in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 3cb9efc2a15223372bc1f87a92d027845a8ca800 Author: SEZ <[email protected]> AuthorDate: Thu Nov 5 07:05:21 2026 +0000 [Fix][seatunnel-cli] Keep the original request when answering a clarification (#13000) Co-authored-by: SEZ9 <[email protected]> --- seatunnel-cli/seatunnel_cli/agents.py | 68 +++++- seatunnel-cli/seatunnel_cli/cli.py | 13 +- .../tests/test_orchestrator_clarification.py | 268 +++++++++++++++++++++ 3 files changed, 335 insertions(+), 14 deletions(-) diff --git a/seatunnel-cli/seatunnel_cli/agents.py b/seatunnel-cli/seatunnel_cli/agents.py index 81c42ed7a4..76ae359b89 100644 --- a/seatunnel-cli/seatunnel_cli/agents.py +++ b/seatunnel-cli/seatunnel_cli/agents.py @@ -1270,6 +1270,12 @@ class Orchestrator: self.on_stream = on_stream or (lambda *a: None) self.memory_store = memory_store self.pending_question: str | None = None + # The request that started the current exchange. A clarifying question + # splits one requirement across two turns, and only the planner reads + # the conversation history -- skill matching, slot filling and the + # enriched prompt are all built from the latest message alone. Without + # this the reply replaces the requirement instead of adding to it. + self._original_request: str | None = None self._connector_metadata_block: str = "" # ─── Context window management ─── @@ -1283,8 +1289,15 @@ class Orchestrator: SUMMARY_THRESHOLD = 24 # only summarize when total exceeds this def load_history(self, history: list[dict]) -> None: - """Replace conversation history (e.g., when resuming a session).""" + """Replace conversation history (e.g., when resuming a session). + + The clarification state belongs to the exchange being replaced, so it + is cleared too. Leaving it would merge the first message after a resume + into whatever requirement happened to be open before. + """ self.conversation_history = list(history) + self.pending_question = None + self._original_request = None def _trimmed_history(self) -> list[dict]: """Return a context-window-friendly view of conversation_history. @@ -1366,6 +1379,34 @@ class Orchestrator: system = system + "\n\n" + block return system + def _effective_request(self, user_input: str) -> str: + """The requirement as it now stands, across a clarifying question. + + With no question outstanding this is just the message, which also + becomes the requirement the next reply will be added to. While a + question is outstanding the reply is appended rather than substituted, + and stays appended across further rounds. + + A message sent while a question is outstanding is treated as an answer + to it. There is no reliable way to tell an answer from an unrelated new + request, and merging is the safer default: the planner reads the whole + history regardless, so the alternative loses the requirement outright. + """ + if not self.pending_question: + self._original_request = user_input + return user_input + + if not self._original_request: + self._original_request = user_input + return user_input + + self._original_request = ( + f"{self._original_request}\n\n" + f"Additional details, in reply to a clarifying question:\n" + f"{user_input}" + ) + return self._original_request + def process_user_input(self, user_input: str) -> dict: """Process a user message and return result. @@ -1382,14 +1423,24 @@ class Orchestrator: "content": [{"text": user_input}], }) + # When this message answers a question we asked, the requirement lives + # in the earlier turn, so everything downstream has to see both. A + # reply like "use sensible defaults" carries no source, no sink and no + # table name, and on its own it produces an empty plan and an empty + # connector-metadata block. + effective_request = self._effective_request(user_input) + # Phase 1: Planning self.on_status("thinking", "Analyzing your request...") plan_result = self._run_planner() if plan_result["type"] == "question": + self.pending_question = plan_result.get("content") return plan_result if plan_result["type"] == "chat": + self.pending_question = None + self._original_request = None self.conversation_history.append({ "role": "assistant", "content": [{"text": plan_result["content"]}], @@ -1402,16 +1453,18 @@ class Orchestrator: structured_plan = parse_structured_plan(plan_result["content"]) # Trigger-based skill matching: uses plan + user input keywords - matched_skills = SkillRouter.match(structured_plan, user_input) + matched_skills = SkillRouter.match(structured_plan, effective_request) skill = SkillExecutor(structured_plan, matched_skills) # Fill slots (with pipeline expansion) and check for missing info - missing = skill.fill_and_check(user_input, self.memory_store, self.client) + missing = skill.fill_and_check( + effective_request, self.memory_store, self.client) if missing: question = ( "I need a few more details to generate the config:\n" + "\n".join(f"- {m}" for m in missing) ) + self.pending_question = question return { "type": "question", "content": question, @@ -1422,7 +1475,7 @@ class Orchestrator: # Fetch metadata and build enriched prompt self._connector_metadata_block = skill.fetch_all_metadata(self.on_status) enriched_prompt = skill.build_enriched_prompt( - user_input, self.on_status, self.memory_store, + effective_request, self.on_status, self.memory_store, ) # Phase 2: Config Generation (with enriched prompt from Skill) @@ -1450,6 +1503,11 @@ class Orchestrator: elif dryrun["valid"]: dryrun_note = "\n\n**Dry-run:** PASSED" + # The requirement has been served, so the next message starts a + # new one rather than adding to this. + self.pending_question = None + self._original_request = None + # Add assistant message to history self.conversation_history.append({ "role": "assistant", @@ -1474,6 +1532,8 @@ class Orchestrator: break # Return best effort after max rounds + self.pending_question = None + self._original_request = None self.conversation_history.append({ "role": "assistant", "content": [{"text": f"Here is the generated config:\n```hocon\n{config}\n```\n\n{explanation}"}], diff --git a/seatunnel-cli/seatunnel_cli/cli.py b/seatunnel-cli/seatunnel_cli/cli.py index c81db9bf44..40fe715224 100644 --- a/seatunnel-cli/seatunnel_cli/cli.py +++ b/seatunnel-cli/seatunnel_cli/cli.py @@ -870,16 +870,9 @@ class SeaTunnelCLI: self._handle_command(user_input) continue - # If previous turn was a clarification question, merge the - # original request with the user's answer so the planner and - # slot-checker see the full context. - if self._pending_request: - user_input = ( - f"{self._pending_request}\n\n" - f"Additional details from user: {user_input}" - ) - self._pending_request = None - + # The Orchestrator merges a clarification reply with the request it + # answers; it owns `pending_question`, so doing it here as well + # would prepend the requirement twice. # Process natural language request self._process_request(user_input) diff --git a/seatunnel-cli/tests/test_orchestrator_clarification.py b/seatunnel-cli/tests/test_orchestrator_clarification.py new file mode 100644 index 0000000000..918ca747fa --- /dev/null +++ b/seatunnel-cli/tests/test_orchestrator_clarification.py @@ -0,0 +1,268 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A clarifying question must not drop the requirement that preceded it. + +Only the planner reads the conversation history. Skill matching, slot filling +and the enriched prompt are all built from the latest message, so before this +was fixed a reply such as "use sensible defaults" became the whole request and +the generator never saw what the user had actually asked for. +""" + +import unittest +from unittest import mock + +from seatunnel_cli.agents import Orchestrator + + +REQUEST = ( + "Sync the users table from MySQL at localhost:3306, database shop, " + "user root into the existing users_replica table in the same database." +) +REPLY = ( + "Use sensible defaults for anything unspecified. Do not ask further " + "questions - generate the config now." +) + + +class EffectiveRequestTest(unittest.TestCase): + def setUp(self): + self.orchestrator = Orchestrator(client=mock.MagicMock()) + + def test_plain_message_is_the_request(self): + self.assertEqual(self.orchestrator._effective_request(REQUEST), REQUEST) + + def test_reply_to_a_question_is_added_not_substituted(self): + self.orchestrator._effective_request(REQUEST) + self.orchestrator.pending_question = "Which table should I write to?" + + merged = self.orchestrator._effective_request(REPLY) + + # The requirement has to survive; the reply is extra detail, and on its + # own it names no source, no sink and no table. + self.assertIn("users_replica", merged) + self.assertIn("localhost:3306", merged) + self.assertIn(REPLY, merged) + + def test_several_rounds_keep_accumulating(self): + self.orchestrator._effective_request(REQUEST) + self.orchestrator.pending_question = "q1" + self.orchestrator._effective_request("The target table already exists.") + self.orchestrator.pending_question = "q2" + + merged = self.orchestrator._effective_request("Batch mode is fine.") + + self.assertIn("users_replica", merged) + self.assertIn("The target table already exists.", merged) + self.assertIn("Batch mode is fine.", merged) + + def test_the_requirement_is_not_prepended_twice(self): + """Only one component may do the merge. + + The interactive loop in `cli.py` used to merge the reply with the + request before calling in, which together with this would have + prepended the requirement twice. That merge was removed in favour of + this one, because the Orchestrator owns `pending_question` and the + programmatic callers -- the benchmark among them -- never went through + `cli.py` at all. + """ + self.orchestrator._effective_request(REQUEST) + self.orchestrator.pending_question = "Which table?" + merged = self.orchestrator._effective_request(REPLY) + + self.assertEqual(merged.count("users_replica"), 1) + + def test_a_new_message_after_delivery_starts_over(self): + self.orchestrator._effective_request(REQUEST) + # Delivering a config clears the exchange. + self.orchestrator.pending_question = None + self.orchestrator._original_request = None + + fresh = self.orchestrator._effective_request("Print the orders table.") + + self.assertEqual(fresh, "Print the orders table.") + self.assertNotIn("users_replica", fresh) + + +class EnrichedPromptAfterClarificationTest(unittest.TestCase): + """End to end over `process_user_input`, which is where the loss happened. + + The planner is stubbed because the point under test is what the *generator* + receives, not how the plan is produced. + """ + + def _orchestrator_with_stubs(self, seen): + orchestrator = Orchestrator(client=mock.MagicMock()) + + skill = mock.MagicMock() + skill.fill_and_check.return_value = [] + skill.fetch_all_metadata.return_value = "## Connector Option Rules\n" + + def build_enriched_prompt(request, on_status, memory_store=None): + seen.append(request) + return f"## User Request:\n{request}" + + skill.build_enriched_prompt.side_effect = build_enriched_prompt + return orchestrator, skill + + def test_generator_still_sees_the_original_request(self): + seen: list[str] = [] + orchestrator, skill = self._orchestrator_with_stubs(seen) + + with mock.patch.object( + orchestrator, "_run_planner", + side_effect=[ + {"type": "question", "content": "Which database?", + "config": None, "explanation": None}, + {"type": "plan", "content": "source: Jdbc\nsink: Jdbc\nmode: BATCH"}, + ], + ), mock.patch("seatunnel_cli.skills.SkillExecutor", return_value=skill), \ + mock.patch("seatunnel_cli.skills.SkillRouter.match", return_value=[]), \ + mock.patch("seatunnel_cli.skills.parse_structured_plan") as parse_plan, \ + mock.patch.object( + orchestrator, "_run_config_generator", + return_value={"config": "env {}\nsource {}\nsink {}", + "explanation": ""}), \ + mock.patch.object(orchestrator, "_run_validator", return_value="PASS"), \ + mock.patch("seatunnel_cli.agents.dry_run_config", + return_value={"valid": True, "phase1_local": "VALID", + "phase2_check": "PASS", "phase3_api": None, + "summary": ""}): + parse_plan.return_value = mock.MagicMock(pipelines=[mock.MagicMock()]) + + first = orchestrator.process_user_input(REQUEST) + self.assertEqual(first["type"], "question") + + second = orchestrator.process_user_input(REPLY) + self.assertEqual(second["type"], "config") + + self.assertEqual(len(seen), 1, "the generator should run once, after the reply") + prompt_request = seen[0] + self.assertIn("users_replica", prompt_request) + self.assertIn("localhost:3306", prompt_request) + + def test_a_question_records_the_pending_state(self): + seen: list[str] = [] + orchestrator, skill = self._orchestrator_with_stubs(seen) + + with mock.patch.object( + orchestrator, "_run_planner", + return_value={"type": "question", "content": "Which database?", + "config": None, "explanation": None}, + ): + orchestrator.process_user_input(REQUEST) + + self.assertEqual(orchestrator.pending_question, "Which database?") + self.assertEqual(orchestrator._original_request, REQUEST) + + +class PendingStateIsClearedTest(unittest.TestCase): + """Every exit that serves or abandons the requirement must clear it. + + Without these, removing any single reset would leave a stale requirement to + be merged into the next, unrelated, request and no test would go red. + """ + + def _orchestrator(self): + orchestrator = Orchestrator(client=mock.MagicMock()) + orchestrator.pending_question = "Which database?" + orchestrator._original_request = REQUEST + return orchestrator + + def _assert_cleared(self, orchestrator): + self.assertIsNone(orchestrator.pending_question) + self.assertIsNone(orchestrator._original_request) + + def _stub_skill(self): + skill = mock.MagicMock() + skill.fill_and_check.return_value = [] + skill.fetch_all_metadata.return_value = "## Connector Option Rules\n" + skill.build_enriched_prompt.return_value = "## User Request:\nx" + return skill + + def test_a_delivered_config_clears_it(self): + orchestrator = self._orchestrator() + + with mock.patch.object( + orchestrator, "_run_planner", + return_value={"type": "plan", + "content": "source: Jdbc\nsink: Console\nmode: BATCH"}, + ), mock.patch("seatunnel_cli.skills.SkillExecutor", + return_value=self._stub_skill()), \ + mock.patch("seatunnel_cli.skills.SkillRouter.match", return_value=[]), \ + mock.patch.object( + orchestrator, "_run_config_generator", + return_value={"config": "env {}\nsource {}\nsink {}", + "explanation": ""}), \ + mock.patch.object(orchestrator, "_run_validator", return_value="PASS"), \ + mock.patch("seatunnel_cli.agents.dry_run_config", + return_value={"valid": True, "phase1_local": "VALID", + "phase2_check": "PASS", "phase3_api": None, + "summary": ""}): + result = orchestrator.process_user_input("Batch is fine.") + + self.assertEqual(result["type"], "config") + self._assert_cleared(orchestrator) + + def test_a_best_effort_config_after_three_rounds_clears_it(self): + orchestrator = self._orchestrator() + + with mock.patch.object( + orchestrator, "_run_planner", + return_value={"type": "plan", + "content": "source: Jdbc\nsink: Console\nmode: BATCH"}, + ), mock.patch("seatunnel_cli.skills.SkillExecutor", + return_value=self._stub_skill()), \ + mock.patch("seatunnel_cli.skills.SkillRouter.match", return_value=[]), \ + mock.patch.object( + orchestrator, "_run_config_generator", + return_value={"config": "env {}\nsource {}\nsink {}", + "explanation": ""}), \ + mock.patch.object(orchestrator, "_run_validator", + return_value="FAIL: still broken"), \ + mock.patch.object(orchestrator, "_run_fix", + return_value={"config": "env {}\nsource {}\nsink {}", + "explanation": ""}): + result = orchestrator.process_user_input("Batch is fine.") + + self.assertEqual(result["type"], "config") + self._assert_cleared(orchestrator) + + def test_a_chat_reply_clears_it(self): + orchestrator = self._orchestrator() + + with mock.patch.object( + orchestrator, "_run_planner", + return_value={"type": "chat", "content": "SeaTunnel is an ETL tool."}, + ): + result = orchestrator.process_user_input("What is SeaTunnel?") + + self.assertEqual(result["type"], "chat") + self._assert_cleared(orchestrator) + + def test_resuming_a_session_clears_it(self): + orchestrator = self._orchestrator() + + orchestrator.load_history([ + {"role": "user", "content": [{"text": "an unrelated earlier session"}]}, + ]) + + self._assert_cleared(orchestrator) + + +if __name__ == "__main__": + unittest.main()
