This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/dev/pr-13000-a3743d7f52e9200a7733857ca813a0836bf66bba
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit 3cb9efc2a15223372bc1f87a92d027845a8ca800
Author: SEZ <[email protected]>
AuthorDate: Thu Nov 5 07:05:21 2026 +0000

    [Fix][seatunnel-cli] Keep the original request when answering a 
clarification (#13000)
    
    Co-authored-by: SEZ9 <[email protected]>
---
 seatunnel-cli/seatunnel_cli/agents.py              |  68 +++++-
 seatunnel-cli/seatunnel_cli/cli.py                 |  13 +-
 .../tests/test_orchestrator_clarification.py       | 268 +++++++++++++++++++++
 3 files changed, 335 insertions(+), 14 deletions(-)

diff --git a/seatunnel-cli/seatunnel_cli/agents.py 
b/seatunnel-cli/seatunnel_cli/agents.py
index 81c42ed7a4..76ae359b89 100644
--- a/seatunnel-cli/seatunnel_cli/agents.py
+++ b/seatunnel-cli/seatunnel_cli/agents.py
@@ -1270,6 +1270,12 @@ class Orchestrator:
         self.on_stream = on_stream or (lambda *a: None)
         self.memory_store = memory_store
         self.pending_question: str | None = None
+        # The request that started the current exchange. A clarifying question
+        # splits one requirement across two turns, and only the planner reads
+        # the conversation history -- skill matching, slot filling and the
+        # enriched prompt are all built from the latest message alone. Without
+        # this the reply replaces the requirement instead of adding to it.
+        self._original_request: str | None = None
         self._connector_metadata_block: str = ""
 
     # ─── Context window management ───
@@ -1283,8 +1289,15 @@ class Orchestrator:
     SUMMARY_THRESHOLD = 24     # only summarize when total exceeds this
 
     def load_history(self, history: list[dict]) -> None:
-        """Replace conversation history (e.g., when resuming a session)."""
+        """Replace conversation history (e.g., when resuming a session).
+
+        The clarification state belongs to the exchange being replaced, so it
+        is cleared too. Leaving it would merge the first message after a resume
+        into whatever requirement happened to be open before.
+        """
         self.conversation_history = list(history)
+        self.pending_question = None
+        self._original_request = None
 
     def _trimmed_history(self) -> list[dict]:
         """Return a context-window-friendly view of conversation_history.
@@ -1366,6 +1379,34 @@ class Orchestrator:
                 system = system + "\n\n" + block
         return system
 
+    def _effective_request(self, user_input: str) -> str:
+        """The requirement as it now stands, across a clarifying question.
+
+        With no question outstanding this is just the message, which also
+        becomes the requirement the next reply will be added to. While a
+        question is outstanding the reply is appended rather than substituted,
+        and stays appended across further rounds.
+
+        A message sent while a question is outstanding is treated as an answer
+        to it. There is no reliable way to tell an answer from an unrelated new
+        request, and merging is the safer default: the planner reads the whole
+        history regardless, so the alternative loses the requirement outright.
+        """
+        if not self.pending_question:
+            self._original_request = user_input
+            return user_input
+
+        if not self._original_request:
+            self._original_request = user_input
+            return user_input
+
+        self._original_request = (
+            f"{self._original_request}\n\n"
+            f"Additional details, in reply to a clarifying question:\n"
+            f"{user_input}"
+        )
+        return self._original_request
+
     def process_user_input(self, user_input: str) -> dict:
         """Process a user message and return result.
 
@@ -1382,14 +1423,24 @@ class Orchestrator:
             "content": [{"text": user_input}],
         })
 
+        # When this message answers a question we asked, the requirement lives
+        # in the earlier turn, so everything downstream has to see both. A
+        # reply like "use sensible defaults" carries no source, no sink and no
+        # table name, and on its own it produces an empty plan and an empty
+        # connector-metadata block.
+        effective_request = self._effective_request(user_input)
+
         # Phase 1: Planning
         self.on_status("thinking", "Analyzing your request...")
         plan_result = self._run_planner()
 
         if plan_result["type"] == "question":
+            self.pending_question = plan_result.get("content")
             return plan_result
 
         if plan_result["type"] == "chat":
+            self.pending_question = None
+            self._original_request = None
             self.conversation_history.append({
                 "role": "assistant",
                 "content": [{"text": plan_result["content"]}],
@@ -1402,16 +1453,18 @@ class Orchestrator:
         structured_plan = parse_structured_plan(plan_result["content"])
 
         # Trigger-based skill matching: uses plan + user input keywords
-        matched_skills = SkillRouter.match(structured_plan, user_input)
+        matched_skills = SkillRouter.match(structured_plan, effective_request)
         skill = SkillExecutor(structured_plan, matched_skills)
 
         # Fill slots (with pipeline expansion) and check for missing info
-        missing = skill.fill_and_check(user_input, self.memory_store, 
self.client)
+        missing = skill.fill_and_check(
+            effective_request, self.memory_store, self.client)
         if missing:
             question = (
                 "I need a few more details to generate the config:\n"
                 + "\n".join(f"- {m}" for m in missing)
             )
+            self.pending_question = question
             return {
                 "type": "question",
                 "content": question,
@@ -1422,7 +1475,7 @@ class Orchestrator:
         # Fetch metadata and build enriched prompt
         self._connector_metadata_block = 
skill.fetch_all_metadata(self.on_status)
         enriched_prompt = skill.build_enriched_prompt(
-            user_input, self.on_status, self.memory_store,
+            effective_request, self.on_status, self.memory_store,
         )
 
         # Phase 2: Config Generation (with enriched prompt from Skill)
@@ -1450,6 +1503,11 @@ class Orchestrator:
                 elif dryrun["valid"]:
                     dryrun_note = "\n\n**Dry-run:** PASSED"
 
+                # The requirement has been served, so the next message starts a
+                # new one rather than adding to this.
+                self.pending_question = None
+                self._original_request = None
+
                 # Add assistant message to history
                 self.conversation_history.append({
                     "role": "assistant",
@@ -1474,6 +1532,8 @@ class Orchestrator:
                 break
 
         # Return best effort after max rounds
+        self.pending_question = None
+        self._original_request = None
         self.conversation_history.append({
             "role": "assistant",
             "content": [{"text": f"Here is the generated 
config:\n```hocon\n{config}\n```\n\n{explanation}"}],
diff --git a/seatunnel-cli/seatunnel_cli/cli.py 
b/seatunnel-cli/seatunnel_cli/cli.py
index c81db9bf44..40fe715224 100644
--- a/seatunnel-cli/seatunnel_cli/cli.py
+++ b/seatunnel-cli/seatunnel_cli/cli.py
@@ -870,16 +870,9 @@ class SeaTunnelCLI:
                 self._handle_command(user_input)
                 continue
 
-            # If previous turn was a clarification question, merge the
-            # original request with the user's answer so the planner and
-            # slot-checker see the full context.
-            if self._pending_request:
-                user_input = (
-                    f"{self._pending_request}\n\n"
-                    f"Additional details from user: {user_input}"
-                )
-                self._pending_request = None
-
+            # The Orchestrator merges a clarification reply with the request it
+            # answers; it owns `pending_question`, so doing it here as well
+            # would prepend the requirement twice.
             # Process natural language request
             self._process_request(user_input)
 
diff --git a/seatunnel-cli/tests/test_orchestrator_clarification.py 
b/seatunnel-cli/tests/test_orchestrator_clarification.py
new file mode 100644
index 0000000000..918ca747fa
--- /dev/null
+++ b/seatunnel-cli/tests/test_orchestrator_clarification.py
@@ -0,0 +1,268 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A clarifying question must not drop the requirement that preceded it.
+
+Only the planner reads the conversation history. Skill matching, slot filling
+and the enriched prompt are all built from the latest message, so before this
+was fixed a reply such as "use sensible defaults" became the whole request and
+the generator never saw what the user had actually asked for.
+"""
+
+import unittest
+from unittest import mock
+
+from seatunnel_cli.agents import Orchestrator
+
+
+REQUEST = (
+    "Sync the users table from MySQL at localhost:3306, database shop, "
+    "user root into the existing users_replica table in the same database."
+)
+REPLY = (
+    "Use sensible defaults for anything unspecified. Do not ask further "
+    "questions - generate the config now."
+)
+
+
+class EffectiveRequestTest(unittest.TestCase):
+    def setUp(self):
+        self.orchestrator = Orchestrator(client=mock.MagicMock())
+
+    def test_plain_message_is_the_request(self):
+        self.assertEqual(self.orchestrator._effective_request(REQUEST), 
REQUEST)
+
+    def test_reply_to_a_question_is_added_not_substituted(self):
+        self.orchestrator._effective_request(REQUEST)
+        self.orchestrator.pending_question = "Which table should I write to?"
+
+        merged = self.orchestrator._effective_request(REPLY)
+
+        # The requirement has to survive; the reply is extra detail, and on its
+        # own it names no source, no sink and no table.
+        self.assertIn("users_replica", merged)
+        self.assertIn("localhost:3306", merged)
+        self.assertIn(REPLY, merged)
+
+    def test_several_rounds_keep_accumulating(self):
+        self.orchestrator._effective_request(REQUEST)
+        self.orchestrator.pending_question = "q1"
+        self.orchestrator._effective_request("The target table already 
exists.")
+        self.orchestrator.pending_question = "q2"
+
+        merged = self.orchestrator._effective_request("Batch mode is fine.")
+
+        self.assertIn("users_replica", merged)
+        self.assertIn("The target table already exists.", merged)
+        self.assertIn("Batch mode is fine.", merged)
+
+    def test_the_requirement_is_not_prepended_twice(self):
+        """Only one component may do the merge.
+
+        The interactive loop in `cli.py` used to merge the reply with the
+        request before calling in, which together with this would have
+        prepended the requirement twice. That merge was removed in favour of
+        this one, because the Orchestrator owns `pending_question` and the
+        programmatic callers -- the benchmark among them -- never went through
+        `cli.py` at all.
+        """
+        self.orchestrator._effective_request(REQUEST)
+        self.orchestrator.pending_question = "Which table?"
+        merged = self.orchestrator._effective_request(REPLY)
+
+        self.assertEqual(merged.count("users_replica"), 1)
+
+    def test_a_new_message_after_delivery_starts_over(self):
+        self.orchestrator._effective_request(REQUEST)
+        # Delivering a config clears the exchange.
+        self.orchestrator.pending_question = None
+        self.orchestrator._original_request = None
+
+        fresh = self.orchestrator._effective_request("Print the orders table.")
+
+        self.assertEqual(fresh, "Print the orders table.")
+        self.assertNotIn("users_replica", fresh)
+
+
+class EnrichedPromptAfterClarificationTest(unittest.TestCase):
+    """End to end over `process_user_input`, which is where the loss happened.
+
+    The planner is stubbed because the point under test is what the *generator*
+    receives, not how the plan is produced.
+    """
+
+    def _orchestrator_with_stubs(self, seen):
+        orchestrator = Orchestrator(client=mock.MagicMock())
+
+        skill = mock.MagicMock()
+        skill.fill_and_check.return_value = []
+        skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+
+        def build_enriched_prompt(request, on_status, memory_store=None):
+            seen.append(request)
+            return f"## User Request:\n{request}"
+
+        skill.build_enriched_prompt.side_effect = build_enriched_prompt
+        return orchestrator, skill
+
+    def test_generator_still_sees_the_original_request(self):
+        seen: list[str] = []
+        orchestrator, skill = self._orchestrator_with_stubs(seen)
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            side_effect=[
+                {"type": "question", "content": "Which database?",
+                 "config": None, "explanation": None},
+                {"type": "plan", "content": "source: Jdbc\nsink: Jdbc\nmode: 
BATCH"},
+            ],
+        ), mock.patch("seatunnel_cli.skills.SkillExecutor", 
return_value=skill), \
+             mock.patch("seatunnel_cli.skills.SkillRouter.match", 
return_value=[]), \
+             mock.patch("seatunnel_cli.skills.parse_structured_plan") as 
parse_plan, \
+             mock.patch.object(
+                 orchestrator, "_run_config_generator",
+                 return_value={"config": "env {}\nsource {}\nsink {}",
+                               "explanation": ""}), \
+             mock.patch.object(orchestrator, "_run_validator", 
return_value="PASS"), \
+             mock.patch("seatunnel_cli.agents.dry_run_config",
+                        return_value={"valid": True, "phase1_local": "VALID",
+                                      "phase2_check": "PASS", "phase3_api": 
None,
+                                      "summary": ""}):
+            parse_plan.return_value = 
mock.MagicMock(pipelines=[mock.MagicMock()])
+
+            first = orchestrator.process_user_input(REQUEST)
+            self.assertEqual(first["type"], "question")
+
+            second = orchestrator.process_user_input(REPLY)
+            self.assertEqual(second["type"], "config")
+
+        self.assertEqual(len(seen), 1, "the generator should run once, after 
the reply")
+        prompt_request = seen[0]
+        self.assertIn("users_replica", prompt_request)
+        self.assertIn("localhost:3306", prompt_request)
+
+    def test_a_question_records_the_pending_state(self):
+        seen: list[str] = []
+        orchestrator, skill = self._orchestrator_with_stubs(seen)
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            return_value={"type": "question", "content": "Which database?",
+                          "config": None, "explanation": None},
+        ):
+            orchestrator.process_user_input(REQUEST)
+
+        self.assertEqual(orchestrator.pending_question, "Which database?")
+        self.assertEqual(orchestrator._original_request, REQUEST)
+
+
+class PendingStateIsClearedTest(unittest.TestCase):
+    """Every exit that serves or abandons the requirement must clear it.
+
+    Without these, removing any single reset would leave a stale requirement to
+    be merged into the next, unrelated, request and no test would go red.
+    """
+
+    def _orchestrator(self):
+        orchestrator = Orchestrator(client=mock.MagicMock())
+        orchestrator.pending_question = "Which database?"
+        orchestrator._original_request = REQUEST
+        return orchestrator
+
+    def _assert_cleared(self, orchestrator):
+        self.assertIsNone(orchestrator.pending_question)
+        self.assertIsNone(orchestrator._original_request)
+
+    def _stub_skill(self):
+        skill = mock.MagicMock()
+        skill.fill_and_check.return_value = []
+        skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+        skill.build_enriched_prompt.return_value = "## User Request:\nx"
+        return skill
+
+    def test_a_delivered_config_clears_it(self):
+        orchestrator = self._orchestrator()
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            return_value={"type": "plan",
+                          "content": "source: Jdbc\nsink: Console\nmode: 
BATCH"},
+        ), mock.patch("seatunnel_cli.skills.SkillExecutor",
+                      return_value=self._stub_skill()), \
+             mock.patch("seatunnel_cli.skills.SkillRouter.match", 
return_value=[]), \
+             mock.patch.object(
+                 orchestrator, "_run_config_generator",
+                 return_value={"config": "env {}\nsource {}\nsink {}",
+                               "explanation": ""}), \
+             mock.patch.object(orchestrator, "_run_validator", 
return_value="PASS"), \
+             mock.patch("seatunnel_cli.agents.dry_run_config",
+                        return_value={"valid": True, "phase1_local": "VALID",
+                                      "phase2_check": "PASS", "phase3_api": 
None,
+                                      "summary": ""}):
+            result = orchestrator.process_user_input("Batch is fine.")
+
+        self.assertEqual(result["type"], "config")
+        self._assert_cleared(orchestrator)
+
+    def test_a_best_effort_config_after_three_rounds_clears_it(self):
+        orchestrator = self._orchestrator()
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            return_value={"type": "plan",
+                          "content": "source: Jdbc\nsink: Console\nmode: 
BATCH"},
+        ), mock.patch("seatunnel_cli.skills.SkillExecutor",
+                      return_value=self._stub_skill()), \
+             mock.patch("seatunnel_cli.skills.SkillRouter.match", 
return_value=[]), \
+             mock.patch.object(
+                 orchestrator, "_run_config_generator",
+                 return_value={"config": "env {}\nsource {}\nsink {}",
+                               "explanation": ""}), \
+             mock.patch.object(orchestrator, "_run_validator",
+                               return_value="FAIL: still broken"), \
+             mock.patch.object(orchestrator, "_run_fix",
+                               return_value={"config": "env {}\nsource 
{}\nsink {}",
+                                             "explanation": ""}):
+            result = orchestrator.process_user_input("Batch is fine.")
+
+        self.assertEqual(result["type"], "config")
+        self._assert_cleared(orchestrator)
+
+    def test_a_chat_reply_clears_it(self):
+        orchestrator = self._orchestrator()
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            return_value={"type": "chat", "content": "SeaTunnel is an ETL 
tool."},
+        ):
+            result = orchestrator.process_user_input("What is SeaTunnel?")
+
+        self.assertEqual(result["type"], "chat")
+        self._assert_cleared(orchestrator)
+
+    def test_resuming_a_session_clears_it(self):
+        orchestrator = self._orchestrator()
+
+        orchestrator.load_history([
+            {"role": "user", "content": [{"text": "an unrelated earlier 
session"}]},
+        ])
+
+        self._assert_cleared(orchestrator)
+
+
+if __name__ == "__main__":
+    unittest.main()

Reply via email to