This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/dev/pr-13002-79d60f0708d67e386bf77f7d11e2243bcde2938d
in repository https://gitbox.apache.org/repos/asf/seatunnel.git

commit 9b8252eaf0c0eed5d80d5fd10a3b15eafeab9f58
Author: SEZ <[email protected]>
AuthorDate: Fri Nov 6 13:53:03 2026 +0000

    [Fix][seatunnel-cli] Catch a config that answers the wrong request (#13002)
    
    Co-authored-by: SEZ9 <[email protected]>
---
 seatunnel-cli/seatunnel_cli/agents.py              | 137 +++++++-
 .../tests/test_orchestrator_plan_conformance.py    | 346 +++++++++++++++++++++
 2 files changed, 478 insertions(+), 5 deletions(-)

diff --git a/seatunnel-cli/seatunnel_cli/agents.py 
b/seatunnel-cli/seatunnel_cli/agents.py
index 94219efbc2..edc190cb75 100644
--- a/seatunnel-cli/seatunnel_cli/agents.py
+++ b/seatunnel-cli/seatunnel_cli/agents.py
@@ -275,6 +275,85 @@ def _check_conditional_mismatches(
             )
 
 
+def _registered_connector_names() -> dict[str, set[str]] | None:
+    """Lower-cased source and sink plugin names the engine registers.
+
+    Returns None when the registry is unavailable -- no metadata exported, or
+    the lookup failed -- in which case the caller compares plan names as given.
+    That is the pre-existing behaviour and the conservative choice: without a
+    registry we cannot tell an alias from a real mismatch.
+    """
+    try:
+        from .connectors import list_connector_names
+
+        names = list_connector_names()
+    except Exception:
+        return None
+    sources = {n.lower() for n in names.get("sources", []) if n}
+    sinks = {n.lower() for n in names.get("sinks", []) if n}
+    if not sources and not sinks:
+        return None
+    return {"source": sources, "sink": sinks}
+
+
+def _plan_conformance_errors(structured_plan, config_str: str) -> list[str]:
+    """Report connectors the config uses that the plan did not ask for.
+
+    The planner resolves the user's words into connector names; the generator
+    then has to use those names. When it does not, every later gate still
+    passes -- the config is valid, it just moves the wrong data or moves it the
+    wrong way -- so nothing downstream catches it.
+
+    Only a connector named by the plan and absent from the config is reported.
+    Extra connectors are not: a plan for one pipeline can legitimately be
+    realised with an added Assert sink or a second branch, and flagging those
+    would make the check fight normal practice.
+
+    The plan is model output, not ground truth, so a planned name that is not a
+    registered plugin is ignored rather than enforced. Otherwise an alias the
+    planner invented -- ``S3`` where the plugin is ``S3File`` -- would condemn 
a
+    correct config and spend every repair round moving it away from the right
+    connector. Comparison is case-insensitive.
+    """
+    pipelines = getattr(structured_plan, "pipelines", None) or []
+    if not pipelines:
+        return []
+
+    used: dict[str, set[str]] = {"source": set(), "sink": set()}
+    for section, name, _block in _extract_connector_blocks_raw(config_str):
+        if section in used and name:
+            used[section].add(name.strip().lower())
+    if not used["source"] and not used["sink"]:
+        # Nothing recognisable to compare against; the HOCON checks own that.
+        return []
+
+    known = _registered_connector_names()
+
+    errors: list[str] = []
+    for section, attribute in (("source", "source_connector"),
+                               ("sink", "sink_connector")):
+        planned = {
+            str(getattr(p, attribute, "") or "").strip().lower()
+            for p in pipelines
+        }
+        planned.discard("")
+        if known is not None:
+            # Keep only names the engine actually registers for this section,
+            # so an invented alias cannot condemn a correct config.
+            planned &= known.get(section, set())
+        if not planned or not used[section]:
+            continue
+        unmet = planned - used[section]
+        if unmet:
+            errors.append(
+                f"{section}: the plan resolved this request to "
+                f"{', '.join(sorted(unmet))} but the config uses "
+                f"{', '.join(sorted(used[section]))}. Use the connector the "
+                f"request calls for."
+            )
+    return errors
+
+
 def _extract_connector_blocks_raw(
     config_str: str,
 ) -> list[tuple[str, str, str]]:
@@ -1518,6 +1597,26 @@ class Orchestrator:
             self.on_status("validating", f"Validating config (round {round_num 
+ 1})...")
             validation = self._run_validator(config)
 
+            # A config can be entirely valid and still answer the wrong
+            # question. Picking Jdbc where the plan said MySQL-CDC produces
+            # HOCON that parses, loads and runs, so the validator passes it and
+            # the loop delivers it -- the generator's choice is never 
revisited.
+            # The plan already names the connectors, so this is a comparison
+            # rather than a judgement, and it is fed through the same repair
+            # loop as any other validation failure.
+            drift = _plan_conformance_errors(structured_plan, config)
+            if drift:
+                # The validator's contract is "PASS", "PASS_WITH_NOTES: ..." or
+                # "FAIL: <issue list>", so its first line carries a real issue.
+                # Keep its text whole and prepend the drift: dropping a line
+                # here would remove a genuine defect from the repair prompt in
+                # exactly the case where two things are wrong at once.
+                drift_text = "FAIL: plan drift\n" + "\n".join(drift)
+                validation = (
+                    drift_text if validation.startswith("PASS")
+                    else drift_text + "\n" + validation
+                )
+
             if validation.startswith("PASS"):
                 # Phase 4: Dry-run validation (engine-level)
                 self.on_status("validating", "Running dry-run check...")
@@ -1548,7 +1647,11 @@ class Orchestrator:
 
             # Validation failed — try to fix
             self.on_status("fixing", f"Fixing issues (round {round_num + 
1})...")
-            fix_result = self._run_fix(config, validation)
+            fix_result = self._run_fix(
+                config, validation,
+                request=self._original_request,
+                plan_text=getattr(structured_plan, "raw_plan_text", "") or "",
+            )
             if fix_result.get("config"):
                 config = fix_result["config"]
                 if fix_result.get("explanation"):
@@ -1766,8 +1869,15 @@ Check for semantic correctness, required parameters, and 
best practices."""
         result = self.client.quick_chat(prompt, system=VALIDATOR_SYSTEM)
         return result.strip()
 
-    def _run_fix(self, config: str, validation_errors: str) -> dict:
-        """Attempt to fix config based on validation errors."""
+    def _run_fix(self, config: str, validation_errors: str,
+                 request: str | None = None, plan_text: str = "") -> dict:
+        """Attempt to fix config based on validation errors.
+
+        The request and the plan are included because without them a repair
+        cannot correct a wrong connector: the prompt held only the config and
+        the errors, so the model had nothing to compare the config against and
+        was told to keep everything not explicitly flagged.
+        """
         from .cli import _replace_creds_with_placeholders, 
_restore_creds_from_placeholders
         system = self._build_config_system()
 
@@ -1775,8 +1885,23 @@ Check for semantic correctness, required parameters, and 
best practices."""
         # After LLM returns the fixed config, restore the original values.
         safe_config, cred_map = _replace_creds_with_placeholders(config)
 
-        prompt = f"""The following SeaTunnel config has validation issues. Fix 
them.
+        # The config is credential-swapped above for a reason, and the request
+        # is free text the user typed -- it can hold a password just as easily.
+        # `redact_credentials` is the same scrubber the planner path and the
+        # session writer use, so this keeps one rule for what leaves the host.
+        from .memory import redact_credentials
+
+        context = ""
+        if request:
+            context += (
+                "\n## The request this config must satisfy:\n"
+                f"{redact_credentials(request)}\n"
+            )
+        if plan_text:
+            context += f"\n## The resolved 
plan:\n{redact_credentials(plan_text)}\n"
 
+        prompt = f"""The following SeaTunnel config has validation issues. Fix 
them.
+{context}
 ## Current Config:
 ```hocon
 {safe_config}
@@ -1785,7 +1910,9 @@ Check for semantic correctness, required parameters, and 
best practices."""
 ## Validation Errors:
 {validation_errors}
 
-Fix ALL the issues and return the corrected config. Keep all existing correct 
parts unchanged."""
+Fix ALL the issues and return the corrected config. Keep the parts that are
+both correct and consistent with the request unchanged. If the config uses a
+connector the request does not call for, replace it rather than preserving 
it."""
 
         messages = [{"role": "user", "content": [{"text": prompt}]}]
 
diff --git a/seatunnel-cli/tests/test_orchestrator_plan_conformance.py 
b/seatunnel-cli/tests/test_orchestrator_plan_conformance.py
new file mode 100644
index 0000000000..a9fe596236
--- /dev/null
+++ b/seatunnel-cli/tests/test_orchestrator_plan_conformance.py
@@ -0,0 +1,346 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A valid config that answers the wrong question must not be delivered.
+
+Choosing Jdbc where the plan resolved to MySQL-CDC produces HOCON that parses,
+loads and runs, so every existing gate passes it. The generator's connector
+choice was therefore never revisited, and the repair prompt held only the
+config and the validation errors - nothing to compare the config against.
+"""
+
+import unittest
+from unittest import mock
+
+from seatunnel_cli.agents import Orchestrator, _plan_conformance_errors
+from seatunnel_cli.skills import PipelineSlot, StructuredPlan
+
+
+CDC_TO_CONSOLE = StructuredPlan(
+    pipelines=[PipelineSlot("pipeline_1", "MySQL-CDC", "Console", 
mode="STREAMING")],
+    raw_plan_text="source: MySQL-CDC\nsink: Console\nmode: STREAMING",
+)
+
+WRONG_SOURCE = """env {
+  job.mode = "STREAMING"
+}
+source {
+  Jdbc {
+    url = "jdbc:mysql://localhost:3306/shop"
+    query = "select * from users"
+  }
+}
+sink {
+  Console {}
+}
+"""
+
+RIGHT_SOURCE = """env {
+  job.mode = "STREAMING"
+}
+source {
+  MySQL-CDC {
+    base-url = "jdbc:mysql://localhost:3306/shop"
+    table-names = ["shop.users"]
+  }
+}
+sink {
+  Console {}
+}
+"""
+
+
+# The conformance check intersects planned names with what the engine
+# registers, so a test that does not pin the registry depends on whichever
+# metadata happens to be exported on the machine running it. On a host whose
+# metadata omits MySQL-CDC the planned name would be filtered out, the expected
+# drift would become an empty list, and the test would fail for a reason that
+# has nothing to do with the code. Pinning it here keeps the registry a fixture
+# rather than an input; the unavailable-registry case patches it separately.
+FIXED_CATALOG = {
+    "sources": ["MySQL-CDC", "Postgres-CDC", "Jdbc", "Kafka", "S3File", 
"FakeSource"],
+    "sinks": ["Console", "Jdbc", "Clickhouse", "Assert", "LocalFile", 
"S3File"],
+    "transforms": ["Sql", "Filter"],
+}
+
+
+class PinnedRegistryMixin:
+    """Pin the connector registry for every test in the class."""
+
+    def setUp(self):
+        super().setUp()
+        patcher = mock.patch(
+            "seatunnel_cli.connectors.list_connector_names",
+            return_value=FIXED_CATALOG,
+        )
+        patcher.start()
+        self.addCleanup(patcher.stop)
+
+
+class ConformanceCheckTest(PinnedRegistryMixin, unittest.TestCase):
+    def test_wrong_source_connector_is_reported(self):
+        errors = _plan_conformance_errors(CDC_TO_CONSOLE, WRONG_SOURCE)
+        self.assertEqual(len(errors), 1)
+        self.assertIn("mysql-cdc", errors[0])
+        self.assertIn("jdbc", errors[0])
+
+    def test_matching_connectors_report_nothing(self):
+        self.assertEqual(_plan_conformance_errors(CDC_TO_CONSOLE, 
RIGHT_SOURCE), [])
+
+    def test_case_differences_are_not_drift(self):
+        plan = StructuredPlan(
+            pipelines=[PipelineSlot("p", "jdbc", "console")])
+        config = "source {\n  Jdbc {}\n}\nsink {\n  Console {}\n}\n"
+        self.assertEqual(_plan_conformance_errors(plan, config), [])
+
+    def test_an_extra_sink_is_allowed(self):
+        """A plan for one pipeline may legitimately gain an Assert sink."""
+        plan = StructuredPlan(
+            pipelines=[PipelineSlot("p", "Jdbc", "Console")])
+        config = ("source {\n  Jdbc {}\n}\n"
+                  "sink {\n  Console {}\n  Assert {}\n}\n")
+        self.assertEqual(_plan_conformance_errors(plan, config), [])
+
+    def test_no_plan_means_no_opinion(self):
+        self.assertEqual(
+            _plan_conformance_errors(StructuredPlan(pipelines=[]), 
WRONG_SOURCE), [])
+
+    def test_a_planned_name_the_engine_does_not_register_is_ignored(self):
+        """The plan is model output; an invented alias must not condemn a 
config.
+
+        A planner that writes ``S3`` where the plugin is ``S3File`` would
+        otherwise spend every repair round moving a correct config away from
+        the right connector.
+        """
+        plan = StructuredPlan(pipelines=[PipelineSlot("p", "S3", "Console")])
+        config = "source {\n  S3File {}\n}\nsink {\n  Console {}\n}\n"
+
+        with mock.patch("seatunnel_cli.connectors.list_connector_names",
+                        return_value={"sources": ["S3File", "Jdbc"],
+                                      "sinks": ["Console"], "transforms": []}):
+            self.assertEqual(_plan_conformance_errors(plan, config), [])
+
+    def test_a_registered_name_is_still_enforced(self):
+        plan = StructuredPlan(pipelines=[PipelineSlot("p", "MySQL-CDC", 
"Console")])
+
+        with mock.patch("seatunnel_cli.connectors.list_connector_names",
+                        return_value={"sources": ["MySQL-CDC", "Jdbc"],
+                                      "sinks": ["Console"], "transforms": []}):
+            errors = _plan_conformance_errors(plan, WRONG_SOURCE)
+
+        self.assertEqual(len(errors), 1)
+        self.assertIn("mysql-cdc", errors[0])
+
+    def test_an_unavailable_registry_falls_back_to_comparing_as_given(self):
+        with mock.patch("seatunnel_cli.connectors.list_connector_names",
+                        side_effect=RuntimeError("no metadata exported")):
+            errors = _plan_conformance_errors(CDC_TO_CONSOLE, WRONG_SOURCE)
+
+        self.assertEqual(len(errors), 1)
+
+    def test_unrecognisable_config_is_left_to_the_hocon_checks(self):
+        self.assertEqual(_plan_conformance_errors(CDC_TO_CONSOLE, "not 
hocon"), [])
+
+
+class DriftReachesTheRepairLoopTest(PinnedRegistryMixin, unittest.TestCase):
+    def _run(self, generated_configs):
+        orchestrator = Orchestrator(client=mock.MagicMock())
+        skill = mock.MagicMock()
+        skill.fill_and_check.return_value = []
+        skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+        skill.build_enriched_prompt.return_value = "## User Request:\nx"
+
+        fix_prompts: list[tuple[str, str]] = []
+
+        def run_fix(config, validation_errors, request=None, plan_text=""):
+            fix_prompts.append((validation_errors, f"{request}|{plan_text}"))
+            return {"config": generated_configs.pop(0), "explanation": ""}
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            return_value={"type": "plan", "content": 
CDC_TO_CONSOLE.raw_plan_text},
+        ), mock.patch("seatunnel_cli.skills.SkillExecutor", 
return_value=skill), \
+             mock.patch("seatunnel_cli.skills.SkillRouter.match", 
return_value=[]), \
+             mock.patch("seatunnel_cli.skills.parse_structured_plan",
+                        return_value=CDC_TO_CONSOLE), \
+             mock.patch.object(
+                 orchestrator, "_run_config_generator",
+                 return_value={"config": generated_configs.pop(0),
+                               "explanation": ""}), \
+             mock.patch.object(orchestrator, "_run_validator", 
return_value="PASS"), \
+             mock.patch.object(orchestrator, "_run_fix", side_effect=run_fix), 
\
+             mock.patch("seatunnel_cli.agents.dry_run_config",
+                        return_value={"valid": True, "phase1_local": "VALID",
+                                      "phase2_check": "PASS", "phase3_api": 
None,
+                                      "summary": ""}):
+            result = orchestrator.process_user_input(
+                "Watch the users table in MySQL at localhost:3306 and show "
+                "changes as they happen.")
+        return result, fix_prompts
+
+    def test_a_valid_but_wrong_config_is_sent_for_repair(self):
+        # The validator says PASS, so before this the loop delivered it.
+        result, fix_prompts = self._run([WRONG_SOURCE, RIGHT_SOURCE])
+
+        self.assertEqual(len(fix_prompts), 1, "drift should trigger one 
repair")
+        errors, context = fix_prompts[0]
+        self.assertIn("mysql-cdc", errors)
+        self.assertEqual(result["type"], "config")
+        self.assertIn("MySQL-CDC", result["config"])
+
+    def test_the_repair_is_told_the_request_and_the_plan(self):
+        _result, fix_prompts = self._run([WRONG_SOURCE, RIGHT_SOURCE])
+
+        _errors, context = fix_prompts[0]
+        self.assertIn("show changes as they happen", context)
+        self.assertIn("MySQL-CDC", context)
+
+    def test_a_conforming_config_is_delivered_without_repair(self):
+        result, fix_prompts = self._run([RIGHT_SOURCE])
+
+        self.assertEqual(fix_prompts, [])
+        self.assertEqual(result["type"], "config")
+
+
+class DriftMergedWithValidatorOutputTest(PinnedRegistryMixin, 
unittest.TestCase):
+    """Drift must not displace the validator's own findings.
+
+    The validator's contract is "FAIL: <issue list>", so its first line carries
+    a real issue. An earlier version sliced that line off when both drift and a
+    validation failure occurred, removing a genuine defect from the repair
+    prompt in exactly the case where two things were wrong at once.
+    """
+
+    def _validation_seen_by_repair(self, validator_output):
+        orchestrator = Orchestrator(client=mock.MagicMock())
+        skill = mock.MagicMock()
+        skill.fill_and_check.return_value = []
+        skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+        skill.build_enriched_prompt.return_value = "## User Request:\nx"
+        seen: list[str] = []
+
+        def run_fix(config, validation_errors, request=None, plan_text=""):
+            seen.append(validation_errors)
+            return {"config": RIGHT_SOURCE, "explanation": ""}
+
+        with mock.patch.object(
+            orchestrator, "_run_planner",
+            return_value={"type": "plan", "content": 
CDC_TO_CONSOLE.raw_plan_text},
+        ), mock.patch("seatunnel_cli.skills.SkillExecutor", 
return_value=skill), \
+             mock.patch("seatunnel_cli.skills.SkillRouter.match", 
return_value=[]), \
+             mock.patch("seatunnel_cli.skills.parse_structured_plan",
+                        return_value=CDC_TO_CONSOLE), \
+             mock.patch.object(
+                 orchestrator, "_run_config_generator",
+                 return_value={"config": WRONG_SOURCE, "explanation": ""}), \
+             mock.patch.object(orchestrator, "_run_validator",
+                               return_value=validator_output), \
+             mock.patch.object(orchestrator, "_run_fix", side_effect=run_fix), 
\
+             mock.patch("seatunnel_cli.agents.dry_run_config",
+                        return_value={"valid": True, "phase1_local": "VALID",
+                                      "phase2_check": "PASS", "phase3_api": 
None,
+                                      "summary": ""}):
+            orchestrator.process_user_input("Watch the users table for 
changes.")
+        return seen[0]
+
+    def test_every_validator_issue_survives_the_merge(self):
+        merged = self._validation_seen_by_repair(
+            "FAIL: missing env block\nplugin_input is not wired\nunknown 
option foo")
+
+        self.assertIn("missing env block", merged)
+        self.assertIn("plugin_input is not wired", merged)
+        self.assertIn("unknown option foo", merged)
+        self.assertIn("mysql-cdc", merged)
+
+    def test_drift_alone_when_the_validator_passed(self):
+        merged = self._validation_seen_by_repair("PASS")
+
+        self.assertIn("mysql-cdc", merged)
+        self.assertNotIn("PASS", merged)
+
+
+class RepairPromptRedactionTest(unittest.TestCase):
+    """The request is free text the user typed and can hold a password.
+
+    `_run_fix` already swaps credentials out of the config before calling the
+    model; adding the request and the plan unredacted would route around that.
+    These assert that the existing scrubber is applied to the new context, so
+    the request path has the same protection as the config path.
+
+    How much `redact_credentials` itself catches is a separate matter: it keys
+    on `key = value`, `key: value` and "password is X", so a bare "password X"
+    in prose is not matched. Widening it would change a helper that session
+    persistence and memory also use, and would start redacting phrases like
+    "password rotation policy". That belongs in its own change, not here.
+    """
+
+    def test_a_password_in_the_request_does_not_reach_the_model(self):
+        orchestrator = Orchestrator(client=mock.MagicMock())
+        captured = {}
+
+        def chat(messages, system="", **kwargs):
+            captured["text"] = "".join(
+                block.get("text", "")
+                for message in messages for block in message.get("content", 
[]))
+            return {"output": {"message": {"content": [
+                {"text": "```hocon\nenv {}\nsource {}\nsink {}\n```"}]}},
+                "stopReason": "end_turn"}
+
+        orchestrator.client.chat.side_effect = chat
+
+        orchestrator._run_fix(
+            "source {\n  Jdbc {\n    url = \"jdbc:mysql://h/db\"\n  }\n}",
+            "FAIL: missing driver",
+            request=("Copy users from MySQL at localhost:3306. The password "
+                     "is Hunter2Secret! and the user is root."),
+            plan_text="source: Jdbc\nsink: Console",
+        )
+
+        self.assertNotIn("Hunter2Secret!", captured["text"])
+        # The rest of the request must survive, or the repair loses the context
+        # that lets it tell a wrong connector from a right one.
+        self.assertIn("Copy users from MySQL", captured["text"])
+        self.assertIn("localhost:3306", captured["text"])
+        self.assertIn("source: Jdbc", captured["text"])
+
+    def test_a_key_value_secret_in_the_plan_is_redacted_too(self):
+        orchestrator = Orchestrator(client=mock.MagicMock())
+        captured = {}
+
+        def chat(messages, system="", **kwargs):
+            captured["text"] = "".join(
+                block.get("text", "")
+                for message in messages for block in message.get("content", 
[]))
+            return {"output": {"message": {"content": [
+                {"text": "```hocon\nenv {}\nsource {}\nsink {}\n```"}]}},
+                "stopReason": "end_turn"}
+
+        orchestrator.client.chat.side_effect = chat
+
+        orchestrator._run_fix(
+            "source {\n  Jdbc {}\n}",
+            "FAIL: missing url",
+            request="Load the table.",
+            plan_text="source: Jdbc\nsecret_key=AKIAIOSFODNN7EXAMPLE",
+        )
+
+        self.assertNotIn("AKIAIOSFODNN7EXAMPLE", captured["text"])
+        self.assertIn("source: Jdbc", captured["text"])
+
+
+if __name__ == "__main__":
+    unittest.main()

Reply via email to