This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 9b8252eaf0 [Fix][seatunnel-cli] Catch a config that answers the wrong
request (#13002)
9b8252eaf0 is described below
commit 9b8252eaf0c0eed5d80d5fd10a3b15eafeab9f58
Author: SEZ <[email protected]>
AuthorDate: Fri Nov 6 13:53:03 2026 +0000
[Fix][seatunnel-cli] Catch a config that answers the wrong request (#13002)
Co-authored-by: SEZ9 <[email protected]>
---
seatunnel-cli/seatunnel_cli/agents.py | 137 +++++++-
.../tests/test_orchestrator_plan_conformance.py | 346 +++++++++++++++++++++
2 files changed, 478 insertions(+), 5 deletions(-)
diff --git a/seatunnel-cli/seatunnel_cli/agents.py
b/seatunnel-cli/seatunnel_cli/agents.py
index 94219efbc2..edc190cb75 100644
--- a/seatunnel-cli/seatunnel_cli/agents.py
+++ b/seatunnel-cli/seatunnel_cli/agents.py
@@ -275,6 +275,85 @@ def _check_conditional_mismatches(
)
+def _registered_connector_names() -> dict[str, set[str]] | None:
+ """Lower-cased source and sink plugin names the engine registers.
+
+ Returns None when the registry is unavailable -- no metadata exported, or
+ the lookup failed -- in which case the caller compares plan names as given.
+ That is the pre-existing behaviour and the conservative choice: without a
+ registry we cannot tell an alias from a real mismatch.
+ """
+ try:
+ from .connectors import list_connector_names
+
+ names = list_connector_names()
+ except Exception:
+ return None
+ sources = {n.lower() for n in names.get("sources", []) if n}
+ sinks = {n.lower() for n in names.get("sinks", []) if n}
+ if not sources and not sinks:
+ return None
+ return {"source": sources, "sink": sinks}
+
+
+def _plan_conformance_errors(structured_plan, config_str: str) -> list[str]:
+ """Report connectors the config uses that the plan did not ask for.
+
+ The planner resolves the user's words into connector names; the generator
+ then has to use those names. When it does not, every later gate still
+ passes -- the config is valid, it just moves the wrong data or moves it the
+ wrong way -- so nothing downstream catches it.
+
+ Only a connector named by the plan and absent from the config is reported.
+ Extra connectors are not: a plan for one pipeline can legitimately be
+ realised with an added Assert sink or a second branch, and flagging those
+ would make the check fight normal practice.
+
+ The plan is model output, not ground truth, so a planned name that is not a
+ registered plugin is ignored rather than enforced. Otherwise an alias the
+ planner invented -- ``S3`` where the plugin is ``S3File`` -- would condemn
a
+ correct config and spend every repair round moving it away from the right
+ connector. Comparison is case-insensitive.
+ """
+ pipelines = getattr(structured_plan, "pipelines", None) or []
+ if not pipelines:
+ return []
+
+ used: dict[str, set[str]] = {"source": set(), "sink": set()}
+ for section, name, _block in _extract_connector_blocks_raw(config_str):
+ if section in used and name:
+ used[section].add(name.strip().lower())
+ if not used["source"] and not used["sink"]:
+ # Nothing recognisable to compare against; the HOCON checks own that.
+ return []
+
+ known = _registered_connector_names()
+
+ errors: list[str] = []
+ for section, attribute in (("source", "source_connector"),
+ ("sink", "sink_connector")):
+ planned = {
+ str(getattr(p, attribute, "") or "").strip().lower()
+ for p in pipelines
+ }
+ planned.discard("")
+ if known is not None:
+ # Keep only names the engine actually registers for this section,
+ # so an invented alias cannot condemn a correct config.
+ planned &= known.get(section, set())
+ if not planned or not used[section]:
+ continue
+ unmet = planned - used[section]
+ if unmet:
+ errors.append(
+ f"{section}: the plan resolved this request to "
+ f"{', '.join(sorted(unmet))} but the config uses "
+ f"{', '.join(sorted(used[section]))}. Use the connector the "
+ f"request calls for."
+ )
+ return errors
+
+
def _extract_connector_blocks_raw(
config_str: str,
) -> list[tuple[str, str, str]]:
@@ -1518,6 +1597,26 @@ class Orchestrator:
self.on_status("validating", f"Validating config (round {round_num
+ 1})...")
validation = self._run_validator(config)
+ # A config can be entirely valid and still answer the wrong
+ # question. Picking Jdbc where the plan said MySQL-CDC produces
+ # HOCON that parses, loads and runs, so the validator passes it and
+ # the loop delivers it -- the generator's choice is never
revisited.
+ # The plan already names the connectors, so this is a comparison
+ # rather than a judgement, and it is fed through the same repair
+ # loop as any other validation failure.
+ drift = _plan_conformance_errors(structured_plan, config)
+ if drift:
+ # The validator's contract is "PASS", "PASS_WITH_NOTES: ..." or
+ # "FAIL: <issue list>", so its first line carries a real issue.
+ # Keep its text whole and prepend the drift: dropping a line
+ # here would remove a genuine defect from the repair prompt in
+ # exactly the case where two things are wrong at once.
+ drift_text = "FAIL: plan drift\n" + "\n".join(drift)
+ validation = (
+ drift_text if validation.startswith("PASS")
+ else drift_text + "\n" + validation
+ )
+
if validation.startswith("PASS"):
# Phase 4: Dry-run validation (engine-level)
self.on_status("validating", "Running dry-run check...")
@@ -1548,7 +1647,11 @@ class Orchestrator:
# Validation failed — try to fix
self.on_status("fixing", f"Fixing issues (round {round_num +
1})...")
- fix_result = self._run_fix(config, validation)
+ fix_result = self._run_fix(
+ config, validation,
+ request=self._original_request,
+ plan_text=getattr(structured_plan, "raw_plan_text", "") or "",
+ )
if fix_result.get("config"):
config = fix_result["config"]
if fix_result.get("explanation"):
@@ -1766,8 +1869,15 @@ Check for semantic correctness, required parameters, and
best practices."""
result = self.client.quick_chat(prompt, system=VALIDATOR_SYSTEM)
return result.strip()
- def _run_fix(self, config: str, validation_errors: str) -> dict:
- """Attempt to fix config based on validation errors."""
+ def _run_fix(self, config: str, validation_errors: str,
+ request: str | None = None, plan_text: str = "") -> dict:
+ """Attempt to fix config based on validation errors.
+
+ The request and the plan are included because without them a repair
+ cannot correct a wrong connector: the prompt held only the config and
+ the errors, so the model had nothing to compare the config against and
+ was told to keep everything not explicitly flagged.
+ """
from .cli import _replace_creds_with_placeholders,
_restore_creds_from_placeholders
system = self._build_config_system()
@@ -1775,8 +1885,23 @@ Check for semantic correctness, required parameters, and
best practices."""
# After LLM returns the fixed config, restore the original values.
safe_config, cred_map = _replace_creds_with_placeholders(config)
- prompt = f"""The following SeaTunnel config has validation issues. Fix
them.
+ # The config is credential-swapped above for a reason, and the request
+ # is free text the user typed -- it can hold a password just as easily.
+ # `redact_credentials` is the same scrubber the planner path and the
+ # session writer use, so this keeps one rule for what leaves the host.
+ from .memory import redact_credentials
+
+ context = ""
+ if request:
+ context += (
+ "\n## The request this config must satisfy:\n"
+ f"{redact_credentials(request)}\n"
+ )
+ if plan_text:
+ context += f"\n## The resolved
plan:\n{redact_credentials(plan_text)}\n"
+ prompt = f"""The following SeaTunnel config has validation issues. Fix
them.
+{context}
## Current Config:
```hocon
{safe_config}
@@ -1785,7 +1910,9 @@ Check for semantic correctness, required parameters, and
best practices."""
## Validation Errors:
{validation_errors}
-Fix ALL the issues and return the corrected config. Keep all existing correct
parts unchanged."""
+Fix ALL the issues and return the corrected config. Keep the parts that are
+both correct and consistent with the request unchanged. If the config uses a
+connector the request does not call for, replace it rather than preserving
it."""
messages = [{"role": "user", "content": [{"text": prompt}]}]
diff --git a/seatunnel-cli/tests/test_orchestrator_plan_conformance.py
b/seatunnel-cli/tests/test_orchestrator_plan_conformance.py
new file mode 100644
index 0000000000..a9fe596236
--- /dev/null
+++ b/seatunnel-cli/tests/test_orchestrator_plan_conformance.py
@@ -0,0 +1,346 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""A valid config that answers the wrong question must not be delivered.
+
+Choosing Jdbc where the plan resolved to MySQL-CDC produces HOCON that parses,
+loads and runs, so every existing gate passes it. The generator's connector
+choice was therefore never revisited, and the repair prompt held only the
+config and the validation errors - nothing to compare the config against.
+"""
+
+import unittest
+from unittest import mock
+
+from seatunnel_cli.agents import Orchestrator, _plan_conformance_errors
+from seatunnel_cli.skills import PipelineSlot, StructuredPlan
+
+
+CDC_TO_CONSOLE = StructuredPlan(
+ pipelines=[PipelineSlot("pipeline_1", "MySQL-CDC", "Console",
mode="STREAMING")],
+ raw_plan_text="source: MySQL-CDC\nsink: Console\nmode: STREAMING",
+)
+
+WRONG_SOURCE = """env {
+ job.mode = "STREAMING"
+}
+source {
+ Jdbc {
+ url = "jdbc:mysql://localhost:3306/shop"
+ query = "select * from users"
+ }
+}
+sink {
+ Console {}
+}
+"""
+
+RIGHT_SOURCE = """env {
+ job.mode = "STREAMING"
+}
+source {
+ MySQL-CDC {
+ base-url = "jdbc:mysql://localhost:3306/shop"
+ table-names = ["shop.users"]
+ }
+}
+sink {
+ Console {}
+}
+"""
+
+
+# The conformance check intersects planned names with what the engine
+# registers, so a test that does not pin the registry depends on whichever
+# metadata happens to be exported on the machine running it. On a host whose
+# metadata omits MySQL-CDC the planned name would be filtered out, the expected
+# drift would become an empty list, and the test would fail for a reason that
+# has nothing to do with the code. Pinning it here keeps the registry a fixture
+# rather than an input; the unavailable-registry case patches it separately.
+FIXED_CATALOG = {
+ "sources": ["MySQL-CDC", "Postgres-CDC", "Jdbc", "Kafka", "S3File",
"FakeSource"],
+ "sinks": ["Console", "Jdbc", "Clickhouse", "Assert", "LocalFile",
"S3File"],
+ "transforms": ["Sql", "Filter"],
+}
+
+
+class PinnedRegistryMixin:
+ """Pin the connector registry for every test in the class."""
+
+ def setUp(self):
+ super().setUp()
+ patcher = mock.patch(
+ "seatunnel_cli.connectors.list_connector_names",
+ return_value=FIXED_CATALOG,
+ )
+ patcher.start()
+ self.addCleanup(patcher.stop)
+
+
+class ConformanceCheckTest(PinnedRegistryMixin, unittest.TestCase):
+ def test_wrong_source_connector_is_reported(self):
+ errors = _plan_conformance_errors(CDC_TO_CONSOLE, WRONG_SOURCE)
+ self.assertEqual(len(errors), 1)
+ self.assertIn("mysql-cdc", errors[0])
+ self.assertIn("jdbc", errors[0])
+
+ def test_matching_connectors_report_nothing(self):
+ self.assertEqual(_plan_conformance_errors(CDC_TO_CONSOLE,
RIGHT_SOURCE), [])
+
+ def test_case_differences_are_not_drift(self):
+ plan = StructuredPlan(
+ pipelines=[PipelineSlot("p", "jdbc", "console")])
+ config = "source {\n Jdbc {}\n}\nsink {\n Console {}\n}\n"
+ self.assertEqual(_plan_conformance_errors(plan, config), [])
+
+ def test_an_extra_sink_is_allowed(self):
+ """A plan for one pipeline may legitimately gain an Assert sink."""
+ plan = StructuredPlan(
+ pipelines=[PipelineSlot("p", "Jdbc", "Console")])
+ config = ("source {\n Jdbc {}\n}\n"
+ "sink {\n Console {}\n Assert {}\n}\n")
+ self.assertEqual(_plan_conformance_errors(plan, config), [])
+
+ def test_no_plan_means_no_opinion(self):
+ self.assertEqual(
+ _plan_conformance_errors(StructuredPlan(pipelines=[]),
WRONG_SOURCE), [])
+
+ def test_a_planned_name_the_engine_does_not_register_is_ignored(self):
+ """The plan is model output; an invented alias must not condemn a
config.
+
+ A planner that writes ``S3`` where the plugin is ``S3File`` would
+ otherwise spend every repair round moving a correct config away from
+ the right connector.
+ """
+ plan = StructuredPlan(pipelines=[PipelineSlot("p", "S3", "Console")])
+ config = "source {\n S3File {}\n}\nsink {\n Console {}\n}\n"
+
+ with mock.patch("seatunnel_cli.connectors.list_connector_names",
+ return_value={"sources": ["S3File", "Jdbc"],
+ "sinks": ["Console"], "transforms": []}):
+ self.assertEqual(_plan_conformance_errors(plan, config), [])
+
+ def test_a_registered_name_is_still_enforced(self):
+ plan = StructuredPlan(pipelines=[PipelineSlot("p", "MySQL-CDC",
"Console")])
+
+ with mock.patch("seatunnel_cli.connectors.list_connector_names",
+ return_value={"sources": ["MySQL-CDC", "Jdbc"],
+ "sinks": ["Console"], "transforms": []}):
+ errors = _plan_conformance_errors(plan, WRONG_SOURCE)
+
+ self.assertEqual(len(errors), 1)
+ self.assertIn("mysql-cdc", errors[0])
+
+ def test_an_unavailable_registry_falls_back_to_comparing_as_given(self):
+ with mock.patch("seatunnel_cli.connectors.list_connector_names",
+ side_effect=RuntimeError("no metadata exported")):
+ errors = _plan_conformance_errors(CDC_TO_CONSOLE, WRONG_SOURCE)
+
+ self.assertEqual(len(errors), 1)
+
+ def test_unrecognisable_config_is_left_to_the_hocon_checks(self):
+ self.assertEqual(_plan_conformance_errors(CDC_TO_CONSOLE, "not
hocon"), [])
+
+
+class DriftReachesTheRepairLoopTest(PinnedRegistryMixin, unittest.TestCase):
+ def _run(self, generated_configs):
+ orchestrator = Orchestrator(client=mock.MagicMock())
+ skill = mock.MagicMock()
+ skill.fill_and_check.return_value = []
+ skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+ skill.build_enriched_prompt.return_value = "## User Request:\nx"
+
+ fix_prompts: list[tuple[str, str]] = []
+
+ def run_fix(config, validation_errors, request=None, plan_text=""):
+ fix_prompts.append((validation_errors, f"{request}|{plan_text}"))
+ return {"config": generated_configs.pop(0), "explanation": ""}
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ return_value={"type": "plan", "content":
CDC_TO_CONSOLE.raw_plan_text},
+ ), mock.patch("seatunnel_cli.skills.SkillExecutor",
return_value=skill), \
+ mock.patch("seatunnel_cli.skills.SkillRouter.match",
return_value=[]), \
+ mock.patch("seatunnel_cli.skills.parse_structured_plan",
+ return_value=CDC_TO_CONSOLE), \
+ mock.patch.object(
+ orchestrator, "_run_config_generator",
+ return_value={"config": generated_configs.pop(0),
+ "explanation": ""}), \
+ mock.patch.object(orchestrator, "_run_validator",
return_value="PASS"), \
+ mock.patch.object(orchestrator, "_run_fix", side_effect=run_fix),
\
+ mock.patch("seatunnel_cli.agents.dry_run_config",
+ return_value={"valid": True, "phase1_local": "VALID",
+ "phase2_check": "PASS", "phase3_api":
None,
+ "summary": ""}):
+ result = orchestrator.process_user_input(
+ "Watch the users table in MySQL at localhost:3306 and show "
+ "changes as they happen.")
+ return result, fix_prompts
+
+ def test_a_valid_but_wrong_config_is_sent_for_repair(self):
+ # The validator says PASS, so before this the loop delivered it.
+ result, fix_prompts = self._run([WRONG_SOURCE, RIGHT_SOURCE])
+
+ self.assertEqual(len(fix_prompts), 1, "drift should trigger one
repair")
+ errors, context = fix_prompts[0]
+ self.assertIn("mysql-cdc", errors)
+ self.assertEqual(result["type"], "config")
+ self.assertIn("MySQL-CDC", result["config"])
+
+ def test_the_repair_is_told_the_request_and_the_plan(self):
+ _result, fix_prompts = self._run([WRONG_SOURCE, RIGHT_SOURCE])
+
+ _errors, context = fix_prompts[0]
+ self.assertIn("show changes as they happen", context)
+ self.assertIn("MySQL-CDC", context)
+
+ def test_a_conforming_config_is_delivered_without_repair(self):
+ result, fix_prompts = self._run([RIGHT_SOURCE])
+
+ self.assertEqual(fix_prompts, [])
+ self.assertEqual(result["type"], "config")
+
+
+class DriftMergedWithValidatorOutputTest(PinnedRegistryMixin,
unittest.TestCase):
+ """Drift must not displace the validator's own findings.
+
+ The validator's contract is "FAIL: <issue list>", so its first line carries
+ a real issue. An earlier version sliced that line off when both drift and a
+ validation failure occurred, removing a genuine defect from the repair
+ prompt in exactly the case where two things were wrong at once.
+ """
+
+ def _validation_seen_by_repair(self, validator_output):
+ orchestrator = Orchestrator(client=mock.MagicMock())
+ skill = mock.MagicMock()
+ skill.fill_and_check.return_value = []
+ skill.fetch_all_metadata.return_value = "## Connector Option Rules\n"
+ skill.build_enriched_prompt.return_value = "## User Request:\nx"
+ seen: list[str] = []
+
+ def run_fix(config, validation_errors, request=None, plan_text=""):
+ seen.append(validation_errors)
+ return {"config": RIGHT_SOURCE, "explanation": ""}
+
+ with mock.patch.object(
+ orchestrator, "_run_planner",
+ return_value={"type": "plan", "content":
CDC_TO_CONSOLE.raw_plan_text},
+ ), mock.patch("seatunnel_cli.skills.SkillExecutor",
return_value=skill), \
+ mock.patch("seatunnel_cli.skills.SkillRouter.match",
return_value=[]), \
+ mock.patch("seatunnel_cli.skills.parse_structured_plan",
+ return_value=CDC_TO_CONSOLE), \
+ mock.patch.object(
+ orchestrator, "_run_config_generator",
+ return_value={"config": WRONG_SOURCE, "explanation": ""}), \
+ mock.patch.object(orchestrator, "_run_validator",
+ return_value=validator_output), \
+ mock.patch.object(orchestrator, "_run_fix", side_effect=run_fix),
\
+ mock.patch("seatunnel_cli.agents.dry_run_config",
+ return_value={"valid": True, "phase1_local": "VALID",
+ "phase2_check": "PASS", "phase3_api":
None,
+ "summary": ""}):
+ orchestrator.process_user_input("Watch the users table for
changes.")
+ return seen[0]
+
+ def test_every_validator_issue_survives_the_merge(self):
+ merged = self._validation_seen_by_repair(
+ "FAIL: missing env block\nplugin_input is not wired\nunknown
option foo")
+
+ self.assertIn("missing env block", merged)
+ self.assertIn("plugin_input is not wired", merged)
+ self.assertIn("unknown option foo", merged)
+ self.assertIn("mysql-cdc", merged)
+
+ def test_drift_alone_when_the_validator_passed(self):
+ merged = self._validation_seen_by_repair("PASS")
+
+ self.assertIn("mysql-cdc", merged)
+ self.assertNotIn("PASS", merged)
+
+
+class RepairPromptRedactionTest(unittest.TestCase):
+ """The request is free text the user typed and can hold a password.
+
+ `_run_fix` already swaps credentials out of the config before calling the
+ model; adding the request and the plan unredacted would route around that.
+ These assert that the existing scrubber is applied to the new context, so
+ the request path has the same protection as the config path.
+
+ How much `redact_credentials` itself catches is a separate matter: it keys
+ on `key = value`, `key: value` and "password is X", so a bare "password X"
+ in prose is not matched. Widening it would change a helper that session
+ persistence and memory also use, and would start redacting phrases like
+ "password rotation policy". That belongs in its own change, not here.
+ """
+
+ def test_a_password_in_the_request_does_not_reach_the_model(self):
+ orchestrator = Orchestrator(client=mock.MagicMock())
+ captured = {}
+
+ def chat(messages, system="", **kwargs):
+ captured["text"] = "".join(
+ block.get("text", "")
+ for message in messages for block in message.get("content",
[]))
+ return {"output": {"message": {"content": [
+ {"text": "```hocon\nenv {}\nsource {}\nsink {}\n```"}]}},
+ "stopReason": "end_turn"}
+
+ orchestrator.client.chat.side_effect = chat
+
+ orchestrator._run_fix(
+ "source {\n Jdbc {\n url = \"jdbc:mysql://h/db\"\n }\n}",
+ "FAIL: missing driver",
+ request=("Copy users from MySQL at localhost:3306. The password "
+ "is Hunter2Secret! and the user is root."),
+ plan_text="source: Jdbc\nsink: Console",
+ )
+
+ self.assertNotIn("Hunter2Secret!", captured["text"])
+ # The rest of the request must survive, or the repair loses the context
+ # that lets it tell a wrong connector from a right one.
+ self.assertIn("Copy users from MySQL", captured["text"])
+ self.assertIn("localhost:3306", captured["text"])
+ self.assertIn("source: Jdbc", captured["text"])
+
+ def test_a_key_value_secret_in_the_plan_is_redacted_too(self):
+ orchestrator = Orchestrator(client=mock.MagicMock())
+ captured = {}
+
+ def chat(messages, system="", **kwargs):
+ captured["text"] = "".join(
+ block.get("text", "")
+ for message in messages for block in message.get("content",
[]))
+ return {"output": {"message": {"content": [
+ {"text": "```hocon\nenv {}\nsource {}\nsink {}\n```"}]}},
+ "stopReason": "end_turn"}
+
+ orchestrator.client.chat.side_effect = chat
+
+ orchestrator._run_fix(
+ "source {\n Jdbc {}\n}",
+ "FAIL: missing url",
+ request="Load the table.",
+ plan_text="source: Jdbc\nsecret_key=AKIAIOSFODNN7EXAMPLE",
+ )
+
+ self.assertNotIn("AKIAIOSFODNN7EXAMPLE", captured["text"])
+ self.assertIn("source: Jdbc", captured["text"])
+
+
+if __name__ == "__main__":
+ unittest.main()