This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-13002-79d60f0708d67e386bf77f7d11e2243bcde2938d in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 9b8252eaf0c0eed5d80d5fd10a3b15eafeab9f58 Author: SEZ <[email protected]> AuthorDate: Fri Nov 6 13:53:03 2026 +0000 [Fix][seatunnel-cli] Catch a config that answers the wrong request (#13002) Co-authored-by: SEZ9 <[email protected]> --- seatunnel-cli/seatunnel_cli/agents.py | 137 +++++++- .../tests/test_orchestrator_plan_conformance.py | 346 +++++++++++++++++++++ 2 files changed, 478 insertions(+), 5 deletions(-) diff --git a/seatunnel-cli/seatunnel_cli/agents.py b/seatunnel-cli/seatunnel_cli/agents.py index 94219efbc2..edc190cb75 100644 --- a/seatunnel-cli/seatunnel_cli/agents.py +++ b/seatunnel-cli/seatunnel_cli/agents.py @@ -275,6 +275,85 @@ def _check_conditional_mismatches( ) +def _registered_connector_names() -> dict[str, set[str]] | None: + """Lower-cased source and sink plugin names the engine registers. + + Returns None when the registry is unavailable -- no metadata exported, or + the lookup failed -- in which case the caller compares plan names as given. + That is the pre-existing behaviour and the conservative choice: without a + registry we cannot tell an alias from a real mismatch. + """ + try: + from .connectors import list_connector_names + + names = list_connector_names() + except Exception: + return None + sources = {n.lower() for n in names.get("sources", []) if n} + sinks = {n.lower() for n in names.get("sinks", []) if n} + if not sources and not sinks: + return None + return {"source": sources, "sink": sinks} + + +def _plan_conformance_errors(structured_plan, config_str: str) -> list[str]: + """Report connectors the config uses that the plan did not ask for. + + The planner resolves the user's words into connector names; the generator + then has to use those names. When it does not, every later gate still + passes -- the config is valid, it just moves the wrong data or moves it the + wrong way -- so nothing downstream catches it. + + Only a connector named by the plan and absent from the config is reported. + Extra connectors are not: a plan for one pipeline can legitimately be + realised with an added Assert sink or a second branch, and flagging those + would make the check fight normal practice. + + The plan is model output, not ground truth, so a planned name that is not a + registered plugin is ignored rather than enforced. Otherwise an alias the + planner invented -- ``S3`` where the plugin is ``S3File`` -- would condemn a + correct config and spend every repair round moving it away from the right + connector. Comparison is case-insensitive. + """ + pipelines = getattr(structured_plan, "pipelines", None) or [] + if not pipelines: + return [] + + used: dict[str, set[str]] = {"source": set(), "sink": set()} + for section, name, _block in _extract_connector_blocks_raw(config_str): + if section in used and name: + used[section].add(name.strip().lower()) + if not used["source"] and not used["sink"]: + # Nothing recognisable to compare against; the HOCON checks own that. + return [] + + known = _registered_connector_names() + + errors: list[str] = [] + for section, attribute in (("source", "source_connector"), + ("sink", "sink_connector")): + planned = { + str(getattr(p, attribute, "") or "").strip().lower() + for p in pipelines + } + planned.discard("") + if known is not None: + # Keep only names the engine actually registers for this section, + # so an invented alias cannot condemn a correct config. + planned &= known.get(section, set()) + if not planned or not used[section]: + continue + unmet = planned - used[section] + if unmet: + errors.append( + f"{section}: the plan resolved this request to " + f"{', '.join(sorted(unmet))} but the config uses " + f"{', '.join(sorted(used[section]))}. Use the connector the " + f"request calls for." + ) + return errors + + def _extract_connector_blocks_raw( config_str: str, ) -> list[tuple[str, str, str]]: @@ -1518,6 +1597,26 @@ class Orchestrator: self.on_status("validating", f"Validating config (round {round_num + 1})...") validation = self._run_validator(config) + # A config can be entirely valid and still answer the wrong + # question. Picking Jdbc where the plan said MySQL-CDC produces + # HOCON that parses, loads and runs, so the validator passes it and + # the loop delivers it -- the generator's choice is never revisited. + # The plan already names the connectors, so this is a comparison + # rather than a judgement, and it is fed through the same repair + # loop as any other validation failure. + drift = _plan_conformance_errors(structured_plan, config) + if drift: + # The validator's contract is "PASS", "PASS_WITH_NOTES: ..." or + # "FAIL: <issue list>", so its first line carries a real issue. + # Keep its text whole and prepend the drift: dropping a line + # here would remove a genuine defect from the repair prompt in + # exactly the case where two things are wrong at once. + drift_text = "FAIL: plan drift\n" + "\n".join(drift) + validation = ( + drift_text if validation.startswith("PASS") + else drift_text + "\n" + validation + ) + if validation.startswith("PASS"): # Phase 4: Dry-run validation (engine-level) self.on_status("validating", "Running dry-run check...") @@ -1548,7 +1647,11 @@ class Orchestrator: # Validation failed — try to fix self.on_status("fixing", f"Fixing issues (round {round_num + 1})...") - fix_result = self._run_fix(config, validation) + fix_result = self._run_fix( + config, validation, + request=self._original_request, + plan_text=getattr(structured_plan, "raw_plan_text", "") or "", + ) if fix_result.get("config"): config = fix_result["config"] if fix_result.get("explanation"): @@ -1766,8 +1869,15 @@ Check for semantic correctness, required parameters, and best practices.""" result = self.client.quick_chat(prompt, system=VALIDATOR_SYSTEM) return result.strip() - def _run_fix(self, config: str, validation_errors: str) -> dict: - """Attempt to fix config based on validation errors.""" + def _run_fix(self, config: str, validation_errors: str, + request: str | None = None, plan_text: str = "") -> dict: + """Attempt to fix config based on validation errors. + + The request and the plan are included because without them a repair + cannot correct a wrong connector: the prompt held only the config and + the errors, so the model had nothing to compare the config against and + was told to keep everything not explicitly flagged. + """ from .cli import _replace_creds_with_placeholders, _restore_creds_from_placeholders system = self._build_config_system() @@ -1775,8 +1885,23 @@ Check for semantic correctness, required parameters, and best practices.""" # After LLM returns the fixed config, restore the original values. safe_config, cred_map = _replace_creds_with_placeholders(config) - prompt = f"""The following SeaTunnel config has validation issues. Fix them. + # The config is credential-swapped above for a reason, and the request + # is free text the user typed -- it can hold a password just as easily. + # `redact_credentials` is the same scrubber the planner path and the + # session writer use, so this keeps one rule for what leaves the host. + from .memory import redact_credentials + + context = "" + if request: + context += ( + "\n## The request this config must satisfy:\n" + f"{redact_credentials(request)}\n" + ) + if plan_text: + context += f"\n## The resolved plan:\n{redact_credentials(plan_text)}\n" + prompt = f"""The following SeaTunnel config has validation issues. Fix them. +{context} ## Current Config: ```hocon {safe_config} @@ -1785,7 +1910,9 @@ Check for semantic correctness, required parameters, and best practices.""" ## Validation Errors: {validation_errors} -Fix ALL the issues and return the corrected config. Keep all existing correct parts unchanged.""" +Fix ALL the issues and return the corrected config. Keep the parts that are +both correct and consistent with the request unchanged. If the config uses a +connector the request does not call for, replace it rather than preserving it.""" messages = [{"role": "user", "content": [{"text": prompt}]}] diff --git a/seatunnel-cli/tests/test_orchestrator_plan_conformance.py b/seatunnel-cli/tests/test_orchestrator_plan_conformance.py new file mode 100644 index 0000000000..a9fe596236 --- /dev/null +++ b/seatunnel-cli/tests/test_orchestrator_plan_conformance.py @@ -0,0 +1,346 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""A valid config that answers the wrong question must not be delivered. + +Choosing Jdbc where the plan resolved to MySQL-CDC produces HOCON that parses, +loads and runs, so every existing gate passes it. The generator's connector +choice was therefore never revisited, and the repair prompt held only the +config and the validation errors - nothing to compare the config against. +""" + +import unittest +from unittest import mock + +from seatunnel_cli.agents import Orchestrator, _plan_conformance_errors +from seatunnel_cli.skills import PipelineSlot, StructuredPlan + + +CDC_TO_CONSOLE = StructuredPlan( + pipelines=[PipelineSlot("pipeline_1", "MySQL-CDC", "Console", mode="STREAMING")], + raw_plan_text="source: MySQL-CDC\nsink: Console\nmode: STREAMING", +) + +WRONG_SOURCE = """env { + job.mode = "STREAMING" +} +source { + Jdbc { + url = "jdbc:mysql://localhost:3306/shop" + query = "select * from users" + } +} +sink { + Console {} +} +""" + +RIGHT_SOURCE = """env { + job.mode = "STREAMING" +} +source { + MySQL-CDC { + base-url = "jdbc:mysql://localhost:3306/shop" + table-names = ["shop.users"] + } +} +sink { + Console {} +} +""" + + +# The conformance check intersects planned names with what the engine +# registers, so a test that does not pin the registry depends on whichever +# metadata happens to be exported on the machine running it. On a host whose +# metadata omits MySQL-CDC the planned name would be filtered out, the expected +# drift would become an empty list, and the test would fail for a reason that +# has nothing to do with the code. Pinning it here keeps the registry a fixture +# rather than an input; the unavailable-registry case patches it separately. +FIXED_CATALOG = { + "sources": ["MySQL-CDC", "Postgres-CDC", "Jdbc", "Kafka", "S3File", "FakeSource"], + "sinks": ["Console", "Jdbc", "Clickhouse", "Assert", "LocalFile", "S3File"], + "transforms": ["Sql", "Filter"], +} + + +class PinnedRegistryMixin: + """Pin the connector registry for every test in the class.""" + + def setUp(self): + super().setUp() + patcher = mock.patch( + "seatunnel_cli.connectors.list_connector_names", + return_value=FIXED_CATALOG, + ) + patcher.start() + self.addCleanup(patcher.stop) + + +class ConformanceCheckTest(PinnedRegistryMixin, unittest.TestCase): + def test_wrong_source_connector_is_reported(self): + errors = _plan_conformance_errors(CDC_TO_CONSOLE, WRONG_SOURCE) + self.assertEqual(len(errors), 1) + self.assertIn("mysql-cdc", errors[0]) + self.assertIn("jdbc", errors[0]) + + def test_matching_connectors_report_nothing(self): + self.assertEqual(_plan_conformance_errors(CDC_TO_CONSOLE, RIGHT_SOURCE), []) + + def test_case_differences_are_not_drift(self): + plan = StructuredPlan( + pipelines=[PipelineSlot("p", "jdbc", "console")]) + config = "source {\n Jdbc {}\n}\nsink {\n Console {}\n}\n" + self.assertEqual(_plan_conformance_errors(plan, config), []) + + def test_an_extra_sink_is_allowed(self): + """A plan for one pipeline may legitimately gain an Assert sink.""" + plan = StructuredPlan( + pipelines=[PipelineSlot("p", "Jdbc", "Console")]) + config = ("source {\n Jdbc {}\n}\n" + "sink {\n Console {}\n Assert {}\n}\n") + self.assertEqual(_plan_conformance_errors(plan, config), []) + + def test_no_plan_means_no_opinion(self): + self.assertEqual( + _plan_conformance_errors(StructuredPlan(pipelines=[]), WRONG_SOURCE), []) + + def test_a_planned_name_the_engine_does_not_register_is_ignored(self): + """The plan is model output; an invented alias must not condemn a config. + + A planner that writes ``S3`` where the plugin is ``S3File`` would + otherwise spend every repair round moving a correct config away from + the right connector. + """ + plan = StructuredPlan(pipelines=[PipelineSlot("p", "S3", "Console")]) + config = "source {\n S3File {}\n}\nsink {\n Console {}\n}\n" + + with mock.patch("seatunnel_cli.connectors.list_connector_names", + return_value={"sources": ["S3File", "Jdbc"], + "sinks": ["Console"], "transforms": []}): + self.assertEqual(_plan_conformance_errors(plan, config), []) + + def test_a_registered_name_is_still_enforced(self): + plan = StructuredPlan(pipelines=[PipelineSlot("p", "MySQL-CDC", "Console")]) + + with mock.patch("seatunnel_cli.connectors.list_connector_names", + return_value={"sources": ["MySQL-CDC", "Jdbc"], + "sinks": ["Console"], "transforms": []}): + errors = _plan_conformance_errors(plan, WRONG_SOURCE) + + self.assertEqual(len(errors), 1) + self.assertIn("mysql-cdc", errors[0]) + + def test_an_unavailable_registry_falls_back_to_comparing_as_given(self): + with mock.patch("seatunnel_cli.connectors.list_connector_names", + side_effect=RuntimeError("no metadata exported")): + errors = _plan_conformance_errors(CDC_TO_CONSOLE, WRONG_SOURCE) + + self.assertEqual(len(errors), 1) + + def test_unrecognisable_config_is_left_to_the_hocon_checks(self): + self.assertEqual(_plan_conformance_errors(CDC_TO_CONSOLE, "not hocon"), []) + + +class DriftReachesTheRepairLoopTest(PinnedRegistryMixin, unittest.TestCase): + def _run(self, generated_configs): + orchestrator = Orchestrator(client=mock.MagicMock()) + skill = mock.MagicMock() + skill.fill_and_check.return_value = [] + skill.fetch_all_metadata.return_value = "## Connector Option Rules\n" + skill.build_enriched_prompt.return_value = "## User Request:\nx" + + fix_prompts: list[tuple[str, str]] = [] + + def run_fix(config, validation_errors, request=None, plan_text=""): + fix_prompts.append((validation_errors, f"{request}|{plan_text}")) + return {"config": generated_configs.pop(0), "explanation": ""} + + with mock.patch.object( + orchestrator, "_run_planner", + return_value={"type": "plan", "content": CDC_TO_CONSOLE.raw_plan_text}, + ), mock.patch("seatunnel_cli.skills.SkillExecutor", return_value=skill), \ + mock.patch("seatunnel_cli.skills.SkillRouter.match", return_value=[]), \ + mock.patch("seatunnel_cli.skills.parse_structured_plan", + return_value=CDC_TO_CONSOLE), \ + mock.patch.object( + orchestrator, "_run_config_generator", + return_value={"config": generated_configs.pop(0), + "explanation": ""}), \ + mock.patch.object(orchestrator, "_run_validator", return_value="PASS"), \ + mock.patch.object(orchestrator, "_run_fix", side_effect=run_fix), \ + mock.patch("seatunnel_cli.agents.dry_run_config", + return_value={"valid": True, "phase1_local": "VALID", + "phase2_check": "PASS", "phase3_api": None, + "summary": ""}): + result = orchestrator.process_user_input( + "Watch the users table in MySQL at localhost:3306 and show " + "changes as they happen.") + return result, fix_prompts + + def test_a_valid_but_wrong_config_is_sent_for_repair(self): + # The validator says PASS, so before this the loop delivered it. + result, fix_prompts = self._run([WRONG_SOURCE, RIGHT_SOURCE]) + + self.assertEqual(len(fix_prompts), 1, "drift should trigger one repair") + errors, context = fix_prompts[0] + self.assertIn("mysql-cdc", errors) + self.assertEqual(result["type"], "config") + self.assertIn("MySQL-CDC", result["config"]) + + def test_the_repair_is_told_the_request_and_the_plan(self): + _result, fix_prompts = self._run([WRONG_SOURCE, RIGHT_SOURCE]) + + _errors, context = fix_prompts[0] + self.assertIn("show changes as they happen", context) + self.assertIn("MySQL-CDC", context) + + def test_a_conforming_config_is_delivered_without_repair(self): + result, fix_prompts = self._run([RIGHT_SOURCE]) + + self.assertEqual(fix_prompts, []) + self.assertEqual(result["type"], "config") + + +class DriftMergedWithValidatorOutputTest(PinnedRegistryMixin, unittest.TestCase): + """Drift must not displace the validator's own findings. + + The validator's contract is "FAIL: <issue list>", so its first line carries + a real issue. An earlier version sliced that line off when both drift and a + validation failure occurred, removing a genuine defect from the repair + prompt in exactly the case where two things were wrong at once. + """ + + def _validation_seen_by_repair(self, validator_output): + orchestrator = Orchestrator(client=mock.MagicMock()) + skill = mock.MagicMock() + skill.fill_and_check.return_value = [] + skill.fetch_all_metadata.return_value = "## Connector Option Rules\n" + skill.build_enriched_prompt.return_value = "## User Request:\nx" + seen: list[str] = [] + + def run_fix(config, validation_errors, request=None, plan_text=""): + seen.append(validation_errors) + return {"config": RIGHT_SOURCE, "explanation": ""} + + with mock.patch.object( + orchestrator, "_run_planner", + return_value={"type": "plan", "content": CDC_TO_CONSOLE.raw_plan_text}, + ), mock.patch("seatunnel_cli.skills.SkillExecutor", return_value=skill), \ + mock.patch("seatunnel_cli.skills.SkillRouter.match", return_value=[]), \ + mock.patch("seatunnel_cli.skills.parse_structured_plan", + return_value=CDC_TO_CONSOLE), \ + mock.patch.object( + orchestrator, "_run_config_generator", + return_value={"config": WRONG_SOURCE, "explanation": ""}), \ + mock.patch.object(orchestrator, "_run_validator", + return_value=validator_output), \ + mock.patch.object(orchestrator, "_run_fix", side_effect=run_fix), \ + mock.patch("seatunnel_cli.agents.dry_run_config", + return_value={"valid": True, "phase1_local": "VALID", + "phase2_check": "PASS", "phase3_api": None, + "summary": ""}): + orchestrator.process_user_input("Watch the users table for changes.") + return seen[0] + + def test_every_validator_issue_survives_the_merge(self): + merged = self._validation_seen_by_repair( + "FAIL: missing env block\nplugin_input is not wired\nunknown option foo") + + self.assertIn("missing env block", merged) + self.assertIn("plugin_input is not wired", merged) + self.assertIn("unknown option foo", merged) + self.assertIn("mysql-cdc", merged) + + def test_drift_alone_when_the_validator_passed(self): + merged = self._validation_seen_by_repair("PASS") + + self.assertIn("mysql-cdc", merged) + self.assertNotIn("PASS", merged) + + +class RepairPromptRedactionTest(unittest.TestCase): + """The request is free text the user typed and can hold a password. + + `_run_fix` already swaps credentials out of the config before calling the + model; adding the request and the plan unredacted would route around that. + These assert that the existing scrubber is applied to the new context, so + the request path has the same protection as the config path. + + How much `redact_credentials` itself catches is a separate matter: it keys + on `key = value`, `key: value` and "password is X", so a bare "password X" + in prose is not matched. Widening it would change a helper that session + persistence and memory also use, and would start redacting phrases like + "password rotation policy". That belongs in its own change, not here. + """ + + def test_a_password_in_the_request_does_not_reach_the_model(self): + orchestrator = Orchestrator(client=mock.MagicMock()) + captured = {} + + def chat(messages, system="", **kwargs): + captured["text"] = "".join( + block.get("text", "") + for message in messages for block in message.get("content", [])) + return {"output": {"message": {"content": [ + {"text": "```hocon\nenv {}\nsource {}\nsink {}\n```"}]}}, + "stopReason": "end_turn"} + + orchestrator.client.chat.side_effect = chat + + orchestrator._run_fix( + "source {\n Jdbc {\n url = \"jdbc:mysql://h/db\"\n }\n}", + "FAIL: missing driver", + request=("Copy users from MySQL at localhost:3306. The password " + "is Hunter2Secret! and the user is root."), + plan_text="source: Jdbc\nsink: Console", + ) + + self.assertNotIn("Hunter2Secret!", captured["text"]) + # The rest of the request must survive, or the repair loses the context + # that lets it tell a wrong connector from a right one. + self.assertIn("Copy users from MySQL", captured["text"]) + self.assertIn("localhost:3306", captured["text"]) + self.assertIn("source: Jdbc", captured["text"]) + + def test_a_key_value_secret_in_the_plan_is_redacted_too(self): + orchestrator = Orchestrator(client=mock.MagicMock()) + captured = {} + + def chat(messages, system="", **kwargs): + captured["text"] = "".join( + block.get("text", "") + for message in messages for block in message.get("content", [])) + return {"output": {"message": {"content": [ + {"text": "```hocon\nenv {}\nsource {}\nsink {}\n```"}]}}, + "stopReason": "end_turn"} + + orchestrator.client.chat.side_effect = chat + + orchestrator._run_fix( + "source {\n Jdbc {}\n}", + "FAIL: missing url", + request="Load the table.", + plan_text="source: Jdbc\nsecret_key=AKIAIOSFODNN7EXAMPLE", + ) + + self.assertNotIn("AKIAIOSFODNN7EXAMPLE", captured["text"]) + self.assertIn("source: Jdbc", captured["text"]) + + +if __name__ == "__main__": + unittest.main()
