This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 8513c1d580f Add Mask planning scenario to MCP LLM E2E (#39289)
8513c1d580f is described below
commit 8513c1d580f7fc6a543ee631a53ee2df0030c522
Author: Liang Zhang <[email protected]>
AuthorDate: Fri Jul 31 19:41:06 2026 +0800
Add Mask planning scenario to MCP LLM E2E (#39289)
* Add Mask planning scenario to MCP LLM E2E
* Add Mask planning scenario to MCP LLM E2E
---
.github/workflows/e2e-mcp.yml | 2 +
.../test/e2e/mcp/llm/LLMHttpE2ETest.java | 95 +++++++++++++++++++---
.../llm/conversation/LLMConversationRunner.java | 11 +--
.../conversation/LLMMCPToolDefinitionFactory.java | 32 +++-----
4 files changed, 103 insertions(+), 37 deletions(-)
diff --git a/.github/workflows/e2e-mcp.yml b/.github/workflows/e2e-mcp.yml
index 9c4b750cdad..ebf6e3e4c27 100644
--- a/.github/workflows/e2e-mcp.yml
+++ b/.github/workflows/e2e-mcp.yml
@@ -269,6 +269,8 @@ jobs:
test: LLMHttpE2ETest#assertReadOnlyQuery
- id: http-metadata-discovery
test: LLMHttpE2ETest#assertMetadataDiscovery
+ - id: http-mask-planning
+ test: LLMHttpE2ETest#assertMaskPlanning
- id: http-side-effect-preview
test: LLMHttpE2ETest#assertSideEffectPreview
- id: http-invalid-resource-recovery
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
index c3f9e358c9b..3d9433e9042 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/LLMHttpE2ETest.java
@@ -48,6 +48,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Collection;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
@@ -74,14 +75,26 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
private static final String STALE_TABLE_RESOURCE_URI =
"shardingsphere://databases/logic_db/schemas/logic_db/tables/missing_orders";
+ private static final String READ_RESOURCE_TOOL_NAME = "mcp_read_resource";
+
+ private static final String SEARCH_METADATA_TOOL_NAME =
"database_gateway_search_metadata";
+
+ private static final String EXECUTE_QUERY_TOOL_NAME =
"database_gateway_execute_query";
+
+ private static final String EXECUTE_UPDATE_TOOL_NAME =
"database_gateway_execute_update";
+
+ private static final String PLAN_MASK_RULE_TOOL_NAME =
"database_gateway_plan_mask_rule";
+
private static final Set<String> EXPECTED_METADATA_NAMES =
Set.of("active_orders", "order_items", "orders");
private static final List<String> ARTIFACT_FILES = List.of(
"run-context.json", "system-prompt.md", "question.txt",
"answer.txt", "raw-model-output.txt", "available-tools.json",
"interaction-trace.json", "assertion-report.json");
+ // JSON boolean and null literals are metadata rather than credential
values.
private static final Pattern UNREDACTED_SECRET_PATTERN = Pattern.compile(
-
"(?i)(?<![a-z0-9_])\"?(?:api[_-]?key|access[_-]?token|token|authorization|password|passwd|pwd|secret)\"?\\s*[:=]\\s*[\"']?(?!<redacted>)[^\\s,\"'}]+"
+
"(?i)(?<![a-z0-9_])\"?(?:api[_-]?key|access[_-]?token|token|authorization|password|passwd|pwd|secret)\"?\\s*[:=]\\s*"
+ +
"(?!(?:true|false|null)(?:\\s*[,}\\]]|\\s*$))[\"']?(?!<redacted>)[^\\s,\"'}]+"
+ "|(Bearer\\s+)(?!<redacted>)[A-Za-z0-9._~+/=-]+|jdbc:");
private static LLMRuntimeSupport.ModelRuntime llmRuntime;
@@ -116,7 +129,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
runScenario(new Scenario(
"read-only-query",
"How many rows are currently in the orders table of the
logic_db runtime database? Inspect the live MCP server and answer concisely.",
- false,
+ Set.of(EXECUTE_QUERY_TOOL_NAME),
this::evaluateReadOnlyQuery));
}
@@ -125,10 +138,22 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
runScenario(new Scenario(
"metadata-discovery",
"List every table or view currently visible through the live
MCP server. The user does not know the database or schema names, so discover
the required scope first.",
- false,
+ Set.of(SEARCH_METADATA_TOOL_NAME),
this::evaluateMetadataDiscovery));
}
+ @Test
+ void assertMaskPlanning() throws IOException {
+ runScenario(new Scenario(
+ "mask-planning",
+ "The target metadata has already been verified, so do not run
separate metadata discovery. Create a reviewable Mask rule plan, without
applying it, "
+ + "for the status column of the orders table in the
logic_db database and logic_db schema. "
+ + "Use the create operation, KEEP_FIRST_N_LAST_M
algorithm, and primary properties first-n=1, last-m=1, replace-char=*. "
+ + "Report the plan ID and explicitly confirm that
nothing was applied.",
+ Set.of(PLAN_MASK_RULE_TOOL_NAME),
+ this::evaluateMaskPlanning));
+ }
+
@Test
void assertSideEffectPreview() throws IOException, SQLException {
prepareRuntimeFixture();
@@ -137,7 +162,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
"side-effect-preview",
"A user wants to change the status of order 1 in the orders
table to REVIEW_PENDING. Preview the proposed change without executing it, then
report whether it "
+ "was only previewed and whether any data changed.
Inspect the live MCP server to discover any required runtime scope.",
- true,
+ Set.of(SEARCH_METADATA_TOOL_NAME, EXECUTE_UPDATE_TOOL_NAME),
(answer, trace) -> evaluateSideEffectPreview(answer, trace,
statusBefore)));
}
@@ -147,7 +172,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
"invalid-resource-recovery",
"A user pasted stale resource `" + STALE_TABLE_RESOURCE_URI +
"`. Inspect that resource, then follow the first safe read-only action in its
top-level "
+ "`next_actions` by reading its `resource_uri`
exactly. Do not guess another URI. Then report how many rows are currently in
the orders table.",
- false,
+ Set.of(READ_RESOURCE_TOOL_NAME, EXECUTE_QUERY_TOOL_NAME),
this::evaluateInvalidResourceRecovery));
}
@@ -189,7 +214,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
Set<String> actualMetadataNames = new LinkedHashSet<>();
for (int index = unscopedSearchIndex; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
- if (isValidModelAction(each, "database_gateway_search_metadata")
&& (unscopedSearchIndex == index || each.getModelTurn() > discoveryTurn)) {
+ if (isValidModelAction(each, SEARCH_METADATA_TOOL_NAME) &&
(unscopedSearchIndex == index || each.getModelTurn() > discoveryTurn)) {
actualMetadataNames.addAll(getMetadataNames(each.getStructuredContent()));
}
}
@@ -202,9 +227,57 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
: LLME2EAssertionReport.failure("answer_mismatch", "The answer
omitted an object returned by the live MCP metadata searches.");
}
+ private LLME2EAssertionReport evaluateMaskPlanning(final String answer,
final List<MCPInteractionTraceRecord> trace) {
+ Optional<MCPInteractionTraceRecord> plannedAction = trace.stream()
+ .filter(each -> isValidModelAction(each,
PLAN_MASK_RULE_TOOL_NAME))
+ .filter(each ->
"planned".equals(each.getStructuredContent().get("status")))
+ .findFirst();
+ if (plannedAction.isEmpty()) {
+ return LLME2EAssertionReport.failure("missing_mask_plan", "The
model did not produce a planned Mask workflow.");
+ }
+ if (!hasExpectedMaskPlanArguments(trace)) {
+ return
LLME2EAssertionReport.failure("mask_plan_arguments_mismatch", "The model did
not send the requested Mask rule inputs.");
+ }
+ Map<String, Object> response =
plannedAction.get().getStructuredContent();
+ String planId = Objects.toString(response.get("plan_id"), "").trim();
+ List<Map<String, Object>> distSQLArtifacts =
getObjectList(response.get("distsql_artifacts"));
+ Map<String, Object> primaryProperties =
getObjectMap(getObjectMap(response.get("masked_property_preview")).get("primary"));
+ if (planId.isEmpty() ||
!"mask.rule".equals(response.get("workflow_kind")) ||
!"review".equals(response.get("current_step"))
+ || !getList(response.get("missing_required_inputs")).isEmpty()
|| distSQLArtifacts.isEmpty()
+ || !Objects.toString(distSQLArtifacts.getFirst().get("sql"),
"").startsWith("CREATE MASK RULE `orders`")
+ ||
!"1".equals(Objects.toString(primaryProperties.get("first-n"), ""))
+ ||
!"1".equals(Objects.toString(primaryProperties.get("last-m"), ""))
+ || !"*".equals(primaryProperties.get("replace-char"))) {
+ return
LLME2EAssertionReport.failure("mask_plan_evidence_mismatch", "The MCP response
did not contain the requested reviewable Mask plan.");
+ }
+ String normalizedAnswer = answer.toLowerCase(Locale.ENGLISH);
+ boolean answerReportsNoApplication = normalizedAnswer.contains("not
applied") || normalizedAnswer.contains("not been applied")
+ || normalizedAnswer.contains("nothing was applied") ||
normalizedAnswer.contains("without applying")
+ || normalizedAnswer.contains("not executed") ||
normalizedAnswer.contains("no changes were applied");
+ return answer.contains(planId) && answerReportsNoApplication
+ ? LLME2EAssertionReport.success("The answer reported the live
Mask plan ID and confirmed that it was not applied.")
+ : LLME2EAssertionReport.failure("answer_mismatch", "The answer
did not report the Mask plan ID and confirm that it was not applied.");
+ }
+
+ private boolean hasExpectedMaskPlanArguments(final
List<MCPInteractionTraceRecord> trace) {
+ Map<String, Object> arguments = new LinkedHashMap<>();
+ Map<String, Object> primaryProperties = new LinkedHashMap<>();
+ for (MCPInteractionTraceRecord each : trace) {
+ if (isValidModelAction(each, PLAN_MASK_RULE_TOOL_NAME)) {
+ arguments.putAll(each.getArguments());
+
primaryProperties.putAll(getObjectMap(each.getArguments().get("primary_algorithm_properties")));
+ }
+ }
+ return DATABASE_NAME.equals(arguments.get("database")) &&
TABLE_NAME.equals(arguments.get("table")) &&
"status".equals(arguments.get("column"))
+ && "create".equals(arguments.get("operation_type")) &&
"KEEP_FIRST_N_LAST_M".equals(arguments.get("algorithm_type"))
+ &&
"1".equals(Objects.toString(primaryProperties.get("first-n"), ""))
+ &&
"1".equals(Objects.toString(primaryProperties.get("last-m"), ""))
+ && "*".equals(primaryProperties.get("replace-char"));
+ }
+
private LLME2EAssertionReport evaluateSideEffectPreview(final String
answer, final List<MCPInteractionTraceRecord> trace, final String statusBefore)
{
Optional<MCPInteractionTraceRecord> preview = trace.stream()
- .filter(each -> isValidModelAction(each,
"database_gateway_execute_update"))
+ .filter(each -> isValidModelAction(each,
EXECUTE_UPDATE_TOOL_NAME))
.filter(each ->
"preview".equals(each.getArguments().get("execution_mode")))
.filter(this::isSentinelPreview)
.findFirst();
@@ -258,7 +331,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
private int findStaleResourceIndex(final List<MCPInteractionTraceRecord>
trace) {
for (int index = 0; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
- if (isValidModelAction(each, "mcp_read_resource")
+ if (isValidModelAction(each, READ_RESOURCE_TOOL_NAME)
&&
STALE_TABLE_RESOURCE_URI.equals(each.getArguments().get("uri"))
&& hasRecoveryCategory(each.getStructuredContent(),
"object_not_visible")) {
return index;
@@ -280,7 +353,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
for (int index = staleResourceIndex + 1; index < trace.size();
index++) {
MCPInteractionTraceRecord each = trace.get(index);
if (each.getModelTurn() > staleResource.getModelTurn()
- && isValidModelAction(each, "mcp_read_resource")
+ && isValidModelAction(each, READ_RESOURCE_TOOL_NAME)
&& Objects.equals(recoveryAction.get("resource_uri"),
each.getArguments().get("uri"))
&& containsOrdersTable(each.getStructuredContent())) {
return index;
@@ -302,7 +375,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
private Optional<Integer> findQueryCount(final
List<MCPInteractionTraceRecord> trace, final int previousModelTurn) {
for (MCPInteractionTraceRecord each : trace) {
- if (each.getModelTurn() <= previousModelTurn ||
!isValidModelAction(each, "database_gateway_execute_query") ||
!isOrdersCountQuery(each.getArguments())) {
+ if (each.getModelTurn() <= previousModelTurn ||
!isValidModelAction(each, EXECUTE_QUERY_TOOL_NAME) ||
!isOrdersCountQuery(each.getArguments())) {
continue;
}
for (Map<String, Object> row :
getObjectList(each.getStructuredContent().get("row_objects"))) {
@@ -341,7 +414,7 @@ class LLMHttpE2ETest extends
AbstractConfigBackedRuntimeE2ETest {
private int findUnscopedMetadataSearchIndex(final
List<MCPInteractionTraceRecord> trace) {
for (int index = 0; index < trace.size(); index++) {
MCPInteractionTraceRecord each = trace.get(index);
- if (isValidModelAction(each, "database_gateway_search_metadata")
+ if (isValidModelAction(each, SEARCH_METADATA_TOOL_NAME)
&& !each.getArguments().containsKey("database") &&
!each.getArguments().containsKey("schema")
&&
getObjectList(each.getStructuredContent().get("items")).stream().anyMatch(item
-> DATABASE_NAME.equals(item.get("database")))) {
return index;
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
index 668aead231d..d880d695749 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMConversationRunner.java
@@ -50,8 +50,9 @@ public final class LLMConversationRunner {
private static final String SYSTEM_PROMPT = """
You are evaluating a live ShardingSphere MCP server. Use the
available MCP functions to inspect current state and complete the user's task.
Choose tools from their advertised names, descriptions, and input
schemas. Pass resource URIs only to mcp_read_resource, never as SQL.
- Never guess or stop before retrieving the requested evidence.
Preview side effects without executing them. Use function calls for every
- tool invocation rather than printing a tool-call object as the
answer. When you have enough evidence, answer the user's question concisely.
+ Never guess or stop before retrieving the requested evidence. When
target metadata is explicitly marked as already verified, use the requested
+ planning response as evidence instead of repeating metadata
discovery. Preview side effects without executing them. Use function calls for
+ every tool invocation rather than printing a tool-call object as
the answer. When you have enough evidence, answer the user's question concisely.
""";
private final int maxTurns;
@@ -88,7 +89,7 @@ public final class LLMConversationRunner {
try {
mcpInteractionClient.open();
List<Map<String, Object>> advertisedTools =
mcpInteractionClient.listTools();
- List<Map<String, Object>> toolDefinitions =
toolDefinitionFactory.createFromRemote(advertisedTools,
scenario.allowsSideEffectPreview());
+ List<Map<String, Object>> toolDefinitions =
toolDefinitionFactory.createFromRemote(advertisedTools,
scenario.allowedToolNames());
artifacts.setToolDefinitions(toolDefinitions);
return runTurns(scenario, artifacts, toolDefinitions);
} catch (final IOException ex) {
@@ -224,10 +225,10 @@ public final class LLMConversationRunner {
*
* @param id scenario ID
* @param question question
- * @param allowsSideEffectPreview whether the scenario may expose update
preview
+ * @param allowedToolNames tools exposed only to this scenario
* @param evaluator scenario evidence evaluator
*/
- public record Scenario(String id, String question, boolean
allowsSideEffectPreview,
+ public record Scenario(String id, String question, Set<String>
allowedToolNames,
BiFunction<String, List<MCPInteractionTraceRecord>,
LLME2EAssertionReport> evaluator) {
}
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
index fe2c12b3ea1..0e1e55d979a 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/conversation/LLMMCPToolDefinitionFactory.java
@@ -17,45 +17,35 @@
package org.apache.shardingsphere.test.e2e.mcp.llm.conversation;
+import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
final class LLMMCPToolDefinitionFactory {
- private static final String EXECUTE_UPDATE_TOOL_NAME =
"database_gateway_execute_update";
-
- List<Map<String, Object>> createFromRemote(final List<Map<String, Object>>
advertisedTools, final boolean includeUpdatePreview) {
+ List<Map<String, Object>> createFromRemote(final List<Map<String, Object>>
advertisedTools, final Set<String> allowedToolNames) {
List<Map<String, Object>> result = new LinkedList<>();
- result.add(createReadResourceToolDefinition());
- boolean readOnlyToolFound = false;
- boolean updatePreviewToolFound = false;
+ Set<String> missingAllowedToolNames = new
LinkedHashSet<>(allowedToolNames);
+ if
(missingAllowedToolNames.remove(LLMConversationRunner.READ_RESOURCE_TOOL_NAME))
{
+ result.add(createReadResourceToolDefinition());
+ }
for (Map<String, Object> each : advertisedTools) {
String toolName = Objects.toString(each.get("name"), "").trim();
- boolean readOnly = isReadOnly(each);
- boolean updatePreview = includeUpdatePreview &&
EXECUTE_UPDATE_TOOL_NAME.equals(toolName);
- if (!readOnly && !updatePreview) {
+ if (!missingAllowedToolNames.contains(toolName)) {
continue;
}
result.add(createRemoteToolDefinition(each, toolName));
- readOnlyToolFound |= readOnly;
- updatePreviewToolFound |= updatePreview;
- }
- if (!readOnlyToolFound) {
- throw new IllegalStateException("MCP runtime did not advertise any
read-only tools.");
+ missingAllowedToolNames.remove(toolName);
}
- if (includeUpdatePreview && !updatePreviewToolFound) {
- throw new IllegalStateException("MCP runtime did not advertise
required preview tool: " + EXECUTE_UPDATE_TOOL_NAME);
+ if (!missingAllowedToolNames.isEmpty()) {
+ throw new IllegalStateException("MCP runtime did not advertise
required scenario tools: " + missingAllowedToolNames);
}
return result;
}
- private boolean isReadOnly(final Map<String, Object> advertisedTool) {
- Object annotations = advertisedTool.get("annotations");
- return annotations instanceof Map && Boolean.TRUE.equals(((Map<?, ?>)
annotations).get("readOnlyHint"));
- }
-
private Map<String, Object> createRemoteToolDefinition(final Map<String,
Object> advertisedTool, final String toolName) {
Object inputSchema = advertisedTool.get("inputSchema");
if (toolName.isEmpty() || !(inputSchema instanceof Map)) {
