This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 325b0c56874 Improve MCP builder evaluation coverage (#39044)
325b0c56874 is described below
commit 325b0c56874280030e30ca9efc97ada1b87961d5
Author: Liang Zhang <[email protected]>
AuthorDate: Wed Jul 8 10:00:27 2026 +0800
Improve MCP builder evaluation coverage (#39044)
* Improve MCP builder evaluation coverage
Strengthen MCP builder evaluation guidance for feature planning and plugin
resource discovery. Add encrypt completion and tool-result redaction
coverage,
and simplify completion context handling without changing behavior.
* Improve MCP builder evaluation coverage
Strengthen MCP builder evaluation guidance for feature planning and plugin
resource discovery. Add encrypt completion and tool-result redaction
coverage,
and simplify completion context handling without changing behavior.
---
.../MCPCompletionSpecificationFactory.java | 3 +--
.../tool/MCPCallToolResultFactoryTest.java | 3 +++
.../metadata/query/DatabaseTestDataFactory.java | 6 +++---
.../llm/suite/MCPBuilderEvaluationArtifactTest.java | 11 +++++++++++
.../HttpTransportCompletionE2ETest.java | 2 ++
.../llm/evaluation/mcp-builder-evaluation.xml | 21 ++++++++++++---------
6 files changed, 32 insertions(+), 14 deletions(-)
diff --git
a/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
b/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
index 58a5c9947eb..01f7abcc00e 100644
---
a/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
+++
b/mcp/bootstrap/src/main/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/completion/MCPCompletionSpecificationFactory.java
@@ -29,7 +29,6 @@ import
org.apache.shardingsphere.mcp.support.descriptor.MCPCompletionTargetDescr
import
org.apache.shardingsphere.mcp.support.descriptor.MCPDescriptorCatalogIndex;
import java.util.Collection;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -65,7 +64,7 @@ public final class MCPCompletionSpecificationFactory {
final
MCPCompletionTargetDescriptor descriptor) {
String argumentName = request.argument().name();
String prefix = Objects.toString(request.argument().value(), "");
- Map<String, String> contextArguments = new LinkedHashMap<>(null ==
request.context() || null == request.context().arguments() ? Map.of() :
request.context().arguments());
+ Map<String, String> contextArguments = null == request.context() ||
null == request.context().arguments() ? Map.of() :
request.context().arguments();
try {
MCPCompletionResult result =
completionService.complete(exchange.sessionId(), descriptor, argumentName,
prefix, contextArguments);
return new McpSchema.CompleteResult(new
McpSchema.CompleteResult.CompleteCompletion(result.getValues(),
result.getTotal(), result.isHasMore()), result.getMeta());
diff --git
a/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
b/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
index 9841268eec8..ae0b1b029eb 100644
---
a/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
+++
b/mcp/bootstrap/src/test/java/org/apache/shardingsphere/mcp/bootstrap/transport/capability/tool/MCPCallToolResultFactoryTest.java
@@ -120,6 +120,9 @@ class MCPCallToolResultFactoryTest extends
AbstractMCPToolSpecificationFactoryTe
CallToolResult actual =
createRealDescriptorCallToolResult(EncryptFeatureDefinition.PLAN_TOOL_NAME,
createEncryptPlanResponse());
assertFalse(actual.isError(), () ->
String.valueOf(actual.structuredContent()));
assertThat(getStructuredContent(actual).get("status"), is("planned"));
+ String actualText = ((TextContent) actual.content().getFirst()).text();
+ assertFalse(actualText.contains("123456"));
+ assertTrue(actualText.contains("******"));
}
@Test
diff --git
a/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
b/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
index 52b2e58b01f..19c73f03d26 100644
---
a/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
+++
b/mcp/support/src/test/java/org/apache/shardingsphere/mcp/support/database/metadata/query/DatabaseTestDataFactory.java
@@ -52,7 +52,7 @@ final class DatabaseTestDataFactory {
static List<MCPDatabaseMetadata> createDatabaseMetadata() {
return List.of(
- new MCPDatabaseMetadata("logic_db", "FixtureDB", "", List.of(
+ new MCPDatabaseMetadata("logic_db", "MySQL", "", List.of(
new MCPSchemaMetadata("logic_db", "public", List.of(
new MCPTableMetadata("logic_db", "public",
"orders",
List.of(new
MCPColumnMetadata("logic_db", "public", "orders", "", "order_id")),
@@ -62,9 +62,9 @@ final class DatabaseTestDataFactory {
List.of(new MCPViewMetadata("logic_db",
"public", "orders_view",
List.of(new
MCPColumnMetadata("logic_db", "public", "", "orders_view", "order_id")))),
List.of()))),
- new MCPDatabaseMetadata("runtime_db", "FixtureRuntimeDB", "",
List.of(
+ new MCPDatabaseMetadata("runtime_db", "PostgreSQL", "",
List.of(
new MCPSchemaMetadata("runtime_db", "public",
List.of(), List.of(), List.of(new MCPSequenceMetadata("runtime_db", "public",
"order_seq"))))),
- new MCPDatabaseMetadata("warehouse", "FixtureWarehouseDB", "",
List.of(
+ new MCPDatabaseMetadata("warehouse", "Hive", "", List.of(
new MCPSchemaMetadata("warehouse", "warehouse",
List.of(new MCPTableMetadata("warehouse", "warehouse", "facts", List.of(),
List.of())), List.of(), List.of()))));
}
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
index 1c77c02cf4a..5c6f024cac9 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/MCPBuilderEvaluationArtifactTest.java
@@ -102,6 +102,17 @@ final class MCPBuilderEvaluationArtifactTest {
NodeList qaPairs = root.getElementsByTagName("qa_pair");
assertThat(qaPairs.getLength(), is(EXPECTED_QA_PAIR_COUNT));
assertEvaluationQuestions(qaPairs);
+ String content = root.getTextContent();
+ for (String each : Set.of(
+ "database_gateway_plan_encrypt_rule",
+ "database_gateway_plan_mask_rule",
+ "database_gateway_plan_broadcast_rule",
+ "database_gateway_plan_readwrite_splitting_rule",
+ "database_gateway_plan_shadow_rule",
+ "database_gateway_plan_sharding_table_rule",
+ "completion/complete")) {
+ assertTrue(content.contains(each), () -> "Evaluation artifact must
cover " + each);
+ }
}
@Test
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
index b182584f9af..0cb6618201e 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/runtime/programmatic/HttpTransportCompletionE2ETest.java
@@ -77,6 +77,8 @@ class HttpTransportCompletionE2ETest extends
AbstractSharedHttpProgrammaticRunti
launchHttpTransport();
HttpClient httpClient = HttpClient.newHttpClient();
String sessionId = initializeSession(httpClient);
+ assertCompletionValuesContain("encrypt algorithm completion",
httpClient, sessionId,
+ Map.of("type", "ref/prompt", "name", "plan_encrypt_rule"),
"algorithm_type", "AES", "AES");
assertCompletionValuesContain("readwrite-splitting load balancer
completion", httpClient, sessionId,
Map.of("type", "ref/prompt", "name",
"plan_readwrite_splitting_rule"), "load_balancer_type", "ROUND", "ROUND_ROBIN");
assertCompletionValuesContain("shadow algorithm completion",
httpClient, sessionId,
diff --git
a/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
b/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
index ae21d68efd1..ddaa93219ed 100644
--- a/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
+++ b/test/e2e/mcp/src/test/resources/llm/evaluation/mcp-builder-evaluation.xml
@@ -154,10 +154,12 @@
<answer>feature_resources|plan_feature_workflows|no_apply_execute</answer>
<expected_answer>
The model should first use tools/list, resources/templates/list, and
resources/read to discover the feature resources for encrypt,
- broadcast, readwrite-splitting, shadow, and sharding instead of guessing
tool names. It should call the matching database_gateway_plan_*
- workflow tool only after required logical database, rule, table,
storage-unit, or algorithm inputs are known, then report plan_id,
- reviewable DistSQL artifacts, resources_to_read, and next_actions. It
must not call database_gateway_apply_workflow in execute mode,
- execute SQL updates, create physical metadata, or perform migration
while the request is still a read-only review.
+ broadcast, readwrite-splitting, shadow, and sharding instead of guessing
tool names. It should call the matching planning tool only
+ after required logical database, rule, table, storage-unit, or algorithm
inputs are known: database_gateway_plan_encrypt_rule,
+ database_gateway_plan_broadcast_rule,
database_gateway_plan_readwrite_splitting_rule,
database_gateway_plan_shadow_rule, or
+ database_gateway_plan_sharding_table_rule. The model should then report
plan_id, reviewable DistSQL artifacts, resources_to_read,
+ and next_actions. It must not call database_gateway_apply_workflow in
execute mode, execute SQL updates, create physical metadata,
+ or perform migration while the request is still a read-only review.
</expected_answer>
<verification>
<step>Confirm the answer discovers feature resources and tool schemas
before selecting a planning workflow.</step>
@@ -174,11 +176,12 @@
<answer>completion_or_plugin_resources|bounded_candidates|return_next_actions</answer>
<expected_answer>
The model should use completion/complete or resources/read against the
nearest shardingsphere:// plugin resource before choosing
- uncertain algorithm fields: mask algorithms, readwrite-splitting
load-balance algorithms, shadow algorithm plugins, sharding algorithm
- plugins, and sharding key-generate algorithm plugins. It should treat
returned values as bounded candidates, keep context arguments such
- as database or resource reference explicit, and avoid exposing secret
property values as completion suggestions. If completion has no safe
- candidate, it should use the recovery message, resources_to_read, and
next_actions from the planning response instead of inventing an
- algorithm or applying a workflow.
+ uncertain algorithm fields: shardingsphere://features/mask/algorithms,
+
shardingsphere://features/readwrite-splitting/load-balance-algorithm-plugins,
shardingsphere://features/shadow/algorithm-plugins,
+ shardingsphere://features/sharding/algorithm-plugins, and
shardingsphere://features/sharding/key-generate-algorithm-plugins.
+ It should treat returned values as bounded candidates, keep context
arguments such as database or resource reference explicit, and avoid
+ exposing secret property values as completion suggestions. If completion
has no safe candidate, it should use the recovery message,
+ resources_to_read, and next_actions from the planning response instead
of inventing an algorithm or applying a workflow.
</expected_answer>
<verification>
<step>Confirm the answer uses completion/complete or resources/read
plugin resources before selecting uncertain algorithm names.</step>
