This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 6396ac6e3c1 Add LLM usability scenario for natural MD5 mask rule
planning (#38983)
6396ac6e3c1 is described below
commit 6396ac6e3c128e3c38f2d23f6398637396d8e76c
Author: zhaojinchao <[email protected]>
AuthorDate: Fri Jul 3 00:09:42 2026 +0800
Add LLM usability scenario for natural MD5 mask rule planning (#38983)
* Add LLM usability scenario for natural MD5 mask rule planning
* Fix CI
---
.../suite/usability/scenario/LLMUsabilityScenarioCatalog.java | 11 +++++++++++
.../usability/scenario/LLMUsabilityScenarioCatalogTest.java | 9 ++++++++-
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
index 31e75c2d9ff..d0dafb5adbb 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
@@ -91,6 +91,17 @@ public final class LLMUsabilityScenarioCatalog {
createAnswer(databaseName, schemaName, tableName,
query, totalOrders),
workflowActions, workflowRequiredActions),
List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_plan_mask_rule"), List.of(), false, false));
+ result.add(createScenario("natural-mask-rule-md5-" + runtimeKind,
LLMUsabilityDimension.TOOL, runtimeKind,
+ List.of(LLMUsabilityScenario.NATURAL_TASK_TAG, "natural",
"workflow", "mask"),
+ new LLME2EScenario("natural-mask-rule-md5-" + runtimeKind,
SYSTEM_PROMPT,
+ "A user asks you to create a mask rule for logical
database `" + databaseName + "`, schema `" + schemaName + "`, table `" +
tableName
+ + "`, and column `status` with the MD5 mask
algorithm. Use the planning response plan_id for follow-up workflow calls, keep
execution manual-only, "
+ + "validate the workflow with the returned
plan_id, and finish by verifying `" + query + "`.",
+ createAnswer(databaseName, schemaName, tableName,
query, totalOrders),
+ List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_plan_mask_rule", "database_gateway_apply_workflow",
"database_gateway_validate_workflow",
+ "database_gateway_execute_query"),
+ List.of("database_gateway_plan_mask_rule",
"database_gateway_apply_workflow", "database_gateway_validate_workflow",
"database_gateway_execute_query")),
+ List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_plan_mask_rule"), List.of(), false, false));
result.add(createScenario("natural-table-resource-" + runtimeKind,
LLMUsabilityDimension.RESOURCE, runtimeKind,
List.of(LLMUsabilityScenario.NATURAL_TASK_TAG, "natural",
"resource"),
new LLME2EScenario("natural-table-resource-" + runtimeKind,
SYSTEM_PROMPT,
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
index 6e6d328b2a8..59a4b288401 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
@@ -38,7 +38,7 @@ class LLMUsabilityScenarioCatalogTest {
"SELECT COUNT(*) AS total_orders FROM orders", 2);
Map<String, LLMUsabilityScenario> actualScenarios =
actual.stream().collect(Collectors.toMap(LLMUsabilityScenario::getScenarioId,
each -> each));
assertThat(actualScenarios.keySet(),
hasItems("natural-metadata-lookup-mysql", "natural-read-only-sql-mysql",
"natural-side-effect-preview-mysql",
- "natural-workflow-manual-export-mysql",
"natural-table-resource-mysql"));
+ "natural-workflow-manual-export-mysql",
"natural-mask-rule-md5-mysql", "natural-table-resource-mysql"));
assertThat(actualScenarios.get("natural-side-effect-preview-mysql").getLlmScenario().getRequiredToolNames(),
is(List.of("database_gateway_execute_update",
"database_gateway_execute_query")));
assertThat(actualScenarios.get("natural-table-resource-mysql").getExpectedResourceUris(),
is(List.of("shardingsphere://databases/logic_db/schemas/logic_db/tables/orders")));
@@ -47,6 +47,13 @@ class LLMUsabilityScenarioCatalogTest {
assertThat(actualScenarios.get("natural-workflow-manual-export-mysql").getExpectedResourceUris(),
is(List.of()));
assertThat(actualScenarios.get("natural-workflow-manual-export-mysql").getLlmScenario().getUserPrompt(),
containsString("table `orders`, and column `status`"));
+
assertThat(actualScenarios.get("natural-mask-rule-md5-mysql").getLlmScenario().getRequiredToolNames(),
+ is(List.of("database_gateway_plan_mask_rule",
"database_gateway_apply_workflow", "database_gateway_validate_workflow",
"database_gateway_execute_query")));
+
assertThat(actualScenarios.get("natural-mask-rule-md5-mysql").getLlmScenario().getUserPrompt(),
containsString("create a mask rule"));
+
assertThat(actualScenarios.get("natural-mask-rule-md5-mysql").getLlmScenario().getUserPrompt(),
containsString("table `orders`"));
+
assertThat(actualScenarios.get("natural-mask-rule-md5-mysql").getLlmScenario().getUserPrompt(),
containsString("MD5 mask algorithm"));
+
assertThat(actualScenarios.get("natural-mask-rule-md5-mysql").getLlmScenario().getUserPrompt(),
containsString("manual-only"));
+
assertThat(actualScenarios.get("natural-mask-rule-md5-mysql").getLlmScenario().getUserPrompt(),
containsString("validate the workflow"));
assertTrue(actual.stream().allMatch(each ->
each.getTags().contains("natural")));
assertTrue(actual.stream().allMatch(LLMUsabilityScenario::isNaturalTask));
assertTrue(actual.stream().noneMatch(LLMUsabilityScenario::isProtocolContract));
