This is an automated email from the ASF dual-hosted git repository.
terrymanu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 2061ecba9a1 Consolidate E2E workflows and stabilize LLM tests (#38777)
2061ecba9a1 is described below
commit 2061ecba9a170fb2a64a115124c6ef6b0553be2d
Author: Liang Zhang <[email protected]>
AuthorDate: Tue Jun 2 20:51:53 2026 +0800
Consolidate E2E workflows and stabilize LLM tests (#38777)
- Add one MCP E2E workflow for full MySQL runtime validation
- Keep JDK 21 subchain CI focused on MCP unit tests
- Rename MCP build workflow to release validation
- Remove obsolete standalone LLM E2E workflow entry points
- Update release and MCP E2E documentation
- Stabilize LLM usability scoring and recovery scenarios
---
.github/workflows/jdk21-subchain-ci.yml | 149 ++-------------------
.../{mcp-llm-usability-e2e.yml => mcp-e2e.yml} | 44 +++---
.github/workflows/mcp-llm-e2e.yml | 107 ---------------
.../workflows/{mcp-build.yml => mcp-release.yml} | 12 +-
.../content/involved/release/shardingsphere.cn.md | 4 +-
.../content/involved/release/shardingsphere.en.md | 4 +-
.../content/test-manual/mcp-e2e-test/_index.cn.md | 28 ++--
.../content/test-manual/mcp-e2e-test/_index.en.md | 28 ++--
.../assessment/LLMUsabilityMetricCalculator.java | 9 +-
.../LLMUsabilityMetricCalculatorTest.java | 20 ++-
.../scenario/LLMUsabilityScenarioCatalog.java | 18 +--
.../scenario/LLMUsabilityScenarioCatalogTest.java | 9 +-
12 files changed, 110 insertions(+), 322 deletions(-)
diff --git a/.github/workflows/jdk21-subchain-ci.yml
b/.github/workflows/jdk21-subchain-ci.yml
index ff027793a1a..4e12ad1a33c 100644
--- a/.github/workflows/jdk21-subchain-ci.yml
+++ b/.github/workflows/jdk21-subchain-ci.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-name: JDK 21 Subchain - E2E
+name: JDK 21 Subchain - CI
on:
pull_request:
@@ -24,15 +24,10 @@ on:
- '.github/workflows/jdk21-subchain-ci.yml'
- 'pom.xml'
- 'mcp/**'
- - 'distribution/pom.xml'
- - 'distribution/mcp/**'
- - 'test/e2e/pom.xml'
- - 'test/e2e/mcp/**'
- - '.github/workflows/mcp-build.yml'
workflow_dispatch:
concurrency:
- group: jdk21-subchain-e2e-${{ github.workflow }}-${{ github.ref }}
+ group: jdk21-subchain-ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
@@ -42,148 +37,24 @@ env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true
jobs:
- global-environment:
- name: Import Global Environment
- uses: ./.github/workflows/required-reusable.yml
-
- detect-changed-files:
- name: Detect Changed Files
+ mcp-unit-tests:
+ name: MCP - Unit Tests
if: github.repository == 'apache/shardingsphere'
runs-on: ubuntu-latest
- timeout-minutes: 5
- steps:
- - uses: actions/[email protected]
- - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d
- id: filter
- with:
- token: ${{ github.token }}
- filters: |
- mcp_http_contract:
- - '.github/workflows/jdk21-subchain-ci.yml'
- - 'pom.xml'
- - 'mcp/**'
- - 'test/e2e/pom.xml'
- - 'test/e2e/mcp/**'
- mcp_mysql:
- - '.github/workflows/jdk21-subchain-ci.yml'
- - 'pom.xml'
- - 'mcp/**'
- - 'test/e2e/pom.xml'
- - 'test/e2e/mcp/**'
- mcp_distribution:
- - '.github/workflows/jdk21-subchain-ci.yml'
- - '.github/workflows/mcp-build.yml'
- - 'pom.xml'
- - 'mcp/**'
- - 'distribution/pom.xml'
- - 'distribution/mcp/**'
- - 'test/e2e/pom.xml'
- - 'test/e2e/mcp/**'
- - name: Logs
- run: |
- echo "changed-categories=${{ steps.filter.outputs.changes }}"
- echo "mcp-http-contract=${{ steps.filter.outputs.mcp_http_contract
}}"
- echo "mcp-mysql=${{ steps.filter.outputs.mcp_mysql }}"
- echo "mcp-distribution=${{ steps.filter.outputs.mcp_distribution }}"
- outputs:
- mcp_http_contract: ${{ steps.filter.outputs.mcp_http_contract }}
- mcp_mysql: ${{ steps.filter.outputs.mcp_mysql }}
- mcp_distribution: ${{ steps.filter.outputs.mcp_distribution }}
-
- mcp-http-contract-e2e:
- name: MCP E2E - HTTP Contract
- if: github.repository == 'apache/shardingsphere' && (github.event_name ==
'workflow_dispatch' || needs.detect-changed-files.outputs.mcp_http_contract ==
'true')
- needs: [ global-environment, detect-changed-files ]
- runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/[email protected]
- uses: ./.github/workflows/resources/actions/setup-build-environment
with:
java-version: '21'
- cache-prefix: ${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}
cache-suffix: 'mcp'
- enable-docker-setup: 'true'
- - name: Run MCP HTTP Contract E2E
- run: ./mvnw -pl mcp,test/e2e/mcp -am install -DskipITs
-Dspotless.skip=true -Dsurefire.failIfNoSpecifiedTests=false
-Dmcp.e2e.contract.enabled=true -B -ntp
-
- mcp-mysql-e2e:
- name: MCP E2E - MySQL HTTP and STDIO
- if: github.repository == 'apache/shardingsphere' && (github.event_name ==
'workflow_dispatch' || needs.detect-changed-files.outputs.mcp_mysql == 'true')
- needs: [ global-environment, detect-changed-files ]
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/[email protected]
- - uses: ./.github/workflows/resources/actions/setup-build-environment
- with:
- java-version: '21'
- cache-prefix: ${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}
- cache-suffix: 'mcp'
- enable-docker-setup: 'true'
- - name: Install MCP E2E Test Dependencies
- run: ./mvnw -pl test/e2e/mcp -am install -DskipTests -DskipITs
-Dspotless.skip=true -B -ntp
- - name: Run MCP MySQL HTTP and STDIO E2E
- run: >
- ./mvnw -pl test/e2e/mcp test
- -DskipITs -Dspotless.skip=true
-
-Dtest=ProductionMySQLRuntimeE2ETest,HttpProductionProxyEncryptWorkflowE2ETest,HttpProductionProxyMaskWorkflowE2ETest
- -Dsurefire.failIfNoSpecifiedTests=true
- -Dmcp.e2e.production.mysql.enabled=true
- -Dmcp.e2e.production.stdio.enabled=true
- -B -ntp
-
- mcp-distribution-e2e:
- name: MCP E2E - Distribution
- if: github.repository == 'apache/shardingsphere' && (github.event_name ==
'workflow_dispatch' || needs.detect-changed-files.outputs.mcp_distribution ==
'true')
- needs: [ global-environment, detect-changed-files ]
- runs-on: ubuntu-latest
- timeout-minutes: 60
- steps:
- - uses: actions/[email protected]
- - uses: ./.github/workflows/resources/actions/setup-build-environment
- with:
- java-version: '21'
- cache-prefix: ${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}
- cache-suffix: 'mcp'
- enable-docker-setup: 'true'
- - name: Test MCP Registry Metadata Command
- run: ./mvnw -pl mcp/registry -am -DskipITs -Dspotless.skip=true
-Dtest=MCPRegistryMetadataCommandTest -Dsurefire.failIfNoSpecifiedTests=false
install -B -ntp
- - name: Package MCP Distribution
- run: ./mvnw -pl distribution/mcp -am -DskipTests package -B -ntp
- - name: Build MCP Docker Image
- run: docker build -f distribution/mcp/Dockerfile -t
shardingsphere-mcp-ci:local distribution/mcp/target
- - name: Validate MCP Registry Metadata
- shell: bash
- run: |
- set -euo pipefail
- DISTRIBUTION_HOME=$(find distribution/mcp/target -maxdepth 1 -type d
-name 'apache-shardingsphere-mcp-*' | head -n 1)
- if [[ -z "${DISTRIBUTION_HOME}" ]]; then
- echo "Packaged MCP distribution was not found." 1>&2
- exit 1
- fi
- test -f "${DISTRIBUTION_HOME}/conf/mcp-http.yaml"
- test -f "${DISTRIBUTION_HOME}/conf/mcp-stdio.yaml"
- ./mvnw -pl mcp/registry -DskipTests -DskipITs -Dspotless.skip=true \
-
-Dexec.mainClass=org.apache.shardingsphere.mcp.registry.MCPRegistryMetadataCommand
\
- -Dexec.args="--validate-only --allow-snapshot --dockerfile-path
distribution/mcp/Dockerfile" \
- compile exec:java -B -ntp
- - name: Install MCP E2E Test Dependencies
- run: ./mvnw -pl test/e2e/mcp -am install -DskipTests -DskipITs
-Dspotless.skip=true -B -ntp
- - name: Run MCP Distribution E2E
+ - name: Run MCP Unit Tests
shell: bash
run: |
set -euo pipefail
- DISTRIBUTION_HOME=$(find distribution/mcp/target -maxdepth 1 -type d
-name 'apache-shardingsphere-mcp-*' | head -n 1)
- if [[ -z "${DISTRIBUTION_HOME}" ]]; then
- echo "Packaged MCP distribution was not found." 1>&2
- exit 1
- fi
- DISTRIBUTION_HOME=$(cd "${DISTRIBUTION_HOME}" && pwd)
- ./mvnw -pl test/e2e/mcp test -DskipITs -Dspotless.skip=true \
- -Dtest=PackagedDistributionE2ETest \
- -Dsurefire.failIfNoSpecifiedTests=true \
- -Dmcp.e2e.distribution.enabled=true \
- -Dmcp.e2e.container.image=shardingsphere-mcp-ci:local \
- -Dmcp.distribution.home="${DISTRIBUTION_HOME}" \
+
MCP_MODULES=mcp/api,mcp/support,mcp/core,mcp/features/encrypt,mcp/features/mask
+ MCP_MODULES="${MCP_MODULES},mcp/bootstrap,mcp/registry"
+ ./mvnw -pl "${MCP_MODULES}" -am install \
+ -DskipITs -Dspotless.skip=true \
+ -Dsurefire.failIfNoSpecifiedTests=false \
-B -ntp
diff --git a/.github/workflows/mcp-llm-usability-e2e.yml
b/.github/workflows/mcp-e2e.yml
similarity index 72%
rename from .github/workflows/mcp-llm-usability-e2e.yml
rename to .github/workflows/mcp-e2e.yml
index 88222c6cc94..e592908f177 100644
--- a/.github/workflows/mcp-llm-usability-e2e.yml
+++ b/.github/workflows/mcp-e2e.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-name: MCP - LLM Usability E2E
+name: MCP - E2E
on:
pull_request:
@@ -26,7 +26,8 @@ on:
- synchronize
- ready_for_review
paths:
- - '.github/workflows/mcp-llm-usability-e2e.yml'
+ - '.github/workflows/mcp-e2e.yml'
+ - 'pom.xml'
- 'mcp/**'
- 'test/e2e/pom.xml'
- 'test/e2e/mcp/**'
@@ -35,7 +36,7 @@ on:
workflow_dispatch:
concurrency:
- group: mcp-llm-usability-e2e-${{ github.workflow }}-${{ github.ref }}
+ group: mcp-e2e-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
@@ -50,14 +51,9 @@ env:
MCP_LLM_BASE_SERVER_IMAGE_DIGEST:
sha256:988d2695631987e28a29d98970aaf0e979e23b843a26824abb790ac4245d1d57
jobs:
- global-environment:
- name: Import Global Environment
- uses: ./.github/workflows/required-reusable.yml
-
- mcp-llm-usability-e2e:
- name: MCP - LLM Usability E2E
+ mcp-mysql-runtime-e2e:
+ name: MCP - MySQL Runtime E2E
if: github.repository == 'apache/shardingsphere'
- needs: global-environment
runs-on: ubuntu-latest
timeout-minutes: 60
env:
@@ -67,8 +63,7 @@ jobs:
- uses: ./.github/workflows/resources/actions/setup-build-environment
with:
java-version: '21'
- cache-prefix: ${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}
- cache-suffix: 'mcp-llm-usability-e2e'
+ cache-suffix: 'mcp-e2e'
cache-save-enabled: 'false'
enable-docker-setup: 'true'
- name: Set Up Docker Buildx
@@ -92,15 +87,28 @@ jobs:
cache-to: type=gha,mode=max,scope=mcp-llm-runtime,ignore-error=true
- name: Build MCP E2E Test Dependencies
run: ./mvnw -pl test/e2e/mcp -am install -DskipTests -DskipITs
-Dspotless.skip=true -B -ntp
- - name: Verify MCP LLM Usability Selector
- run: test -f
test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/LLMUsabilitySuiteE2ETest.java
- - name: Run MCP LLM Usability Suite
- run: ./mvnw -pl test/e2e/mcp -Pllm-e2e test -DskipITs
-Dspotless.skip=true -Dtest=LLMUsabilitySuiteE2ETest
-Dsurefire.failIfNoSpecifiedTests=true -B -ntp
- - name: Upload MCP LLM Artifacts
+ - name: Run MCP MySQL Runtime E2E
+ shell: bash
+ run: |
+ set -euo pipefail
+
MCP_E2E_TESTS=HttpTransportContractE2ETest,HttpTransportProtocolContractE2ETest,HttpTransportBaselineContractE2ETest
+ MCP_E2E_TESTS="${MCP_E2E_TESTS},ProductionMySQLRuntimeE2ETest"
+
MCP_E2E_TESTS="${MCP_E2E_TESTS},HttpProductionProxyEncryptWorkflowE2ETest"
+
MCP_E2E_TESTS="${MCP_E2E_TESTS},HttpProductionProxyMaskWorkflowE2ETest,LLMUsabilitySuiteE2ETest"
+ ./mvnw -pl test/e2e/mcp test -DskipITs -Dspotless.skip=true \
+ -Dtest="${MCP_E2E_TESTS}" \
+ -Dsurefire.failIfNoSpecifiedTests=true \
+ -Dmcp.e2e.contract.enabled=true \
+ -Dmcp.e2e.production.mysql.enabled=true \
+ -Dmcp.e2e.production.stdio.enabled=true \
+ -Dmcp.e2e.llm.enabled=true \
+ -Dmcp.e2e.llm.excludedGroups= \
+ -B -ntp
+ - name: Upload MCP E2E Artifacts
if: always()
uses: actions/upload-artifact@v6
with:
- name: mcp-llm-usability-e2e-${{ github.run_id }}-${{
github.run_attempt }}
+ name: mcp-e2e-${{ github.run_id }}-${{ github.run_attempt }}
path: |
test/e2e/mcp/target/llm-e2e
test/e2e/mcp/target/surefire-reports
diff --git a/.github/workflows/mcp-llm-e2e.yml
b/.github/workflows/mcp-llm-e2e.yml
deleted file mode 100644
index 21b9193c21a..00000000000
--- a/.github/workflows/mcp-llm-e2e.yml
+++ /dev/null
@@ -1,107 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-name: MCP - LLM E2E
-
-on:
- pull_request:
- branches: [ master ]
- types:
- - opened
- - reopened
- - synchronize
- - ready_for_review
- paths:
- - '.github/workflows/mcp-llm-e2e.yml'
- - 'mcp/**'
- - 'test/e2e/pom.xml'
- - 'test/e2e/mcp/**'
- schedule:
- - cron: '0 17 * * 1-5'
- workflow_dispatch:
-
-concurrency:
- group: mcp-llm-e2e-${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-permissions:
- contents: read
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.http.retryHandler.class=standard
-Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true
- MCP_LLM_READY_TIMEOUT_SECONDS: '900'
- MCP_LLM_REQUEST_TIMEOUT_SECONDS: '300'
- MCP_LLM_ARTIFACT_ROOT: test/e2e/mcp/target/llm-e2e
- MCP_LLM_SERVER_IMAGE: apache/shardingsphere-mcp-llm-runtime:local
- MCP_LLM_BASE_SERVER_IMAGE_DIGEST:
sha256:988d2695631987e28a29d98970aaf0e979e23b843a26824abb790ac4245d1d57
-
-jobs:
- global-environment:
- name: Import Global Environment
- uses: ./.github/workflows/required-reusable.yml
-
- mcp-llm-e2e:
- name: MCP - LLM E2E
- if: github.repository == 'apache/shardingsphere'
- needs: global-environment
- runs-on: ubuntu-latest
- timeout-minutes: 60
- env:
- MCP_LLM_RUN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}
- steps:
- - uses: actions/[email protected]
- - uses: ./.github/workflows/resources/actions/setup-build-environment
- with:
- java-version: '21'
- cache-prefix: ${{
needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}
- cache-suffix: 'mcp-llm-e2e'
- cache-save-enabled: 'false'
- enable-docker-setup: 'true'
- - name: Set Up Docker Buildx
- uses: docker/setup-buildx-action@v3
- - name: Show Docker Buildx Version
- run: docker buildx version
- - name: Check Docker Environment
- run: |
- docker version
- docker system df
- - name: Build MCP LLM Runtime Image
- uses: docker/build-push-action@v6
- with:
- context: test/e2e/mcp/src/test/resources/docker/llm-runtime
- file: test/e2e/mcp/src/test/resources/docker/llm-runtime/Dockerfile
- build-args: |
- BASE_IMAGE=ghcr.io/ggml-org/llama.cpp@${{
env.MCP_LLM_BASE_SERVER_IMAGE_DIGEST }}
- tags: ${{ env.MCP_LLM_SERVER_IMAGE }}
- load: true
- cache-from: type=gha,scope=mcp-llm-runtime
- cache-to: type=gha,mode=max,scope=mcp-llm-runtime,ignore-error=true
- - name: Build MCP E2E Test Dependencies
- run: ./mvnw -pl test/e2e/mcp -am install -DskipTests -DskipITs
-Dspotless.skip=true -B -ntp
- - name: Verify MCP LLM Smoke Selector
- run: test -f
test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/smoke/LLMSmokeE2ETest.java
- - name: Run MCP LLM Smoke
- run: ./mvnw -pl test/e2e/mcp -Pllm-e2e test -DskipITs
-Dspotless.skip=true -Dtest=LLMSmokeE2ETest
-Dsurefire.failIfNoSpecifiedTests=true -B -ntp
- - name: Upload MCP LLM Artifacts
- if: always()
- uses: actions/upload-artifact@v6
- with:
- name: mcp-llm-e2e-${{ github.run_id }}-${{ github.run_attempt }}
- path: |
- test/e2e/mcp/target/llm-e2e
- test/e2e/mcp/target/surefire-reports
- if-no-files-found: warn
diff --git a/.github/workflows/mcp-build.yml b/.github/workflows/mcp-release.yml
similarity index 96%
rename from .github/workflows/mcp-build.yml
rename to .github/workflows/mcp-release.yml
index b953630ad1d..e2a2c07c1ce 100644
--- a/.github/workflows/mcp-build.yml
+++ b/.github/workflows/mcp-release.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-name: MCP - Build
+name: MCP - Release
on:
release:
@@ -28,7 +28,7 @@ on:
type: string
concurrency:
- group: mcp-build-${{ github.event_name }}-${{ github.event_name == 'release'
&& github.event.release.tag_name || inputs.version }}
+ group: mcp-release-${{ github.event_name }}-${{ github.event_name ==
'release' && github.event.release.tag_name || inputs.version }}
cancel-in-progress: true
permissions:
@@ -47,8 +47,8 @@ jobs:
name: Import Global Environment
uses: ./.github/workflows/required-reusable.yml
- mcp-build:
- name: MCP - Build
+ mcp-release:
+ name: MCP - Release
if: github.repository == 'apache/shardingsphere'
needs: global-environment
runs-on: ubuntu-latest
@@ -85,9 +85,9 @@ jobs:
echo "version=${VERSION}" >> "${GITHUB_OUTPUT}"
echo "identifier=${{ env.MCP_IMAGE }}:${VERSION}" >>
"${GITHUB_OUTPUT}"
echo "tags=${TAGS}" >> "${GITHUB_OUTPUT}"
- - name: Run and Install JDK 21 Subchain Tests
+ - name: Run and Install MCP JDK 21 Subchain Tests
run: ./mvnw -pl mcp,test/e2e/mcp -am install -DskipITs
-Dspotless.skip=true -Dsurefire.failIfNoSpecifiedTests=false -B -ntp
- - name: Package JDK 21 Subchain Distribution
+ - name: Package MCP Distribution
run: ./mvnw -pl distribution/mcp -am -DskipTests package -B -ntp
- name: Build Local MCP Docker Image
run: docker build -f distribution/mcp/Dockerfile -t
shardingsphere-mcp-release:local distribution/mcp/target
diff --git a/docs/community/content/involved/release/shardingsphere.cn.md
b/docs/community/content/involved/release/shardingsphere.cn.md
index 4bcefce45c4..9ed64cef53d 100644
--- a/docs/community/content/involved/release/shardingsphere.cn.md
+++ b/docs/community/content/involved/release/shardingsphere.cn.md
@@ -605,7 +605,7 @@ docker logout
3.8 ShardingSphere MCP 发布
-ShardingSphere MCP 通过仓库中的
[`.github/workflows/mcp-build.yml`](https://github.com/apache/shardingsphere/blob/master/.github/workflows/mcp-build.yml)
workflow 发布。
+ShardingSphere MCP 通过仓库中的
[`.github/workflows/mcp-release.yml`](https://github.com/apache/shardingsphere/blob/master/.github/workflows/mcp-release.yml)
workflow 发布。
该 workflow 会构建 MCP 发行包、把
`ghcr.io/apache/shardingsphere-mcp:${RELEASE.VERSION}` 推送到 GHCR，
稳定版额外更新 `latest`，然后通过 GitHub OIDC 把 `mcp/server.json` 发布到官方 MCP Registry。
@@ -617,7 +617,7 @@ ShardingSphere MCP 通过仓库中的
[`.github/workflows/mcp-build.yml`](https:
GitHub release 发布后：
-- 等待 `MCP - Build` workflow 成功完成
+- 等待 `MCP - Release` workflow 成功完成
- 确认 [GitHub
Packages](https://github.com/apache/shardingsphere/pkgs/container/shardingsphere-mcp)
中存在 `ghcr.io/apache/shardingsphere-mcp:${RELEASE.VERSION}`
- 通过下面的命令确认官方 MCP Registry 已返回该 server 的 metadata：
diff --git a/docs/community/content/involved/release/shardingsphere.en.md
b/docs/community/content/involved/release/shardingsphere.en.md
index f7442129528..836fbb7ed13 100644
--- a/docs/community/content/involved/release/shardingsphere.en.md
+++ b/docs/community/content/involved/release/shardingsphere.en.md
@@ -610,7 +610,7 @@ docker logout
3.8 ShardingSphere MCP publication
-ShardingSphere MCP is published through the repository workflow
[`.github/workflows/mcp-build.yml`](https://github.com/apache/shardingsphere/blob/master/.github/workflows/mcp-build.yml).
+ShardingSphere MCP is published through the repository workflow
[`.github/workflows/mcp-release.yml`](https://github.com/apache/shardingsphere/blob/master/.github/workflows/mcp-release.yml).
The workflow builds the MCP distribution, pushes
`ghcr.io/apache/shardingsphere-mcp:${RELEASE.VERSION}` to GHCR,
pushes `latest` for stable releases, and then publishes `mcp/server.json` to
the official MCP Registry through GitHub OIDC.
@@ -622,7 +622,7 @@ Edit release version and release notes, select `Set as the
latest release`, clic
After the GitHub release is published:
-- wait for the `MCP - Build` workflow to finish successfully
+- wait for the `MCP - Release` workflow to finish successfully
- confirm [GitHub
Packages](https://github.com/apache/shardingsphere/pkgs/container/shardingsphere-mcp)
contains `ghcr.io/apache/shardingsphere-mcp:${RELEASE.VERSION}`
- confirm the official MCP Registry returns the published metadata:
diff --git a/docs/document/content/test-manual/mcp-e2e-test/_index.cn.md
b/docs/document/content/test-manual/mcp-e2e-test/_index.cn.md
index cd7f5397003..37bfda3795f 100644
--- a/docs/document/content/test-manual/mcp-e2e-test/_index.cn.md
+++ b/docs/document/content/test-manual/mcp-e2e-test/_index.cn.md
@@ -5,7 +5,7 @@ weight = 5
chapter = true
+++
-本章说明 ShardingSphere-MCP 的端到端契约验证和 LLM smoke 验证。
+本章说明 ShardingSphere-MCP 的端到端契约验证和 LLM usability 验证。
## 范围
@@ -16,7 +16,7 @@ chapter = true
- STDIO runtime。
- MCP baseline contract。
- Tool/resource/prompt/completion discovery。
-- 真实模型驱动的 MCP smoke。
+- 真实模型驱动的 MCP usability。
- Encrypt 和 Mask workflow 可用性验证。
## 本地准备
@@ -48,12 +48,21 @@ sh
test/e2e/mcp/src/test/resources/docker/llm-runtime/build-local.sh --dry-run
sh test/e2e/mcp/src/test/resources/docker/llm-runtime/build-local.sh
```
-## 运行 LLM Smoke
+## 运行 MCP Runtime E2E
```bash
-./mvnw -pl test/e2e/mcp -Pllm-e2e test -DskipITs -Dspotless.skip=true \
- -Dtest=LLMSmokeE2ETest \
- -Dsurefire.failIfNoSpecifiedTests=true
+MCP_E2E_TESTS=HttpTransportContractE2ETest,HttpTransportProtocolContractE2ETest,HttpTransportBaselineContractE2ETest
+MCP_E2E_TESTS="${MCP_E2E_TESTS},ProductionMySQLRuntimeE2ETest"
+MCP_E2E_TESTS="${MCP_E2E_TESTS},HttpProductionProxyEncryptWorkflowE2ETest"
+MCP_E2E_TESTS="${MCP_E2E_TESTS},HttpProductionProxyMaskWorkflowE2ETest,LLMUsabilitySuiteE2ETest"
+./mvnw -pl test/e2e/mcp test -DskipITs -Dspotless.skip=true \
+ -Dtest="${MCP_E2E_TESTS}" \
+ -Dsurefire.failIfNoSpecifiedTests=true \
+ -Dmcp.e2e.contract.enabled=true \
+ -Dmcp.e2e.production.mysql.enabled=true \
+ -Dmcp.e2e.production.stdio.enabled=true \
+ -Dmcp.e2e.llm.enabled=true \
+ -Dmcp.e2e.llm.excludedGroups=
```
## 运行 LLM Usability Suite
@@ -70,7 +79,7 @@ sh
test/e2e/mcp/src/test/resources/docker/llm-runtime/build-local.sh
```bash
./mvnw -pl test/e2e/mcp -Pllm-e2e test -DskipITs -Dspotless.skip=true \
- -Dtest=LLMSmokeE2ETest \
+ -Dtest=LLMUsabilitySuiteE2ETest \
-Dmcp.llm.runtime-mode=external-debug \
-Dmcp.llm.base-url=http://127.0.0.1:8080/v1 \
-Dsurefire.failIfNoSpecifiedTests=true
@@ -88,8 +97,7 @@ test/e2e/mcp/target/llm-e2e/
GitHub Actions 入口：
-- `.github/workflows/mcp-llm-e2e.yml`
-- `.github/workflows/mcp-llm-usability-e2e.yml`
+- `.github/workflows/mcp-e2e.yml`
-这两条检查用于提供额外可见性，不应单独配置成 branch protection 或 ruleset 的 required check。
+这条 workflow 是 MCP runtime E2E 的必跑入口。
如果超大 PR 因 path filter 限制漏触发，可以使用 `workflow_dispatch` 手动补充 evidence。
diff --git a/docs/document/content/test-manual/mcp-e2e-test/_index.en.md
b/docs/document/content/test-manual/mcp-e2e-test/_index.en.md
index 695fdc81d27..9a73528901f 100644
--- a/docs/document/content/test-manual/mcp-e2e-test/_index.en.md
+++ b/docs/document/content/test-manual/mcp-e2e-test/_index.en.md
@@ -5,7 +5,7 @@ weight = 5
chapter = true
+++
-This chapter describes ShardingSphere-MCP end-to-end contract validation and
LLM smoke validation.
+This chapter describes ShardingSphere-MCP end-to-end contract validation and
LLM usability validation.
## Scope
@@ -16,7 +16,7 @@ This chapter describes ShardingSphere-MCP end-to-end contract
validation and LLM
- STDIO runtime.
- MCP baseline contract.
- Tool/resource/prompt/completion discovery.
-- Real-model MCP smoke.
+- Real-model MCP usability.
- Encrypt and Mask workflow usability validation.
## Local preparation
@@ -48,12 +48,21 @@ Build the local runtime image:
sh test/e2e/mcp/src/test/resources/docker/llm-runtime/build-local.sh
```
-## Run LLM Smoke
+## Run MCP Runtime E2E
```bash
-./mvnw -pl test/e2e/mcp -Pllm-e2e test -DskipITs -Dspotless.skip=true \
- -Dtest=LLMSmokeE2ETest \
- -Dsurefire.failIfNoSpecifiedTests=true
+MCP_E2E_TESTS=HttpTransportContractE2ETest,HttpTransportProtocolContractE2ETest,HttpTransportBaselineContractE2ETest
+MCP_E2E_TESTS="${MCP_E2E_TESTS},ProductionMySQLRuntimeE2ETest"
+MCP_E2E_TESTS="${MCP_E2E_TESTS},HttpProductionProxyEncryptWorkflowE2ETest"
+MCP_E2E_TESTS="${MCP_E2E_TESTS},HttpProductionProxyMaskWorkflowE2ETest,LLMUsabilitySuiteE2ETest"
+./mvnw -pl test/e2e/mcp test -DskipITs -Dspotless.skip=true \
+ -Dtest="${MCP_E2E_TESTS}" \
+ -Dsurefire.failIfNoSpecifiedTests=true \
+ -Dmcp.e2e.contract.enabled=true \
+ -Dmcp.e2e.production.mysql.enabled=true \
+ -Dmcp.e2e.production.stdio.enabled=true \
+ -Dmcp.e2e.llm.enabled=true \
+ -Dmcp.e2e.llm.excludedGroups=
```
## Run LLM Usability Suite
@@ -70,7 +79,7 @@ For local debugging only, connect to an already running
OpenAI-compatible endpoi
```bash
./mvnw -pl test/e2e/mcp -Pllm-e2e test -DskipITs -Dspotless.skip=true \
- -Dtest=LLMSmokeE2ETest \
+ -Dtest=LLMUsabilitySuiteE2ETest \
-Dmcp.llm.runtime-mode=external-debug \
-Dmcp.llm.base-url=http://127.0.0.1:8080/v1 \
-Dsurefire.failIfNoSpecifiedTests=true
@@ -88,8 +97,7 @@ test/e2e/mcp/target/llm-e2e/
GitHub Actions entry points:
-- `.github/workflows/mcp-llm-e2e.yml`
-- `.github/workflows/mcp-llm-usability-e2e.yml`
+- `.github/workflows/mcp-e2e.yml`
-These checks provide additional visibility and should not be configured as
required branch protection or ruleset checks by themselves.
+This workflow is the mandatory MCP runtime E2E entry point.
If a very large PR misses a path-filter match, use `workflow_dispatch` to add
manual evidence.
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculator.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculator.java
index f012d9f56b6..c9749ded6f5 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculator.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculator.java
@@ -408,7 +408,14 @@ public final class LLMUsabilityMetricCalculator {
private boolean isMachineAction(final Map<?, ?> action) {
String type = Objects.toString(action.get("type"), "");
- return "resource_read".equals(type) || "tool_call".equals(type) ||
"completion".equals(type);
+ if (!"resource_read".equals(type) && !"tool_call".equals(type) &&
!"completion".equals(type)) {
+ return false;
+ }
+ if (!"tool_call".equals(type) || !(action.get("arguments") instanceof
Map)) {
+ return true;
+ }
+ String executionMode = Objects.toString(((Map<?, ?>)
action.get("arguments")).get("execution_mode"), "");
+ return !"execute".equals(executionMode) &&
!"review-then-execute".equals(executionMode);
}
private boolean matchesAnyNextAction(final List<Map<?, ?>> actions, final
MCPInteractionTraceRecord current, final MCPInteractionTraceRecord next) {
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculatorTest.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculatorTest.java
index 69b51bf944a..6506a4711a7 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculatorTest.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/assessment/LLMUsabilityMetricCalculatorTest.java
@@ -24,6 +24,8 @@ import
org.apache.shardingsphere.test.e2e.mcp.llm.suite.usability.scenario.LLMUs
import
org.apache.shardingsphere.test.e2e.mcp.support.transport.MCPInteractionActionNames;
import
org.apache.shardingsphere.test.e2e.mcp.support.transport.MCPInteractionTraceRecord;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import java.util.List;
import java.util.Map;
@@ -69,13 +71,19 @@ class LLMUsabilityMetricCalculatorTest {
assertTrue(actual.isNextActionFollowed());
}
- @Test
- void assertEvaluateScenarioWithToolCallNextActionFollowed() {
+ @ParameterizedTest(name = "{0}")
+ @CsvSource({
+ "regular tool call guidance, database_gateway_execute_update, '',
database_gateway_execute_update",
+ "execute side-effect guidance, database_gateway_execute_update,
execute, database_gateway_execute_query",
+ "review-then-execute side-effect guidance,
database_gateway_apply_workflow, review-then-execute,
database_gateway_execute_query"
+ })
+ void assertEvaluateScenarioWithToolCallNextActionGuidance(final String
name, final String nextActionToolName, final String executionMode, final String
nextToolName) {
+ Map<String, Object> nextAction = executionMode.isEmpty()
+ ? Map.of("type", "tool_call", "tool_name", nextActionToolName)
+ : Map.of("type", "tool_call", "tool_name", nextActionToolName,
"arguments", Map.of("execution_mode", executionMode));
List<MCPInteractionTraceRecord> trace = List.of(
- createToolCall(1, "database_gateway_execute_update",
Map.of("next_actions", List.of(Map.of(
- "type", "tool_call",
- "tool_name", "database_gateway_execute_update")))),
- createToolCall(2, "database_gateway_execute_update",
Map.of()));
+ createToolCall(1, nextActionToolName, Map.of("next_actions",
List.of(nextAction))),
+ createToolCall(2, nextToolName, Map.of()));
LLMUsabilityScenarioResult actual = new
LLMUsabilityMetricCalculator().evaluateScenario(createScenario(),
createArtifactBundle(trace));
assertTrue(actual.isNextActionFollowed());
assertFalse(actual.isApprovalViolation());
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
index c03378490e5..a6533fde156 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalog.java
@@ -151,12 +151,11 @@ public final class LLMUsabilityScenarioCatalog {
List.of(LLMUsabilityScenario.NATURAL_TASK_TAG, "extended",
"recovery"),
new LLME2EScenario("extended-recovery-missing-database-" +
runtimeKind, SYSTEM_PROMPT,
"The user only remembers schema `" + schemaName + "`
and table `" + tableName + "`. Search metadata broadly with query `" + tableName
- + "` and object type `table` without setting
database or schema. If more than one database contains `" + tableName + "`,
choose logical database `"
- + databaseName + "` and do not use
`analytics_db`. Then verify `" + query + "`.",
+ + "` and object type `table` without setting
database or schema. Then verify `" + query + "`.",
createAnswer(databaseName, schemaName, tableName,
query, totalOrders),
List.of("database_gateway_search_metadata",
"database_gateway_execute_query"),
List.of("database_gateway_search_metadata",
"database_gateway_execute_query")),
- List.of("database_gateway_search_metadata"), List.of(), false,
true, "ambiguous"));
+ List.of("database_gateway_search_metadata"), List.of(), false,
false));
result.add(createScenario("extended-recovery-bad-resource-" +
runtimeKind, LLMUsabilityDimension.RECOVERY, runtimeKind,
List.of(LLMUsabilityScenario.PROTOCOL_CONTRACT_TAG,
"extended", "recovery", "resource"),
new LLME2EScenario("extended-recovery-bad-resource-" +
runtimeKind, SYSTEM_PROMPT,
@@ -165,18 +164,7 @@ public final class LLMUsabilityScenarioCatalog {
createAnswer(databaseName, schemaName, tableName,
query, totalOrders),
List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_execute_query"),
List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_execute_query")),
- List.of(MCPInteractionActionNames.READ_RESOURCE),
List.of(tableResourceUri), true, true, "not_found"));
- if ("mysql".equals(runtimeKind)) {
-
result.add(createScenario("extended-recovery-unsupported-sequence-" +
runtimeKind, LLMUsabilityDimension.RECOVERY, runtimeKind,
- List.of(LLMUsabilityScenario.NATURAL_TASK_TAG, "extended",
"recovery", "unsupported-resource"),
- new
LLME2EScenario("extended-recovery-unsupported-sequence-" + runtimeKind,
SYSTEM_PROMPT,
- "The user asks about sequence metadata near `" +
tableName + "` on this MySQL runtime. Recover safely if sequence resources are
unsupported, then verify `"
- + query + "`." + toolContext,
- createAnswer(databaseName, schemaName, tableName,
query, totalOrders),
- List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_execute_query"),
- List.of(MCPInteractionActionNames.READ_RESOURCE,
"database_gateway_execute_query")),
- List.of(MCPInteractionActionNames.READ_RESOURCE),
List.of(tableResourceUri), true, true, "not_found"));
- }
+ List.of(MCPInteractionActionNames.READ_RESOURCE),
List.of(tableResourceUri), true, true, "unknown_database"));
return result;
}
diff --git
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
index 5e176534175..6e6d328b2a8 100644
---
a/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
+++
b/test/e2e/mcp/src/test/java/org/apache/shardingsphere/test/e2e/mcp/llm/suite/usability/scenario/LLMUsabilityScenarioCatalogTest.java
@@ -59,14 +59,11 @@ class LLMUsabilityScenarioCatalogTest {
"SELECT COUNT(*) AS total_orders FROM orders", 2);
Map<String, LLMUsabilityScenario> actualScenarios =
actual.stream().collect(Collectors.toMap(LLMUsabilityScenario::getScenarioId,
each -> each));
assertThat(actualScenarios.keySet(),
hasItems("extended-prompt-completion-inspect-mysql",
"extended-runtime-status-mysql", "extended-recovery-missing-database-mysql",
- "extended-recovery-unsupported-sequence-mysql"));
+ "extended-recovery-bad-resource-mysql"));
assertThat(actualScenarios.get("extended-runtime-status-mysql").getExpectedResourceUris(),
is(List.of("shardingsphere://runtime")));
-
assertTrue(actualScenarios.get("extended-recovery-missing-database-mysql").isRecoveryExpected());
-
assertThat(actualScenarios.get("extended-recovery-missing-database-mysql").getExpectedRecoveryCategory(),
is("ambiguous"));
+
assertFalse(actualScenarios.get("extended-recovery-missing-database-mysql").isRecoveryExpected());
assertTrue(actualScenarios.get("extended-recovery-bad-resource-mysql").isRecoveryExpected());
-
assertThat(actualScenarios.get("extended-recovery-bad-resource-mysql").getExpectedRecoveryCategory(),
is("not_found"));
-
assertTrue(actualScenarios.get("extended-recovery-unsupported-sequence-mysql").isRecoveryExpected());
-
assertThat(actualScenarios.get("extended-recovery-unsupported-sequence-mysql").getExpectedRecoveryCategory(),
is("not_found"));
+
assertThat(actualScenarios.get("extended-recovery-bad-resource-mysql").getExpectedRecoveryCategory(),
is("unknown_database"));
assertThat(actualScenarios.get("extended-prompt-completion-inspect-mysql").getLlmScenario().getUserPrompt(),
containsString("Use the MCP prompt list"));
assertTrue(actualScenarios.get("extended-recovery-missing-database-mysql").isNaturalTask());
assertTrue(actualScenarios.get("extended-prompt-completion-inspect-mysql").isProtocolContract());
