This is an automated email from the ASF dual-hosted git repository.
hsluoyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
The following commit(s) were added to refs/heads/master by this push:
new 57fd507 fix(authenticity): stop penalizing documented model aliases
and foreign vendor headers
57fd507 is described below
commit 57fd5078f98706b433cb2d7545e7c01e2c9c771c
Author: Yang Luo <[email protected]>
AuthorDate: Thu Sep 3 17:22:39 2026 +0800
fix(authenticity): stop penalizing documented model aliases and foreign
vendor headers
---
object/probe_case.go | 113 +++++++++++-----
object/probe_case_builtin.go | 45 ++-----
object/provider_probe_checks.go | 75 +++++++----
object/provider_probe_score.go | 2 +-
object/provider_probe_vendor.go | 221 +++++++++++++++++++++++++++++++
web/src/components/ProviderAuditCard.tsx | 9 +-
web/src/locales/en/data.json | 2 +
web/src/locales/zh/data.json | 13 +-
8 files changed, 381 insertions(+), 99 deletions(-)
diff --git a/object/probe_case.go b/object/probe_case.go
index fc2b569..ac79ae5 100644
--- a/object/probe_case.go
+++ b/object/probe_case.go
@@ -19,6 +19,8 @@
package object
import (
+ "crypto/sha256"
+ "encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -87,6 +89,11 @@ type ProbeCase struct {
// BuiltIn marks a case Gateway ships, which is what restoring the
defaults
// puts back. An edited built-in case stays built-in.
BuiltIn bool `xorm:"bool" json:"builtIn"`
+ // Shipped fingerprints the built-in case this row was written from. It
is
+ // what tells a case nobody has touched from one someone rewrote, which
is
+ // what lets a later release improve a shipped case without overwriting
+ // anyone's words.
+ Shipped string `xorm:"varchar(64)" json:"-"`
Question string `xorm:"varchar(1000)" json:"question"`
Method string `xorm:"varchar(2000)" json:"method"`
@@ -127,36 +134,44 @@ func GetProbeCases() ([]*ProbeCase, error) {
return cases, nil
}
-// markEditedProbeCases compares each built-in case against the one Gateway
-// ships. Only the words are compared: a case that was reweighted or turned off
-// is still the shipped question, asked the shipped way.
+// markEditedProbeCases compares each built-in case against the words it was
+// written with. Only the question is compared: a case that was reweighted or
+// turned off is still the shipped question, asked the shipped way.
func markEditedProbeCases(cases []*ProbeCase) {
- shipped := map[string]*ProbeCase{}
- for _, probeCase := range builtInProbeCases() {
- shipped[probeCase.Name] = probeCase
- }
-
for _, probeCase := range cases {
- original, ok := shipped[probeCase.Name]
- if !probeCase.BuiltIn || !ok {
- continue
- }
- probeCase.Edited = probeCase.DisplayName !=
original.DisplayName ||
- probeCase.Question != original.Question ||
- probeCase.Method != original.Method ||
- !sameProbeParams(probeCase.Params, original.Params)
+ probeCase.Edited = isProbeCaseEdited(probeCase)
}
}
-// sameProbeParams compares two parameter sets the way a reader would: what is
-// sent and what it is compared against, slices included.
-func sameProbeParams(left ProbeCaseParams, right ProbeCaseParams) bool {
- leftJson, leftErr := json.Marshal(left)
- rightJson, rightErr := json.Marshal(right)
- if leftErr != nil || rightErr != nil {
+func isProbeCaseEdited(probeCase *ProbeCase) bool {
+ if !probeCase.BuiltIn {
return false
}
- return string(leftJson) == string(rightJson)
+ // A row stored before cases were fingerprinted has only its timestamps
to
+ // go on, which are enough: a seeded case is written once, so one that
was
+ // never written again is one nobody has touched.
+ if probeCase.Shipped == "" {
+ return probeCase.UpdatedTime != probeCase.CreatedTime
+ }
+ return probeCaseFingerprint(probeCase) != probeCase.Shipped
+}
+
+// probeCaseFingerprint covers what a case asks and how it judges the answer,
+// which is what a reader would call the case itself.
+func probeCaseFingerprint(probeCase *ProbeCase) string {
+ params, err := json.Marshal(probeCase.Params)
+ if err != nil {
+ params = []byte("{}")
+ }
+ sum := sha256.Sum256([]byte(strings.Join([]string{
+ probeCase.DisplayName,
+ probeCase.Check,
+ probeCase.Protocol,
+ probeCase.Question,
+ probeCase.Method,
+ string(params),
+ }, "|")))
+ return hex.EncodeToString(sum[:])
}
func sortProbeCases(cases []*ProbeCase) {
@@ -255,6 +270,7 @@ func UpdateProbeCase(name string, probeCase *ProbeCase)
error {
probeCase.Name = name
probeCase.BuiltIn = stored.BuiltIn
+ probeCase.Shipped = stored.Shipped
probeCase.CreatedTime = stored.CreatedTime
probeCase.UpdatedTime = util.GetCurrentTime()
_, err = ormer.Engine.ID(name).AllCols().Update(probeCase)
@@ -310,9 +326,10 @@ func normalizeProbeCase(probeCase *ProbeCase) error {
return nil
}
-// InitProbeCases writes the shipped suite on first start. It only ever adds
the
-// cases missing by name, so an edited case survives a restart and a case added
-// in a later release still arrives.
+// InitProbeCases brings the stored suite up to what this release ships: cases
+// that are missing are added, a shipped case nobody has rewritten is rewritten
+// as it now ships, and one this release no longer ships is dropped. A case
+// someone wrote or edited is left exactly as they left it.
func InitProbeCases() {
if err := seedProbeCases(); err != nil {
beego.Error("probe cases could not be created:", err)
@@ -328,20 +345,52 @@ func seedProbeCases() error {
if err != nil {
return err
}
- known := map[string]bool{}
+ known := map[string]*ProbeCase{}
for _, probeCase := range stored {
- known[probeCase.Name] = true
+ known[probeCase.Name] = probeCase
}
+ shipping := map[string]bool{}
for _, probeCase := range builtInProbeCases() {
- if known[probeCase.Name] {
+ shipping[probeCase.Name] = true
+ probeCase.Shipped = probeCaseFingerprint(probeCase)
+
+ existing := known[probeCase.Name]
+ if existing == nil {
+ probeCase.CreatedTime = util.GetCurrentTime()
+ probeCase.UpdatedTime = probeCase.CreatedTime
+ if _, err := ormer.Engine.Insert(probeCase); err != nil
{
+ return err
+ }
+ continue
+ }
+ if existing.Edited || existing.Shipped == probeCase.Shipped {
continue
}
- probeCase.CreatedTime = util.GetCurrentTime()
- probeCase.UpdatedTime = probeCase.CreatedTime
- if _, err := ormer.Engine.Insert(probeCase); err != nil {
+ if err := refreshProbeCase(probeCase); err != nil {
+ return err
+ }
+ }
+
+ // A built-in case this release stopped shipping has nothing left to
run:
+ // the engine it named or the question it asked is gone.
+ for _, probeCase := range stored {
+ if !probeCase.BuiltIn || probeCase.Edited ||
shipping[probeCase.Name] {
+ continue
+ }
+ if err := DeleteProbeCase(probeCase.Name); err != nil {
return err
}
}
return nil
}
+
+// refreshProbeCase rewrites the question and leaves the settings: what a case
+// asks is Gateway's, what it is worth and whether it runs at all is not.
+func refreshProbeCase(probeCase *ProbeCase) error {
+ probeCase.UpdatedTime = util.GetCurrentTime()
+ _, err := ormer.Engine.ID(probeCase.Name).
+ Cols("display_name", "check", "protocol", "question", "method",
"params", "shipped", "updated_time").
+ Update(probeCase)
+ return err
+}
diff --git a/object/probe_case_builtin.go b/object/probe_case_builtin.go
index c5968ec..4890d23 100644
--- a/object/probe_case_builtin.go
+++ b/object/probe_case_builtin.go
@@ -49,9 +49,11 @@ func builtInProbeCases() []*ProbeCase {
BuiltIn: true,
Question: "Does the model that answers name itself
as the one that was asked for?",
Method: "The first request names a model. Every
completion API answers with the model that " +
- "ran, and a version or date suffix on the same
family counts as a match. A different " +
- "name is decisive; a matching one proves
little, since echoing a name back is the " +
- "easiest thing in the world to do.",
+ "ran, and a version or date suffix on the same
family counts as a match. Where the " +
+ "name asked for is one the vendor documents as
moving — deepseek-chat, anything " +
+ "ending in -latest — the answer only has to be
a model of that same vendor, which is " +
+ "what the vendor says it will be. Any other
mismatch is decisive; a matching name " +
+ "proves little, since echoing one back is the
easiest thing in the world to do.",
},
{
Name: "tools-nested-schema",
@@ -153,40 +155,21 @@ func builtInProbeCases() []*ProbeCase {
},
},
{
- Name: "vendor-headers-anthropic",
- DisplayName: "Vendor headers (Anthropic)",
+ Name: "vendor-headers",
+ DisplayName: "Vendor headers",
Check: ProbeVendor,
- Protocol: ProtocolAnthropic,
Enabled: true,
Weight: probeWeightVendor,
Sort: 60,
BuiltIn: true,
Question: "Do the response headers of the vendor's
own API come back?",
- Method: "A reseller in front of the real thing usually
passes some of these through; a " +
- "backend that never talked to the vendor has
none to pass. This case never reaches an " +
- "alert on its own: stripping headers is being
tidy, not necessarily dishonest.",
- Params: ProbeCaseParams{
- Headers:
probeVendorHeaders[ProtocolAnthropic],
- MinHeaders: 2,
- },
- },
- {
- Name: "vendor-headers-openai",
- DisplayName: "Vendor headers (OpenAI)",
- Check: ProbeVendor,
- Protocol: ProtocolOpenAi,
- Enabled: true,
- Weight: probeWeightVendor,
- Sort: 61,
- BuiltIn: true,
- Question: "Do the response headers of the vendor's
own API come back?",
- Method: "A reseller in front of the real thing usually
passes some of these through; a " +
- "backend that never talked to the vendor has
none to pass. This case never reaches an " +
- "alert on its own: stripping headers is being
tidy, not necessarily dishonest.",
- Params: ProbeCaseParams{
- Headers: probeVendorHeaders[ProtocolOpenAi],
- MinHeaders: 2,
- },
+ Method: "This is asked only of an endpoint that is a
vendor's own, against the headers " +
+ "that vendor documents: speaking an
OpenAI-compatible API is not a claim to be OpenAI. " +
+ "A reseller in front of the real thing usually
passes some through and a backend that " +
+ "never talked to the vendor has none to pass,
but stripping headers is also just being " +
+ "tidy, so this case never reaches an alert on
its own. Leave the header list empty to " +
+ "use the vendor's own, or name headers here to
ask for those instead.",
+ Params: ProbeCaseParams{MinHeaders: 2},
},
}
}
diff --git a/object/provider_probe_checks.go b/object/provider_probe_checks.go
index 3575b55..6b492e5 100644
--- a/object/provider_probe_checks.go
+++ b/object/provider_probe_checks.go
@@ -44,7 +44,6 @@ import (
// first request decides whether the upstream is usable at all; everything
after
// it is a question that would be meaningless if it were not.
func runProbeCases(provider *Provider, model string, probe *ProviderProbe,
cases []*ProbeCase) {
- upstream := ProviderProtocol(provider)
toolCases := probeCasesOf(cases, ProbeTools)
var primary *probeBody
@@ -77,7 +76,17 @@ func runProbeCases(provider *Provider, model string, probe
*ProviderProbe, cases
probe.addCheck(probeIdentityCheck(identityCase, model,
primary.Model))
}
for _, vendorCase := range probeCasesOf(cases, ProbeVendor) {
- found := vendorHeadersOf(vendorCase, upstream, headers)
+ wanted := vendorHeadersOf(vendorCase, provider)
+ if len(wanted) == 0 {
+ // Nothing to hold this endpoint to: the vendor it
belongs to is
+ // not one whose headers this build documents, or it
belongs to no
+ // vendor at all. That is a gap here, not a finding
about it.
+ check := checkOf(vendorCase, LlmAuditUnknown)
+ check.Facts = []string{probeVendorUndocumented}
+ probe.addCheck(check)
+ continue
+ }
+ found := headersPresent(wanted, headers)
probe.VendorHeaders = mergeStrings(probe.VendorHeaders, found)
probe.addCheck(probeVendorCheck(vendorCase, found))
}
@@ -329,9 +338,15 @@ func probeOpeningCall(provider *Provider, model string,
probe *ProviderProbe) (*
return &parsed, answer.header
}
+// probeIdentityAlias marks an answer that matched because the name asked for
is
+// one the vendor documents as moving, so that the report can say which of the
+// two things it found.
+const probeIdentityAlias = "alias"
+
// probeIdentityCheck compares the model name that came back with the one that
// was asked for. Matching proves little on its own — echoing a name back is
the
-// easiest thing in the world to do — but not matching is decisive.
+// easiest thing in the world to do — but not matching is decisive, except
where
+// the vendor documents the name asked for as an alias.
func probeIdentityCheck(probeCase *ProbeCase, asked string, answered string)
ProbeCheck {
check := checkOf(probeCase, LlmAuditUnknown)
if strings.TrimSpace(answered) == "" {
@@ -342,6 +357,17 @@ func probeIdentityCheck(probeCase *ProbeCase, asked
string, answered string) Pro
check.Level = LlmAuditOk
return check
}
+
+ // "deepseek-chat" is DeepSeek's own name for whichever chat model is
+ // current, and it answers with the name of the model that ran. A
different
+ // name there is the documented behaviour; what is left to check is
that it
+ // is still a model of the vendor whose alias was asked for.
+ if isProbeModelAlias(asked) && sameModelVendor(asked, answered) {
+ check.Level = LlmAuditOk
+ check.Facts = []string{answered, probeIdentityAlias}
+ return check
+ }
+
check.Level = LlmAuditAlert
return check
}
@@ -365,32 +391,27 @@ func sameModelName(asked string, answered string) bool {
return strings.HasPrefix(left, right) || strings.HasPrefix(right, left)
}
-// The response headers each vendor's own API sets. A reseller in front of the
-// real thing usually passes some of them through; a backend that never talked
-// to the vendor has none to pass. A case may name its own list instead.
-var probeVendorHeaders = map[string][]string{
- ProtocolAnthropic: {
- "Request-Id",
- "Anthropic-Organization-Id",
- "Anthropic-Ratelimit-Requests-Limit",
- "Anthropic-Ratelimit-Input-Tokens-Limit",
- "X-Should-Retry",
- },
- ProtocolOpenAi: {
- "X-Request-Id",
- "Openai-Organization",
- "Openai-Processing-Ms",
- "Openai-Version",
- "X-Ratelimit-Limit-Requests",
- },
-}
-
-func vendorHeadersOf(probeCase *ProbeCase, upstream string, header
http.Header) []string {
- wanted := probeCase.Params.Headers
- if len(wanted) == 0 {
- wanted = probeVendorHeaders[upstream]
+// probeVendorUndocumented is what the header case reports when there is no
+// documented list to hold the endpoint to.
+const probeVendorUndocumented = "undocumented"
+
+// vendorHeadersOf is the list this endpoint is held to: the one the case
names,
+// or failing that the response headers the vendor whose own host it is
+// documents. An endpoint that is not a vendor's own — a reseller, an
+// aggregator, a vendor this build carries no header list for — has no list,
and
+// the case is then not asked. Answering an OpenAI-compatible API is not a
claim
+// to be OpenAI, so OpenAI's headers are not owed by everything that speaks it.
+func vendorHeadersOf(probeCase *ProbeCase, provider *Provider) []string {
+ if len(probeCase.Params.Headers) > 0 {
+ return probeCase.Params.Headers
+ }
+ if vendor := probeVendorOfProvider(provider); vendor != nil {
+ return vendor.headers
}
+ return nil
+}
+func headersPresent(wanted []string, header http.Header) []string {
found := []string{}
for _, name := range wanted {
if header.Get(name) != "" {
diff --git a/object/provider_probe_score.go b/object/provider_probe_score.go
index 077fe51..9d25202 100644
--- a/object/provider_probe_score.go
+++ b/object/provider_probe_score.go
@@ -100,7 +100,7 @@ func probeScoreOf(probe *ProviderProbe) (float64, string) {
return 0, ProbeGradeUnknown
}
- score := math.Round(earned / possible * 1000) / 10
+ score := math.Round(earned/possible*1000) / 10
return score, probeGradeOf(score)
}
diff --git a/object/provider_probe_vendor.go b/object/provider_probe_vendor.go
new file mode 100644
index 0000000..7fabe6c
--- /dev/null
+++ b/object/provider_probe_vendor.go
@@ -0,0 +1,221 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// What each vendor documents about itself, which is the only thing a check is
+// entitled to hold it to. An endpoint or a model name that matches nothing
here
+// is not a finding: it is a question this build cannot ask.
+
+package object
+
+import (
+ "net/url"
+ "strings"
+)
+
+type probeVendor struct {
+ key string
+ // hosts are the vendor's own API hosts. A reseller answers on its own
host
+ // and so matches none of them, which is the point.
+ hosts []string
+ // models are the prefixes of the model names this vendor serves.
+ models []string
+ // aliases are names the vendor documents as pointing at whichever
model is
+ // current. Asking for one and being answered with a different name is
what
+ // the vendor says will happen, so it is not evidence of anything.
+ aliases []string
+ // headers are the response headers this vendor's own API sets. Empty
means
+ // this build does not document them, which leaves the header case
+ // unmeasured rather than failed.
+ headers []string
+}
+
+// The entries below were each read off the vendor's own API reference. A
vendor
+// is here for what it documents, not for being trusted: an official endpoint
+// scores no better for being official.
+var probeVendors = []probeVendor{
+ {
+ key: "openai",
+ hosts: []string{"api.openai.com"},
+ models: []string{"gpt-", "chatgpt-", "o1", "o3", "o4",
"codex-", "text-embedding-"},
+ aliases: []string{"chatgpt-4o-latest"},
+ headers: []string{
+ "X-Request-Id",
+ "Openai-Organization",
+ "Openai-Processing-Ms",
+ "Openai-Version",
+ "X-Ratelimit-Limit-Requests",
+ },
+ },
+ {
+ key: "anthropic",
+ hosts: []string{"api.anthropic.com"},
+ models: []string{"claude-"},
+ headers: []string{
+ "Request-Id",
+ "Anthropic-Organization-Id",
+ "Anthropic-Ratelimit-Requests-Limit",
+ "Anthropic-Ratelimit-Input-Tokens-Limit",
+ "X-Should-Retry",
+ },
+ },
+ {
+ key: "deepseek",
+ hosts: []string{"api.deepseek.com"},
+ // "deepseek-chat" is the non-thinking model of the day and
+ // "deepseek-reasoner" the thinking one; which model that is
changes
+ // with every release, and the answer carries the name it
changed to.
+ models: []string{"deepseek"},
+ aliases: []string{"deepseek-chat", "deepseek-reasoner"},
+ },
+ {
+ key: "moonshot",
+ hosts: []string{"api.moonshot.cn", "api.moonshot.ai"},
+ models: []string{"kimi", "moonshot-"},
+ aliases: []string{"kimi-latest"},
+ },
+ {
+ key: "zhipu",
+ hosts: []string{"open.bigmodel.cn"},
+ models: []string{"glm-", "charglm-", "codegeex-"},
+ },
+ {
+ key: "qwen",
+ hosts: []string{"dashscope.aliyuncs.com",
"dashscope-intl.aliyuncs.com"},
+ models: []string{"qwen", "qwq", "qvq"},
+ },
+ {
+ key: "xai",
+ hosts: []string{"api.x.ai"},
+ models: []string{"grok-"},
+ },
+ {
+ key: "google",
+ hosts: []string{"generativelanguage.googleapis.com"},
+ models: []string{"gemini-", "gemma-"},
+ },
+ {
+ key: "mistral",
+ hosts: []string{"api.mistral.ai"},
+ models: []string{
+ "mistral-", "ministral-", "magistral-", "codestral-",
"devstral-", "pixtral-", "open-mistral-",
+ },
+ },
+ {
+ key: "minimax",
+ hosts: []string{"api.minimaxi.com", "api.minimax.chat",
"api.minimaxi.chat"},
+ models: []string{"minimax-", "abab"},
+ },
+ {
+ key: "stepfun",
+ hosts: []string{"api.stepfun.com", "api.stepfun.ai"},
+ models: []string{"step-"},
+ },
+}
+
+// probeVendorOfProvider is the vendor whose own endpoint this provider points
+// at. Anything else — a reseller, an aggregator, a private deployment — is
nil:
+// its answers are its own, not a vendor's to be checked against.
+func probeVendorOfProvider(provider *Provider) *probeVendor {
+ parsed, err := url.Parse(strings.TrimSpace(provider.BaseUrl))
+ if err != nil {
+ return nil
+ }
+
+ host := strings.ToLower(parsed.Hostname())
+ if host == "" {
+ return nil
+ }
+ for index := range probeVendors {
+ for _, candidate := range probeVendors[index].hosts {
+ if host == candidate || strings.HasSuffix(host,
"."+candidate) {
+ return &probeVendors[index]
+ }
+ }
+ }
+ return nil
+}
+
+// probeVendorOfModel is the vendor a model name belongs to, which is a
separate
+// question from whose endpoint served it: a reseller passing DeepSeek through
+// is still answering with DeepSeek's model names.
+func probeVendorOfModel(name string) *probeVendor {
+ model := probeModelName(name)
+ if model == "" {
+ return nil
+ }
+ for index := range probeVendors {
+ for _, prefix := range probeVendors[index].models {
+ if strings.HasPrefix(model, prefix) {
+ return &probeVendors[index]
+ }
+ }
+ }
+ return nil
+}
+
+// isProbeModelAlias reports whether this name is one the vendor documents as
+// moving. "-latest" says so in the name itself; the rest are listed above.
+func isProbeModelAlias(name string) bool {
+ model := probeModelName(name)
+ if model == "" {
+ return false
+ }
+ if strings.HasSuffix(model, "-latest") {
+ return true
+ }
+
+ vendor := probeVendorOfModel(model)
+ if vendor == nil {
+ return false
+ }
+ for _, alias := range vendor.aliases {
+ if model == alias {
+ return true
+ }
+ }
+ return false
+}
+
+// sameModelVendor reports whether two names come out of the same catalogue:
the
+// vendor that owns both prefixes where one is known, and the leading segment
of
+// the name otherwise, which is how the vendors that are not listed above still
+// name their own models.
+func sameModelVendor(asked string, answered string) bool {
+ left, right := probeModelName(asked), probeModelName(answered)
+ if left == "" || right == "" {
+ return false
+ }
+
+ if vendor := probeVendorOfModel(left); vendor != nil {
+ return probeVendorOfModel(right) == vendor
+ }
+ return probeModelSegment(left) == probeModelSegment(right)
+}
+
+// probeModelName is the model as the vendor names it, with the vendor prefix
an
+// aggregator adds ("deepseek/deepseek-chat") taken off.
+func probeModelName(raw string) string {
+ name := strings.ToLower(strings.TrimSpace(raw))
+ if index := strings.LastIndex(name, "/"); index >= 0 {
+ name = name[index+1:]
+ }
+ return name
+}
+
+func probeModelSegment(name string) string {
+ if index := strings.Index(name, "-"); index > 0 {
+ return name[:index]
+ }
+ return name
+}
diff --git a/web/src/components/ProviderAuditCard.tsx
b/web/src/components/ProviderAuditCard.tsx
index d94217b..4e14b93 100644
--- a/web/src/components/ProviderAuditCard.tsx
+++ b/web/src/components/ProviderAuditCard.tsx
@@ -159,7 +159,11 @@ function probeValue(check: ProbeCheck) {
*/
function probeDetail(check: ProbeCheck, probe: ProviderProbe) {
if (check.level === "unknown" && check.key !== "identity") {
- return i18next.t("audit:This question could not be asked");
+ // An endpoint that is nobody's own has no documented headers to be
missing,
+ // which is a gap in what Gateway knows rather than one in the answer.
+ return check.facts[0] === "undocumented"
+ ? i18next.t("audit:Vendor undocumented")
+ : i18next.t("audit:This question could not be asked");
}
switch (check.key) {
@@ -167,6 +171,9 @@ function probeDetail(check: ProbeCheck, probe:
ProviderProbe) {
if (check.facts.length === 0) {
return i18next.t("audit:No model name came back");
}
+ if (check.facts[1] === "alias") {
+ return fill("audit:Identity alias", {asked: probe.model, answered:
check.facts[0]});
+ }
return check.level === "ok"
? fill("audit:Identity ok", {answered: check.facts[0]})
: fill("audit:Identity different", {asked: probe.model, answered:
check.facts[0]});
diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json
index 706bcd4..9a9ccf9 100644
--- a/web/src/locales/en/data.json
+++ b/web/src/locales/en/data.json
@@ -433,6 +433,7 @@
"Headers looked for": "Headers looked for",
"Headers required": "Headers required",
"Held": "Held",
+ "Identity alias": "Asked for {asked} and {answered} answered, which is the
model this vendor documents that name as pointing at.",
"Identity different": "Asked for {asked} and the answer named {answered}.",
"Identity has no request": "This case sends nothing of its own: it reads
the model name off the first answer of the run.",
"Identity ok": "Answered as {answered}. Echoing a name back is easy, so
this is a floor rather than a proof.",
@@ -523,6 +524,7 @@
"User prompt": "User prompt",
"Vendor detail": "Carries {names}.",
"Vendor headers": "Vendor headers",
+ "Vendor undocumented": "Not asked: this endpoint is not a vendor's own, so
there is no documented header list to hold it to.",
"Warn outside": "Warn outside",
"Weight": "Weight",
"Weight hint": "What this case is worth in the score, relative to the
other cases measured on the same run.",
diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json
index d9b3fea..1786186 100644
--- a/web/src/locales/zh/data.json
+++ b/web/src/locales/zh/data.json
@@ -433,6 +433,7 @@
"Headers looked for": "查找的响应头",
"Headers required": "至少要有几个",
"Held": "接住了",
+ "Identity alias": "请求的是 {asked}，回答的是 {answered}，而这正是该厂商在文档里写明这个名字指向的模型。",
"Identity different": "请求的是 {asked}，返回里写的是 {answered}。",
"Identity has no request": "这条用例不单独发请求：它读的是本次探测第一个回答里的模型名。",
"Identity ok": "返回 {answered}。把名字原样回显是最容易做到的事，所以这只是底线，不是证明。",
@@ -523,6 +524,7 @@
"User prompt": "User 提示词",
"Vendor detail": "带有 {names}。",
"Vendor headers": "厂商响应头",
+ "Vendor undocumented": "未发问：这个地址不是厂商自己的域名，没有可供比对的响应头清单。",
"Warn outside": "超出则存疑",
"Weight": "权重",
"Weight hint": "这条用例在评分中占的分量，与同一次探测中其他被测用例相比。",
@@ -535,7 +537,7 @@
"case.cache-identical-prefix.method":
"同一段长提示词逐字节相同地发两次，间隔一秒半。真缓存会在第一次写入、第二次读回。两个计数器都是零，就说明整段前缀两次都按新输入计费了。",
"case.cache-identical-prefix.question": "缓存是真的，还是把同一段前缀当新输入收了两遍钱？",
"case.identity-model-name.displayName": "模型身份",
- "case.identity-model-name.method": "第一个请求会点名一个模型。所有补全 API
都会在回答里带上实际执行的模型名，同一系列的版本号或日期后缀算匹配。名字不一样是铁证；名字一样说明不了太多——把名字原样回显是天底下最容易的事。",
+ "case.identity-model-name.method": "第一个请求点名一个模型。所有补全 API
都在回答里写明实际执行的模型，同一系列的版本号或日期后缀算匹配。如果请求的名字本就是厂商声明会变的别名——比如 deepseek-chat，或任何以
-latest
结尾的——那回答只需要仍是该厂商的模型，这正是厂商自己文档里写的。除此以外的不一致就是硬证据；而名字对上了并说明不了太多——把名字原样拄回来是最容易做到的事。",
"case.identity-model-name.question": "作答的模型自报的名字，是不是请求里点名的那个？",
"case.stream-events-anthropic.displayName": "流式结构（Anthropic）",
"case.stream-events-anthropic.method": "同样的请求开着流式再发一次。把别的 API 翻译成这套 API
的后端，往往是文字还没露馅、信封先错了：开头和结尾的事件不见了，或者 message_start 里没有输入计数，因为那时它还什么都没数。",
@@ -546,12 +548,9 @@
"case.tools-nested-schema.displayName": "嵌套工具 Schema",
"case.tools-nested-schema.method": "工具调用是强制的，不许拒答，而且 Schema
在两层上都有必填字段，其中一层是对象数组。冒充被售模型的小模型通常会把对象拍平，或者干脆丢掉数组。",
"case.tools-nested-schema.question": "在强制调用的情况下，作答的东西能不能撑住两层嵌套的工具 Schema？",
- "case.vendor-headers-anthropic.displayName": "厂商响应头（Anthropic）",
- "case.vendor-headers-anthropic.method":
"挡在真服务前面的中转通常会透传一部分；从来没跟厂商说过话的后端则一个也拿不出来。这条用例自己不会判到报警级别：把响应头抹掉是干净，不一定是心虚。",
- "case.vendor-headers-anthropic.question": "厂商自己 API 的那些响应头回来了吗？",
- "case.vendor-headers-openai.displayName": "厂商响应头（OpenAI）",
- "case.vendor-headers-openai.method":
"挡在真服务前面的中转通常会透传一部分；从来没跟厂商说过话的后端则一个也拿不出来。这条用例自己不会判到报警级别：把响应头抹掉是干净，不一定是心虚。",
- "case.vendor-headers-openai.question": "厂商自己 API 的那些响应头回来了吗？"
+ "case.vendor-headers.displayName": "厂商响应头",
+ "case.vendor-headers.method": "这一条只向厂商自己的域名发问，比对的也是该厂商文档里写明的响应头：能说 OpenAI
兼容的 API，不等于自称是
OpenAI。真货前面的转售商通常会透传一部分，而从没连过厂商的后端一个也拿不出来；但把响应头抹掉也可能只是干净，所以这一条单独不会判定为异常。响应头列表留空就用该厂商自己的，也可以在这里指定要检查哪些响应头。",
+ "case.vendor-headers.question": "厂商自己 API 的那些响应头回来了吗？"
},
"general": {
"Action": "操作",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
