This is an automated email from the ASF dual-hosted git repository. hsluoyz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
commit 8b15fe2d5829c947276bc014dc085f4763048a05 Author: Yang Luo <[email protected]> AuthorDate: Sun Sep 6 02:04:11 2026 +0800 feat(routing): send each request the model it should get, and step down when that one cannot answer --- controllers/gemini.go | 6 +- controllers/llm_record.go | 2 +- controllers/model_route.go | 157 +++++++ controllers/models.go | 10 +- controllers/proxy.go | 83 ++-- controllers/proxy_record.go | 22 +- controllers/proxy_test.go | 14 +- .../en/1-getting-started/1.3-interface.md | 1 + docs/user-manual/en/3-providers/3.2-routing.md | 7 +- docs/user-manual/en/3-providers/3.5-import.md | 2 +- .../en/3-providers/3.6-model-routing.md | 63 +++ .../user-manual/en/4-governance/4.2-llm-records.md | 8 +- docs/user-manual/en/README.md | 1 + .../zh/1-getting-started/1.3-interface.md | 1 + docs/user-manual/zh/3-providers/3.2-routing.md | 7 +- docs/user-manual/zh/3-providers/3.5-import.md | 2 +- .../zh/3-providers/3.6-model-routing.md | 63 +++ .../user-manual/zh/4-governance/4.2-llm-records.md | 8 +- docs/user-manual/zh/README.md | 1 + object/llm_record.go | 29 +- object/model_route.go | 464 +++++++++++++++++++++ object/model_tier.go | 115 +++++ object/ormer.go | 7 + object/permission_casbin.go | 31 +- object/provider.go | 40 +- object/provider_health.go | 15 - routers/router.go | 8 + web/src/App.tsx | 5 + web/src/backend/ModelRouteBackend.ts | 39 ++ web/src/components/llm/route-edit-dialog.tsx | 294 +++++++++++++ web/src/components/llm/route-preview-card.tsx | 143 +++++++ web/src/locales/en/data.json | 35 ++ web/src/locales/zh/data.json | 35 ++ web/src/nav.ts | 4 + web/src/pages/LlmRecordsPage.tsx | 6 + web/src/pages/ModelRoutesPage.tsx | 243 +++++++++++ web/src/types.ts | 43 ++ 37 files changed, 1906 insertions(+), 108 deletions(-) diff --git a/controllers/gemini.go b/controllers/gemini.go index 79edd54..031e276 100644 --- a/controllers/gemini.go +++ b/controllers/gemini.go @@ -138,7 +138,9 @@ func (c *ApiController) AgentGeminiModels() { c.writeGeminiError(http.StatusBadRequest, "invalid_request_error", err.Error()) return } - c.writeProxyBody(http.StatusOK, mustEncode(geminiModelList(object.ModelsOfProviders(providers)))) + agentId := c.Ctx.Input.Param(":agentId") + c.writeProxyBody(http.StatusOK, + mustEncode(geminiModelList(object.ModelsWithRoutes(object.ModelsOfProviders(providers), agentId)))) } // GeminiModel answers the retrieve-a-model endpoint a Gemini client checks its @@ -150,7 +152,7 @@ func (c *ApiController) GeminiModel() { } model, _ := geminiCall(c.Ctx.Input.Param(":model")) - if _, err := object.GetProvidersByModel(model); err != nil { + if _, err := object.PlanModelRoute(model); err != nil { if errors.Is(err, object.ErrNoProviderAvailable) { c.writeGeminiError(http.StatusNotFound, "not_found_error", err.Error()) } else { diff --git a/controllers/llm_record.go b/controllers/llm_record.go index 77b6c6a..d498588 100644 --- a/controllers/llm_record.go +++ b/controllers/llm_record.go @@ -228,7 +228,7 @@ func (c *ApiController) GetLlmRecord() { // The rates travel with the record, so the detail pane does not match model // names of its own. - price, priced := object.GetLlmPrice(record.Model) + price, priced := object.GetLlmPrice(record.BilledModel()) if !priced { c.ResponseOk(record) return diff --git a/controllers/model_route.go b/controllers/model_route.go new file mode 100644 index 0000000..d30a462 --- /dev/null +++ b/controllers/model_route.go @@ -0,0 +1,157 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "encoding/json" + + "github.com/apache/casbin-gateway/object" +) + +// RoutePreviewStep is one attempt of a plan, as the page shows it: which +// provider would be asked, for which model, and which rule chose it. +type RoutePreviewStep struct { + Provider string `json:"provider"` + DisplayName string `json:"displayName"` + Model string `json:"model"` + Route string `json:"route"` + Suspended bool `json:"suspended"` +} + +// GetModelRoutes is every routing rule, disabled ones included, in the order +// they are read in. +func (c *ApiController) GetModelRoutes() { + if c.RequireAdmin() { + return + } + + routes, err := object.GetModelRoutes() + if err != nil { + c.ResponseError(err.Error()) + return + } + + c.ResponseOk(routes) +} + +func (c *ApiController) AddModelRoute() { + if c.RequireAdmin() { + return + } + + route := object.ModelRoute{} + if err := json.Unmarshal(c.Ctx.Input.RequestBody, &route); err != nil { + c.ResponseError(err.Error()) + return + } + if err := object.AddModelRoute(&route); err != nil { + c.ResponseError(err.Error()) + return + } + + c.ResponseOk(&route) +} + +// UpdateModelRoute writes an edited rule back. The name comes from the query +// rather than the body, so an edit cannot rename the row it is editing. +func (c *ApiController) UpdateModelRoute() { + if c.RequireAdmin() { + return + } + + name := c.Input().Get("name") + if name == "" { + c.ResponseError("no routing rule was named") + return + } + + route := object.ModelRoute{} + if err := json.Unmarshal(c.Ctx.Input.RequestBody, &route); err != nil { + c.ResponseError(err.Error()) + return + } + if err := object.UpdateModelRoute(name, &route); err != nil { + c.ResponseError(err.Error()) + return + } + + c.ResponseOk(&route) +} + +func (c *ApiController) DeleteModelRoute() { + if c.RequireAdmin() { + return + } + + name := c.Input().Get("name") + if name == "" { + c.ResponseError("no routing rule was named") + return + } + if err := object.DeleteModelRoute(name); err != nil { + c.ResponseError(err.Error()) + return + } + + c.ResponseOk(true) +} + +// PreviewModelRoute answers what a request naming this model would actually be +// sent, without sending it. A ladder is only worth the rules it is written from +// if someone can see where the rules land, and the second and later steps are +// exactly the part nobody sees until an upstream is already failing. +func (c *ApiController) PreviewModelRoute() { + if c.RequireAdmin() { + return + } + + model := c.Input().Get("model") + if model == "" { + c.ResponseError("no model was named") + return + } + + attempts, err := c.planPreview(c.Input().Get("agent"), model) + if err != nil { + c.ResponseError(err.Error()) + return + } + + steps := []RoutePreviewStep{} + for _, attempt := range attempts { + steps = append(steps, RoutePreviewStep{ + Provider: attempt.Provider.GetId(), + DisplayName: attempt.Provider.DisplayName, + Model: attempt.Model, + Route: attempt.Route, + Suspended: object.IsProviderSuspended(attempt.Provider.GetId()), + }) + } + c.ResponseOk(steps) +} + +// planPreview is the plan the proxy would build, for an agent or for a bare +// model name. +func (c *ApiController) planPreview(agentId string, model string) ([]object.RouteAttempt, error) { + if agentId == "" { + return object.PlanModelRoute(model) + } + + providers, err := object.GetProvidersByAgent(agentId) + if err != nil { + return nil, err + } + return object.PlanAgentRoute(agentId, model, providers) +} diff --git a/controllers/models.go b/controllers/models.go index 4b9ee59..14cdde3 100644 --- a/controllers/models.go +++ b/controllers/models.go @@ -44,16 +44,16 @@ func (c *ApiController) Models() { // Model answers the retrieve-a-model endpoint an Anthropic client checks the // configured model with. A name reaches an upstream as long as one provider is -// enabled, see GetProvidersByModel(), so that is what "available" means here: -// answering 404 for a name the gateway would happily route would tell the -// client its model is missing when it is not. +// enabled or a rule routes it, see PlanModelRoute(), so that is what +// "available" means here: answering 404 for a name the gateway would happily +// route would tell the client its model is missing when it is not. func (c *ApiController) Model() { if !c.allowModelListing() { return } model := c.Ctx.Input.Param(":modelId") - if _, err := object.GetProvidersByModel(model); err != nil { + if _, err := object.PlanModelRoute(model); err != nil { if errors.Is(err, object.ErrNoProviderAvailable) { c.writeModelsError(http.StatusNotFound, "not_found_error", err.Error()) } else { @@ -92,7 +92,7 @@ func (c *ApiController) AgentModels() { c.writeModelsError(http.StatusBadRequest, "invalid_request_error", err.Error()) return } - c.writeModelList(object.ModelsOfProviders(providers)) + c.writeModelList(object.ModelsWithRoutes(object.ModelsOfProviders(providers), agentId)) } // allowModelListing gates the listing on the same token the proxy endpoints ask diff --git a/controllers/proxy.go b/controllers/proxy.go index ab0c21b..2c5178b 100644 --- a/controllers/proxy.go +++ b/controllers/proxy.go @@ -269,8 +269,9 @@ func (c *ApiController) forwardByModel(route *proxyRoute) { // what a record describes, so this is the only place one is written. defer c.finishLlmRecord(route) - // Match the providers globally, without an owner filter. - providers, err := object.GetProvidersByModel(route.model) + // Match the providers globally, without an owner filter. A routing rule + // covering this model puts its own ladder in front of them. + attempts, err := object.PlanModelRoute(route.model) if err != nil { if errors.Is(err, object.ErrNoProviderAvailable) { route.recordOutcome(http.StatusBadRequest, err.Error()) @@ -283,7 +284,7 @@ func (c *ApiController) forwardByModel(route *proxyRoute) { return } - c.forwardToProviders(providers, route) + c.forwardAttempts(attempts, route) } // proxyByAgent forwards to the provider chain bound to the agent in the path. @@ -333,17 +334,30 @@ func (c *ApiController) forwardByAgent(route *proxyRoute) { return } + // The ladder of a rule covering this agent comes first, resolved against the + // chain, and the chain itself is what it falls back to. + attempts, err := object.PlanAgentRoute(agentId, route.model, providers) + if err != nil { + beego.Error("model route lookup failed:", err) + route.recordOutcome(http.StatusBadGateway, err.Error()) + c.writeProxyError(route.codec, http.StatusBadGateway, "server_error", err.Error()) + return + } + + // Held to the permissions after planning rather than before: a rule may name + // a provider outside the chain and may ask for another model, and neither is + // a way around what this agent is allowed. if guard != nil { - allowed, reason := guard.FilterProviders(providers) + allowed, reason := guard.FilterAttempts(attempts) if len(allowed) == 0 { route.recordOutcome(http.StatusForbidden, reason) c.writeProxyError(route.codec, http.StatusForbidden, "permission_error", reason) return } - providers = allowed + attempts = allowed } - c.forwardToProviders(providers, route) + c.forwardAttempts(attempts, route) } // allowAgentRequest holds one relayed request to the agent's permissions: a @@ -418,21 +432,28 @@ func (c *ApiController) newProxyRoute(target proxyTarget, model string, stream b return route, true } -// forwardToProviders relays the request to the first provider that answers. -func (c *ApiController) forwardToProviders(providers []*object.Provider, route *proxyRoute) { +// forwardAttempts relays the request down the plan until one attempt answers. +// An attempt is a provider and the model to ask it for, so failing over can +// change either: to another upstream serving the same model, or down to a +// lesser model when nothing above it can answer. +func (c *ApiController) forwardAttempts(attempts []object.RouteAttempt, route *proxyRoute) { // Drop the providers this proxy cannot talk to before forwarding, so that - // the last usable provider is known and its response can be relayed as-is. - usableProviders := []*object.Provider{} + // the last usable attempt is known and its response can be relayed as-is. + usable := []object.RouteAttempt{} skipReason := "" - for _, provider := range providers { - if reason := c.providerUnusableReason(provider); reason != "" { - beego.Error("skipped provider", provider.GetId()+":", reason) + reported := map[string]bool{} + for _, attempt := range attempts { + if reason := c.providerUnusableReason(attempt.Provider); reason != "" { + if id := attempt.Provider.GetId(); !reported[id] { + reported[id] = true + beego.Error("skipped provider", id+":", reason) + } skipReason = reason continue } - usableProviders = append(usableProviders, provider) + usable = append(usable, attempt) } - if len(usableProviders) == 0 { + if len(usable) == 0 { message := fmt.Sprintf("no usable provider for %s", route.source) if skipReason != "" { message = skipReason @@ -444,24 +465,24 @@ func (c *ApiController) forwardToProviders(providers []*object.Provider, route * // A provider inside its failure cooldown goes last, so a dead upstream stops // costing every request the time it takes to time out. - usableProviders = object.SortProvidersByHealth(usableProviders) + usable = object.SortAttemptsByHealth(usable) - // Fail over to the next provider as long as nothing has been written to the - // client yet. The last provider is never retried, so the client gets the - // real upstream answer instead of a synthesized error. + // Fail over to the next attempt as long as nothing has been written to the + // client yet. The last one is never retried, so the client gets the real + // upstream answer instead of a synthesized error. lastStatus, lastMessage := http.StatusBadGateway, "upstream connection failed" - for i, provider := range usableProviders { + for i, attempt := range usable { if c.Ctx.Request.Context().Err() != nil { // The client hung up, there is nobody left to fail over for. route.recordOutcome(0, "client disconnected") return } - status, message, written := c.forwardToProvider(provider, route, i == len(usableProviders)-1) + status, message, written := c.forwardToProvider(attempt, route, i == len(usable)-1) if written { return } - route.recordFailure(provider.GetId(), status, message) + route.recordFailure(attempt, status, message) lastStatus, lastMessage = status, message } @@ -469,13 +490,15 @@ func (c *ApiController) forwardToProviders(providers []*object.Provider, route * c.writeProxyError(route.codec, lastStatus, "server_error", lastMessage) } -// forwardToProvider sends the request to a single provider's upstream. It reports -// whether the client response was already written, and when it was not, the -// status and message describing the failure so that the caller can fail over to -// the next provider. The response of the last provider is always relayed, even -// when its status would otherwise be retried. -func (c *ApiController) forwardToProvider(provider *object.Provider, route *proxyRoute, isLast bool) (int, string, bool) { - route.recordAttempt(provider.GetId()) +// forwardToProvider sends the request to a single provider's upstream, asking +// for the model the attempt names. It reports whether the client response was +// already written, and when it was not, the status and message describing the +// failure so that the caller can fail over to the next attempt. The response of +// the last attempt is always relayed, even when its status would otherwise be +// retried. +func (c *ApiController) forwardToProvider(attempt object.RouteAttempt, route *proxyRoute, isLast bool) (int, string, bool) { + provider := attempt.Provider + route.recordAttempt(attempt) upstream, err := protocol.UpstreamOf(object.ProviderProtocol(provider)) if err != nil { @@ -488,7 +511,7 @@ func (c *ApiController) forwardToProvider(provider *object.Provider, route *prox return c.answerCountTokens(route) } - requestBody, err := route.upstreamBody(upstream, object.ProviderModel(provider, route.model)) + requestBody, err := route.upstreamBody(upstream, attempt.Model) if err != nil { return http.StatusBadRequest, err.Error(), false } diff --git a/controllers/proxy_record.go b/controllers/proxy_record.go index 7c89eda..a45f973 100644 --- a/controllers/proxy_record.go +++ b/controllers/proxy_record.go @@ -37,7 +37,7 @@ const ( ) // finishLlmRecord queues the record of one relayed request. Every path out of -// forwardToProviders ends the client request, so this is the only place it is +// forwardAttempts ends the client request, so this is the only place it is // called from. func (c *ApiController) finishLlmRecord(route *proxyRoute) { if route.record == nil { @@ -50,12 +50,20 @@ func (c *ApiController) finishLlmRecord(route *proxyRoute) { object.AddLlmRecord(record, route.body) } -func (route *proxyRoute) recordAttempt(providerId string) { +// recordAttempt names the provider and the model this try is being made +// against. UpstreamModel is filled only when the two names differ, so a record +// says nothing about routing where nothing was routed. +func (route *proxyRoute) recordAttempt(attempt object.RouteAttempt) { if route.record == nil { return } route.record.Attempts++ - route.record.Provider = providerId + route.record.Provider = attempt.Provider.GetId() + route.record.Route = attempt.Route + route.record.UpstreamModel = "" + if attempt.Model != route.model { + route.record.UpstreamModel = attempt.Model + } } func (route *proxyRoute) recordOutcome(status int, message string) { @@ -82,13 +90,15 @@ func (route *proxyRoute) recordErrorBody(raw []byte) { } } -// recordFailure keeps one attempt the chain failed over from. -func (route *proxyRoute) recordFailure(providerId string, status int, message string) { +// recordFailure keeps one attempt the plan failed over from, with the model it +// asked for: a request that ended on a lesser model shows what was tried above. +func (route *proxyRoute) recordFailure(attempt object.RouteAttempt, status int, message string) { if route.record == nil { return } route.record.Failures = append(route.record.Failures, object.LlmFailure{ - Provider: providerId, + Provider: attempt.Provider.GetId(), + Model: attempt.Model, Status: status, Error: auditutil.BoundString(message, llmErrorMessageBytes), }) diff --git a/controllers/proxy_test.go b/controllers/proxy_test.go index c5af929..15e4f46 100644 --- a/controllers/proxy_test.go +++ b/controllers/proxy_test.go @@ -231,6 +231,10 @@ func TestIdleTimeoutReader(t *testing.T) { } } +func attemptOf(provider *object.Provider, model string) object.RouteAttempt { + return object.RouteAttempt{Provider: provider, Model: model} +} + func TestForwardToProvider(t *testing.T) { overloadedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -264,7 +268,7 @@ func TestForwardToProvider(t *testing.T) { // A retryable status fails over instead of reaching the client. c, recorder := newTestApiController() - statusCode, message, written := c.forwardToProvider(overloadedProvider, route, false) + statusCode, message, written := c.forwardToProvider(attemptOf(overloadedProvider, "gpt-4"), route, false) if written { t.Fatal("a retryable status was relayed instead of failing over") } @@ -278,14 +282,14 @@ func TestForwardToProvider(t *testing.T) { // The last provider is relayed as-is, even with a retryable status, so that // the client sees the real upstream answer. c, recorder = newTestApiController() - _, _, written = c.forwardToProvider(overloadedProvider, route, true) + _, _, written = c.forwardToProvider(attemptOf(overloadedProvider, "gpt-4"), route, true) if !written || recorder.Code != http.StatusServiceUnavailable || !strings.Contains(recorder.Body.String(), "overloaded") { t.Errorf("the last provider was not relayed: written = %v, statusCode = %d, body = %s", written, recorder.Code, recorder.Body.String()) } // A healthy provider, with a trailing slash in its base URL. c, recorder = newTestApiController() - _, _, written = c.forwardToProvider(healthyProvider, route, true) + _, _, written = c.forwardToProvider(attemptOf(healthyProvider, "gpt-4"), route, true) if !written || recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), "choices") { t.Errorf("the healthy provider failed: written = %v, statusCode = %d, body = %s", written, recorder.Code, recorder.Body.String()) } @@ -293,7 +297,7 @@ func TestForwardToProvider(t *testing.T) { // stream=true, but the upstream rejected the request: the JSON error must // not be dressed up as an SSE stream. c, recorder = newTestApiController() - c.forwardToProvider(overloadedProvider, newProxyRoute(openAiChat, rawBody, true), true) + c.forwardToProvider(attemptOf(overloadedProvider, "gpt-4"), newProxyRoute(openAiChat, rawBody, true), true) if header := recorder.Header().Get("Content-Type"); header != "application/json" { t.Errorf("Content-Type = %s, expected application/json", header) } @@ -320,7 +324,7 @@ func TestForwardToProviderAnthropic(t *testing.T) { c, recorder := newTestApiController() c.Ctx.Request.Header.Add("Anthropic-Beta", "fine-grained-tool-streaming-2025-05-14") - if _, _, written := c.forwardToProvider(provider, route, true); !written { + if _, _, written := c.forwardToProvider(attemptOf(provider, "claude-opus-5"), route, true); !written { t.Fatal("the anthropic provider was not relayed") } diff --git a/docs/user-manual/en/1-getting-started/1.3-interface.md b/docs/user-manual/en/1-getting-started/1.3-interface.md index 1f209a4..e51bd6d 100644 --- a/docs/user-manual/en/1-getting-started/1.3-interface.md +++ b/docs/user-manual/en/1-getting-started/1.3-interface.md @@ -12,6 +12,7 @@ Everything is at **http://localhost:17000**, in the desktop window or in a brows | **Agent versions** | Installed build against published release, for every agent Gateway knows | [2.2](../2-agents/2.2-versions.md) | | **Providers** | One endpoint in front of your model vendors | [3.1](../3-providers/3.1-providers.md) | | **Import settings** | Bring a `ccswitch://` link, or a CC Switch installation, over | [3.5](../3-providers/3.5-import.md) | +| **Model routing** | Which model each request is really sent, and what it degrades to | [3.6](../3-providers/3.6-model-routing.md) | | **Skills, MCP & Prompts** | Every skill, MCP server and instruction file of every agent, in one table | [2.4](../2-agents/2.4-skills-mcp-prompts.md) | | **Connections** | Connect an application once and write it into whichever agents you pick | [2.5](../2-agents/2.5-connections.md) | | **Authenticity** | Is the API behind that key what it was sold as — a score and a grade per provider | [3.4](../3-providers/3.4-authenticity.md) | diff --git a/docs/user-manual/en/3-providers/3.2-routing.md b/docs/user-manual/en/3-providers/3.2-routing.md index 1a48b3b..a287041 100644 --- a/docs/user-manual/en/3-providers/3.2-routing.md +++ b/docs/user-manual/en/3-providers/3.2-routing.md @@ -50,9 +50,10 @@ The one thing Gateway answers itself is the token count an Anthropic or Gemini c 1. **Authenticated** — the relay token, unless the request came from this machine ([5.2](../5-operations/5.2-security-and-network.md)). 2. **Checked** — against that agent's permissions: the model, the provider, and the tools in the request ([4.1](../4-governance/4.1-permissions.md)). A tool that is switched off is taken out of the request before it leaves this machine. -3. **Translated**, if the provider speaks a different API. -4. **Relayed**, with the provider's key. -5. **Recorded**, to the depth `llmRecordMode` says ([4.2](../4-governance/4.2-llm-records.md)). +3. **Planned** — which provider answers, and under which model name, down to what it degrades to if that one cannot ([3.6](3.6-model-routing.md)). +4. **Translated**, if the provider speaks a different API. +5. **Relayed**, with the provider's key, failing over down the plan until one answers. +6. **Recorded**, to the depth `llmRecordMode` says ([4.2](../4-governance/4.2-llm-records.md)). A request that asks for something switched off comes back as a `permission_error` in the API the agent speaks, so it reads as a refusal rather than as a broken gateway. diff --git a/docs/user-manual/en/3-providers/3.5-import.md b/docs/user-manual/en/3-providers/3.5-import.md index c3d80f6..464b22d 100644 --- a/docs/user-manual/en/3-providers/3.5-import.md +++ b/docs/user-manual/en/3-providers/3.5-import.md @@ -44,4 +44,4 @@ Gateway claims the `ccswitch://` scheme on Windows and Linux when it starts, so The link is handed to Gateway in the body of an API call, not in the address of the page it opens, because a provider link carries an API key — and an address is kept in browser history and sent on to wherever the page navigates next. -Next: [4.1 Permissions](../4-governance/4.1-permissions.md). +Next: [3.6 Model routing and automatic fallback](3.6-model-routing.md). diff --git a/docs/user-manual/en/3-providers/3.6-model-routing.md b/docs/user-manual/en/3-providers/3.6-model-routing.md new file mode 100644 index 0000000..5b58582 --- /dev/null +++ b/docs/user-manual/en/3-providers/3.6-model-routing.md @@ -0,0 +1,63 @@ +# 3.6 Model routing and automatic fallback + +**Model routing** decides which model a request is actually sent, and what it steps down to when that one cannot answer. It lives at **Model routing** in the sidebar. + +The reason it exists: an agent picks its own model names. Claude Code asks for `claude-sonnet-4-5` for your turn and `claude-haiku-4-5` for its background summaries, whatever it is configured with; Codex Desktop remembers the model chosen in its own state and ignores the config file. Point either at a provider that never heard of those names and the model in the request is not a name that provider serves. + +## What happens with no rules at all + +Nothing to configure — this is the default, and for most setups it is enough. + +- A model a provider **does** serve is asked for as written. +- A model it does not serve is answered with **the closest model of the same size class** that provider offers, read off the name: `haiku`, `mini`, `flash`, `lite`, `air` and friends are small; `opus`, `pro`, `max`, `reasoner`, `thinking` are large; everything else is in between. A request for `claude-haiku-4-5` on a DeepSeek provider lands on `deepseek-chat`, not on `deepseek-reasoner`. + + That is the automatic part: background traffic stays on the cheap model without anyone writing a rule. + +- If several providers serve the model, they are tried in priority order, and one inside its failure cooldown goes last. + +## What a rule adds + +Write a rule when you want to say where a model goes rather than let the name decide. + +| Field | What it means | +| --- | --- | +| **When the client asks for** | The model name in the request. `*` stands for any run of characters, so `*haiku*` covers every version of it. Matching ignores case. | +| **Agent** | Limits the rule to one agent id. Empty covers every caller. | +| **Send it to** | The ladder, top rung first. | + +Each rung names a model, a provider, or both: + +- **A model only** — any enabled provider that serves it may answer. +- **A provider only** — that provider answers, asked for the model the client sent. +- **Both** — that provider, asked for that model. + +## The ladder is the fallback + +The first rung is where the request goes. Every rung below it is what the request steps down to when the rung above **could not answer**: a rate limit, an exhausted quota, a rejected key, a 5xx, a timeout, or a model that provider does not have. Up to eight rungs. + +A rule that reads + +1. `glm-4.5-air` @ Zhipu +2. `deepseek-chat` + +sends the covered requests to Zhipu's small model, and when Zhipu is rate-limited sends the same request to whichever enabled provider serves `deepseek-chat`. + +Below the last rung, Gateway still falls back to what it would have done with no rule at all, so a rule whose providers have all been deleted or disabled leaves the request working rather than dead. + +The client is answered in the model name **it asked for**, whichever model actually served the turn. + +## Which rule wins + +Rules are read in order: the **Sort** number first, then the more specific pattern (an exact model name beats `claude-*`, which beats `*`), then a rule tied to an agent before a general one. The first rule whose pattern matches decides; only one rule applies to a request. + +## Routing is not permission + +A rule says where a request goes. It never refuses one — what an agent may ask for is set on the [Permissions](../4-governance/4.1-permissions.md) page, and the ladder is filtered through that agent's permissions before it is walked. A downgrade can never reach a model or a provider the agent is not allowed. + +## Seeing where a request would go + +**Where would this go?** on the same page answers with the whole plan for a model name, without sending a request: every step in the order it would be tried, which rule chose it, and which providers are in a cooldown right now. The rungs below the first are exactly the part nobody sees until an upstream is already failing. + +After the fact, [LLM Records](../4-governance/4.2-llm-records.md) shows the same thing for a request that really happened: **Routed to** is the model that actually went on the wire when it differed from the one asked for, **Routed by** names the rule, and the attempts the request failed over from are listed with the model each of them asked for. The cost is priced against the model that was really billed. + +Next: [4.1 Permissions](../4-governance/4.1-permissions.md). diff --git a/docs/user-manual/en/4-governance/4.2-llm-records.md b/docs/user-manual/en/4-governance/4.2-llm-records.md index 34b10e8..20f293b 100644 --- a/docs/user-manual/en/4-governance/4.2-llm-records.md +++ b/docs/user-manual/en/4-governance/4.2-llm-records.md @@ -37,9 +37,15 @@ llmRecordMaxPayloadBytes = 1048576 - Bodies are **sanitized** before they are stored: anything that looks like a credential is replaced, and the number of replacements is shown with the record. - A body over `llmRecordMaxPayloadBytes` keeps its structure and loses only its longest strings, so a large conversation is still listed message by message. +## When the model asked for was not the model served + +A request can be answered by a model other than the one the client named: a [routing rule](../3-providers/3.6-model-routing.md), or the downgrade taken when the model above it could not answer. The record of such a request carries **Routed to**, the name that actually went on the wire, and **Routed by**, the rule that chose it. Neither appears when nothing was routed. + +Every attempt the request failed over from is listed with the provider, the model it asked for and why it did not answer, so a turn served by a lesser model says what was tried above it. + ## The cost column -Worked out from list prices, which vendors change and resellers do not follow. Two ways to correct it: +Worked out from list prices, which vendors change and resellers do not follow, and priced against the model that was actually billed rather than the one the client asked for. Two ways to correct it: - Edit the price on the **Model pricing** page ([4.3](4.3-usage-and-pricing.md)). - Point `llmPricingFile` at a JSON file of your own rates. diff --git a/docs/user-manual/en/README.md b/docs/user-manual/en/README.md index 052b337..16f01a7 100644 --- a/docs/user-manual/en/README.md +++ b/docs/user-manual/en/README.md @@ -28,6 +28,7 @@ - [3.3 No API key: the caller's own login](3-providers/3.3-caller-login.md) - [3.4 Authenticity](3-providers/3.4-authenticity.md) - [3.5 Import settings and links](3-providers/3.5-import.md) +- [3.6 Model routing and automatic fallback](3-providers/3.6-model-routing.md) **4. Governance** diff --git a/docs/user-manual/zh/1-getting-started/1.3-interface.md b/docs/user-manual/zh/1-getting-started/1.3-interface.md index a4bcc08..9d3b470 100644 --- a/docs/user-manual/zh/1-getting-started/1.3-interface.md +++ b/docs/user-manual/zh/1-getting-started/1.3-interface.md @@ -12,6 +12,7 @@ | **版本管理** | 每个Agent的本机版本对上游发布版本 | [2.2](../2-agents/2.2-versions.md) | | **供应商** | 模型供应商前面的统一入口 | [3.1](../3-providers/3.1-providers.md) | | **导入配置** | 从`ccswitch://`链接，或从一份CC Switch安装里搬过来 | [3.5](../3-providers/3.5-import.md) | +| **模型路由** | 每个请求真正被发往哪个模型，以及它降级到什么 | [3.6](../3-providers/3.6-model-routing.md) | | **技能、MCP与提示词** | 所有Agent的技能、MCP服务器和指令文件，汇成一张表 | [2.4](../2-agents/2.4-skills-mcp-prompts.md) | | **连接** | 连一次应用，写进你勾选的每个Agent | [2.5](../2-agents/2.5-connections.md) | | **真伪检测** | 那个Key背后的API是不是卖给你的那个——每个供应商一个分数和等级 | [3.4](../3-providers/3.4-authenticity.md) | diff --git a/docs/user-manual/zh/3-providers/3.2-routing.md b/docs/user-manual/zh/3-providers/3.2-routing.md index 7cf956a..015dd8e 100644 --- a/docs/user-manual/zh/3-providers/3.2-routing.md +++ b/docs/user-manual/zh/3-providers/3.2-routing.md @@ -50,9 +50,10 @@ Gateway唯一自己回答的，是Anthropic和Gemini客户端在每轮之前问 1. **鉴权**——校验中继令牌，来自本机的请求除外（[5.2](../5-operations/5.2-security-and-network.md)）。 2. **校验**——按该Agent的权限检查模型、供应商，以及请求里带的工具（[4.1](../4-governance/4.1-permissions.md)）。被关掉的工具会在请求离开这台机器之前被摘掉。 -3. **转换**——供应商说的是另一套API时。 -4. **转发**——用供应商的Key。 -5. **记录**——按`llmRecordMode`规定的深度（[4.2](../4-governance/4.2-llm-records.md)）。 +3. **定路径**——由哪个供应商响应、用哪个模型名，以及它响应不了时降级到什么（[3.6](3.6-model-routing.md)）。 +4. **转换**——供应商说的是另一套API时。 +5. **转发**——用供应商的Key，沿着这条路径逐个尝试，直到有一个响应为止。 +6. **记录**——按`llmRecordMode`规定的深度（[4.2](../4-governance/4.2-llm-records.md)）。 请求里要的东西被关掉时，返回的是Agent所说那套API里的`permission_error`，所以它读起来是一次拒绝，而不是网关坏了。 diff --git a/docs/user-manual/zh/3-providers/3.5-import.md b/docs/user-manual/zh/3-providers/3.5-import.md index e4d7280..cdae897 100644 --- a/docs/user-manual/zh/3-providers/3.5-import.md +++ b/docs/user-manual/zh/3-providers/3.5-import.md @@ -44,4 +44,4 @@ Gateway启动时会在Windows和Linux上认领`ccswitch://`协议，所以点这 链接是放在API调用的请求体里交给Gateway的，而不是放在它打开的那个页面的地址里——因为供应商链接携带API Key，而地址会留在浏览器历史里，并且会被带到这个页面接下来跳转的任何地方。 -下一节：[4.1 权限](../4-governance/4.1-permissions.md)。 +下一节：[3.6 模型路由与自动降级](3.6-model-routing.md)。 diff --git a/docs/user-manual/zh/3-providers/3.6-model-routing.md b/docs/user-manual/zh/3-providers/3.6-model-routing.md new file mode 100644 index 0000000..3c242b0 --- /dev/null +++ b/docs/user-manual/zh/3-providers/3.6-model-routing.md @@ -0,0 +1,63 @@ +# 3.6 模型路由与自动降级 + +**模型路由**决定一个请求真正被发往哪个模型，以及那个模型响应不了时降级到什么。它在侧边栏的**模型路由**里。 + +之所以需要它：Agent会自己决定用哪个模型名。Claude Code给你的对话用`claude-sonnet-4-5`，给后台的总结任务固定用`claude-haiku-4-5`，不管它被配成什么；Codex Desktop记的是它自己状态里选的模型，配置文件里写什么都不看。把它们指向一个从没听说过这些名字的供应商，请求里的模型名就不是那个供应商提供的名字。 + +## 一条规则都不写会怎样 + +不用配任何东西——这是默认行为，对大多数场景已经够用。 + +- 供应商**确实提供**的模型，按原样请求。 +- 供应商没有的模型，会换成它提供的模型里**尺寸档位最接近的那个**，档位从名字读出来：`haiku`、`mini`、`flash`、`lite`、`air`这类是小模型，`opus`、`pro`、`max`、`reasoner`、`thinking`是大模型，其余在中间。一个请求`claude-haiku-4-5`落到DeepSeek渠道上，会走`deepseek-chat`而不是`deepseek-reasoner`。 + + 这就是自动的那部分：不写任何规则，后台流量也留在便宜模型上。 + +- 如果有多个供应商提供同一个模型，按优先级依次尝试，处于失败冷却期的排在最后。 + +## 规则多做了什么 + +当你想自己指定一个模型去哪儿、而不是让名字来决定时，就写一条规则。 + +| 字段 | 含义 | +| --- | --- | +| **当客户端请求** | 请求里的模型名。`*`匹配任意长度的字符，所以`*haiku*`能覆盖它的所有版本。匹配时不区分大小写。 | +| **Agent** | 把规则限定到一个agent id。留空则覆盖所有调用方。 | +| **路由到** | 降级阶梯，第一行优先。 | + +每一级可以只写模型、只写供应商，或者两个都写： + +- **只写模型**——任何提供该模型的启用渠道都可以响应。 +- **只写供应商**——由该渠道响应，请求的还是客户端发来的模型名。 +- **都写**——由该渠道响应，并且请求指定的那个模型。 + +## 阶梯就是降级路径 + +第一行是请求实际去的地方。下面每一行，都是上一行**响应不了**时降级到的目标：限流、余额耗尽、Key被拒、5xx、超时，或者那个渠道根本没有这个模型。最多八级。 + +一条这样的规则： + +1. `glm-4.5-air` @ 智谱 +2. `deepseek-chat` + +会把它覆盖的请求发给智谱的小模型；智谱被限流时，同一个请求转发给任意一个提供`deepseek-chat`的启用渠道。 + +最后一级之下，Gateway仍然会退回到"一条规则都没有时它本来会做的事"，所以一条规则里的渠道即使全被删掉或停用，请求也还是能走通，而不是直接失败。 + +不管最终是哪个模型响应的，回给客户端的模型名都是**它请求的那个**。 + +## 哪条规则生效 + +规则按顺序读取：先看**排序**数字，再看模式是否更具体（精确的模型名胜过`claude-*`，`claude-*`胜过`*`），最后绑定了Agent的规则排在通用规则前面。第一条模式匹配上的规则决定结果，一个请求只会命中一条规则。 + +## 路由不等于授权 + +规则只说请求去哪里，它从不拒绝请求。Agent能用哪些模型是在[权限](../4-governance/4.1-permissions.md)页面设置的，而且整条阶梯在被走之前会先经过该Agent的权限过滤。降级永远不可能落到这个Agent无权使用的模型或渠道上。 + +## 看清一个请求会走到哪里 + +同一个页面上的**这个模型会走到哪里？**会直接给出某个模型名的完整路径，而不发真实请求：按尝试顺序列出每一步、是哪条规则选的、以及此刻哪些渠道正在冷却。第一行以下的部分，正是平时上游没出问题就谁也看不见的那部分。 + +事后要看真实发生的请求，去[LLM记录](../4-governance/4.2-llm-records.md)：**实际发往**是真正上到线上的模型名（仅当它和请求的名字不同时才显示），**命中规则**是选中它的那条规则，失败切换掉的每一次尝试也会连同它请求的模型一起列出来。费用按真正计费的那个模型来算。 + +下一节：[4.1 权限](../4-governance/4.1-permissions.md)。 diff --git a/docs/user-manual/zh/4-governance/4.2-llm-records.md b/docs/user-manual/zh/4-governance/4.2-llm-records.md index 4549969..06ff857 100644 --- a/docs/user-manual/zh/4-governance/4.2-llm-records.md +++ b/docs/user-manual/zh/4-governance/4.2-llm-records.md @@ -37,9 +37,15 @@ llmRecordMaxPayloadBytes = 1048576 - 正文在落库前会**脱敏**：像凭据的内容会被替换掉，替换了多少处会连同记录一起显示。 - 超过`llmRecordMaxPayloadBytes`的正文只丢掉最长的那些字符串，结构保留，所以一段很长的对话仍然能一条一条列出来。 +## 请求的模型和响应的模型不是同一个时 + +一个请求可能由客户端请求之外的模型来响应：命中了[路由规则](../3-providers/3.6-model-routing.md)，或者上一级模型响应不了而降级了。这类请求的记录里会带上**实际发往**（真正上到线上的模型名）和**命中规则**（选中它的那条规则）。没发生路由时这两项不显示。 + +失败切换掉的每一次尝试也会列出来，包含渠道、它请求的模型，以及它为什么没响应，所以一次由较小模型完成的对话，能看清它上面试过什么。 + ## 费用那一列 -按标价算出来的，而厂商会改价、中转商不跟。两种纠正方式： +按标价算出来的，而厂商会改价、中转商不跟；计价用的是真正计费的那个模型，而不是客户端请求的那个。两种纠正方式： - 在**模型定价**页改价（[4.3](4.3-usage-and-pricing.md)）。 - 把`llmPricingFile`指向你自己那份费率的JSON文件。 diff --git a/docs/user-manual/zh/README.md b/docs/user-manual/zh/README.md index c3135f6..0f9a504 100644 --- a/docs/user-manual/zh/README.md +++ b/docs/user-manual/zh/README.md @@ -28,6 +28,7 @@ - [3.3 没有API Key：沿用调用方自己的登录](3-providers/3.3-caller-login.md) - [3.4 真伪检测](3-providers/3.4-authenticity.md) - [3.5 导入配置与链接](3-providers/3.5-import.md) +- [3.6 模型路由与自动降级](3-providers/3.6-model-routing.md) **4. 管控** diff --git a/object/llm_record.go b/object/llm_record.go index d38628b..9dab17f 100644 --- a/object/llm_record.go +++ b/object/llm_record.go @@ -43,8 +43,11 @@ var llmStringCaps = []int{16384, 4096, 1024, 256} // the request it was made for. type LlmFailure struct { Provider string `json:"provider"` - Status int `json:"status"` - Error string `json:"error"` + // Model is what that attempt asked for, which is not always what the client + // named: a routing rule and a downgrade both put another name on the wire. + Model string `json:"model"` + Status int `json:"status"` + Error string `json:"error"` } // LlmRecord is one client request relayed by the LLM proxy, together with the @@ -57,6 +60,14 @@ type LlmRecord struct { Protocol string `xorm:"varchar(20)" json:"protocol"` Endpoint string `xorm:"varchar(100)" json:"endpoint"` Model string `xorm:"varchar(255) index" json:"model"` + // UpstreamModel is what the provider was actually asked for, filled only + // when it differs from the model the client named. A routing rule, and the + // downgrade taken when the model above could not answer, both change it, + // and the request is priced against this one rather than the asked-for one. + UpstreamModel string `xorm:"varchar(255)" json:"upstreamModel"` + // Route names the routing rule that chose the provider and model, empty + // when nothing matched and the providers were tried in their own order. + Route string `xorm:"varchar(100)" json:"route"` Provider string `xorm:"varchar(201) index" json:"provider"` Agent string `xorm:"varchar(201) index" json:"agent"` ClientIp string `xorm:"varchar(100) index" json:"clientIp"` @@ -365,10 +376,20 @@ func fillLlmRecordBody(record *LlmRecord, rawBody []byte) { record.Summary = llmRecordSummary(sanitized) } -// fillLlmRecordCost prices a record from its token counters. +// fillLlmRecordCost prices a record from its token counters, against the model +// that was actually billed rather than the one the client asked for. func fillLlmRecordCost(record *LlmRecord) { record.Cost, record.Priced = GetLlmCost( - record.Model, record.PromptTokens, record.CompletionTokens, record.CacheWriteTokens, record.CacheReadTokens) + record.BilledModel(), record.PromptTokens, record.CompletionTokens, record.CacheWriteTokens, record.CacheReadTokens) +} + +// BilledModel is the model the upstream charged for: the one it was asked for +// when a rule or a downgrade renamed it, and otherwise the one the client sent. +func (record *LlmRecord) BilledModel() string { + if record.UpstreamModel != "" { + return record.UpstreamModel + } + return record.Model } // describeLlmRequest counts what the request was made of. diff --git a/object/model_route.go b/object/model_route.go new file mode 100644 index 0000000..624767d --- /dev/null +++ b/object/model_route.go @@ -0,0 +1,464 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Model routing: what the gateway actually asks for when a client names a +// model, and what it steps down to when that cannot answer. An agent picks its +// own model names - Claude Code sends claude-haiku-4-5 for its background work +// whatever it is configured with - so without a rule the only choice the +// gateway has is the provider's first model, at the provider's first price. + +package object + +import ( + "fmt" + "sort" + "strings" + + "github.com/apache/casbin-gateway/util" +) + +const ( + maxRouteTargets = 8 + maxRouteNameChars = 100 + maxRouteMatchChars = 200 + // maxRouteAttempts bounds one request: a rule of eight steps against a + // dozen providers must not turn one client request into a hundred. + maxRouteAttempts = 32 +) + +// RouteTarget is one rung of a route: the model to ask for, and optionally the +// only provider allowed to answer it. An empty Model keeps the name the client +// sent, which is how a rule pins a model to a provider without renaming it. +type RouteTarget struct { + Provider string `json:"provider"` + Model string `json:"model"` +} + +// ModelRoute is one rule read on the way in. Targets is a ladder, most +// preferred first: the rungs below the first are the automatic downgrade, taken +// when the rung above is rate-limited, out of quota or down. +// +// A route decides where a request goes, not whether it may go: nothing here +// refuses a model. What an agent is allowed to ask for is a permission, and the +// ladder is filtered through the agent's own permissions before it is walked, +// so a downgrade is never a way around them. +type ModelRoute struct { + Name string `xorm:"varchar(100) notnull pk" json:"name"` + CreatedTime string `xorm:"varchar(100)" json:"createdTime"` + UpdatedTime string `xorm:"varchar(100)" json:"updatedTime"` + + DisplayName string `xorm:"varchar(200)" json:"displayName"` + // Match is the model the client asks for, with "*" standing for any run of + // characters: "*haiku*" is one rule for every spelling of a vendor's small + // model, across the versions it will have next year. + Match string `xorm:"varchar(200)" json:"match"` + // Agent limits the rule to the requests of one agent, by its agent id. + // Empty applies it to every caller. + Agent string `xorm:"varchar(100) index" json:"agent"` + Targets []RouteTarget `xorm:"mediumtext json" json:"targets"` + // Sort orders the rules that could all match. Equal values fall back to how + // specific the pattern is, so an exact model name beats a wildcard without + // anyone having to number the rules. + Sort int `xorm:"int" json:"sort"` + Enabled bool `xorm:"bool" json:"enabled"` +} + +// RouteAttempt is one thing the proxy tries: a provider, and the model name to +// ask it for. A request is a list of these, walked until one answers. +type RouteAttempt struct { + Provider *Provider `json:"-"` + Model string `json:"model"` + // Route names the rule that put this attempt in the list, empty for the + // attempts the gateway would have made without any rule. + Route string `json:"route"` +} + +func (attempt RouteAttempt) providerId() string { + return attempt.Provider.GetId() +} + +// GetModelRoutes is every rule, disabled ones included, in the order they are +// read in. +func GetModelRoutes() ([]*ModelRoute, error) { + routes := []*ModelRoute{} + if ormer == nil || ormer.Engine == nil { + return routes, nil + } + if err := ormer.Engine.Find(&routes); err != nil { + return nil, err + } + + sortModelRoutes(routes) + for _, route := range routes { + if route.Targets == nil { + route.Targets = []RouteTarget{} + } + } + return routes, nil +} + +func GetModelRoute(name string) (*ModelRoute, error) { + route := &ModelRoute{Name: name} + existed, err := ormer.Engine.Get(route) + if err != nil || !existed { + return nil, err + } + return route, nil +} + +func AddModelRoute(route *ModelRoute) error { + if err := normalizeModelRoute(route); err != nil { + return err + } + existing, err := GetModelRoute(route.Name) + if err != nil { + return err + } + if existing != nil { + return fmt.Errorf("a routing rule named %s already exists", route.Name) + } + + route.CreatedTime = util.GetCurrentTime() + route.UpdatedTime = route.CreatedTime + _, err = ormer.Engine.Insert(route) + return err +} + +// UpdateModelRoute writes an edited rule back. The name identifies the row and +// is taken from the caller rather than the body, so an edit cannot rename it. +func UpdateModelRoute(name string, route *ModelRoute) error { + route.Name = name + if err := normalizeModelRoute(route); err != nil { + return err + } + stored, err := GetModelRoute(name) + if err != nil { + return err + } + if stored == nil { + return fmt.Errorf("the routing rule %s does not exist", name) + } + + route.CreatedTime = stored.CreatedTime + route.UpdatedTime = util.GetCurrentTime() + _, err = ormer.Engine.ID(name).AllCols().Update(route) + return err +} + +func DeleteModelRoute(name string) error { + _, err := ormer.Engine.ID(name).Delete(&ModelRoute{}) + return err +} + +// normalizeModelRoute fills in what a rule may leave out and refuses what it +// cannot mean. A named provider is resolved here so a typo fails at the form +// rather than on the next relayed request. +func normalizeModelRoute(route *ModelRoute) error { + route.Name = strings.TrimSpace(route.Name) + route.Match = strings.TrimSpace(route.Match) + route.Agent = strings.TrimSpace(route.Agent) + + if route.Name == "" || len(route.Name) > maxRouteNameChars { + return fmt.Errorf("the routing rule needs a name of at most %d characters", maxRouteNameChars) + } + if route.Match == "" || len(route.Match) > maxRouteMatchChars { + return fmt.Errorf("the routing rule needs a model pattern of at most %d characters", maxRouteMatchChars) + } + if route.DisplayName == "" { + route.DisplayName = route.Name + } + + targets := []RouteTarget{} + for _, target := range route.Targets { + target.Provider = strings.TrimSpace(target.Provider) + target.Model = strings.TrimSpace(target.Model) + if target.Provider == "" && target.Model == "" { + continue + } + if len(target.Model) > maxProviderModelChars { + return fmt.Errorf("model name is too long: %s", target.Model) + } + if target.Provider != "" { + provider, err := getProviderById(target.Provider) + if err != nil { + return err + } + if provider == nil { + return fmt.Errorf("the provider does not exist: %s", target.Provider) + } + } + targets = append(targets, target) + } + if len(targets) == 0 { + return fmt.Errorf("the routing rule %s names nothing to route to", route.Name) + } + if len(targets) > maxRouteTargets { + return fmt.Errorf("too many steps: %d, at most %d are allowed", len(targets), maxRouteTargets) + } + route.Targets = targets + return nil +} + +// sortModelRoutes puts the rules in the order they are tried: the operator's +// own numbering first, then the more specific pattern, then the rule tied to an +// agent, then the name so the list never reshuffles on its own. +func sortModelRoutes(routes []*ModelRoute) { + sort.SliceStable(routes, func(i, j int) bool { + left, right := routes[i], routes[j] + if left.Sort != right.Sort { + return left.Sort < right.Sort + } + leftScore, rightScore := patternSpecificity(left.Match), patternSpecificity(right.Match) + if leftScore != rightScore { + return leftScore > rightScore + } + if (left.Agent != "") != (right.Agent != "") { + return left.Agent != "" + } + return left.Name < right.Name + }) +} + +// patternSpecificity is how much of a pattern is not a wildcard. An exact model +// name is the most specific thing a rule can say, so it beats "claude-*" +// without the operator having to order the two by hand. +func patternSpecificity(pattern string) int { + if strings.Contains(pattern, "*") { + return len(pattern) - strings.Count(pattern, "*") - maxRouteMatchChars + } + return len(pattern) +} + +// MatchModelPattern reports whether a model name is covered by a pattern, in +// which "*" stands for any run of characters. The comparison ignores case: the +// same model reaches the gateway spelled both ways depending on the client. +func MatchModelPattern(pattern string, model string) bool { + pattern, model = strings.ToLower(pattern), strings.ToLower(model) + if !strings.Contains(pattern, "*") { + return pattern == model + } + + parts := strings.Split(pattern, "*") + if !strings.HasPrefix(model, parts[0]) { + return false + } + model = model[len(parts[0]):] + + last := parts[len(parts)-1] + for _, part := range parts[1 : len(parts)-1] { + index := strings.Index(model, part) + if index < 0 { + return false + } + model = model[index+len(part):] + } + return strings.HasSuffix(model, last) +} + +// MatchModelRoute is the rule a request is routed by, or nil when none covers +// it. A rule tied to this agent is preferred over a general one of the same +// specificity, see sortModelRoutes. +func MatchModelRoute(agentId string, model string) (*ModelRoute, error) { + routes, err := GetModelRoutes() + if err != nil { + return nil, err + } + + for _, route := range routes { + if !route.Enabled { + continue + } + if route.Agent != "" && route.Agent != agentId { + continue + } + if MatchModelPattern(route.Match, model) { + return route, nil + } + } + return nil, nil +} + +// PlanModelRoute is the list of attempts for a request that named a model and +// no agent. The matching rule's ladder comes first; what the gateway would have +// done without any rule is appended below it, so a rule whose providers are all +// gone leaves the request working rather than dead. +func PlanModelRoute(model string) ([]RouteAttempt, error) { + route, err := MatchModelRoute("", model) + if err != nil { + return nil, err + } + + attempts := []RouteAttempt{} + if route != nil { + // A rung that names no provider is resolved against everything in + // rotation, which is what a request with no agent behind it may use. + pool, poolErr := listEnabledProviders() + if poolErr != nil { + return nil, poolErr + } + decryptProviders(pool) + attempts = planRouteTargets(route, model, pool) + } + + providers, err := GetProvidersByModel(model) + if err != nil { + if len(attempts) > 0 { + // The rule already says where this goes, which is the whole point + // of writing one for a model no provider lists. + return attempts, nil + } + return nil, err + } + return appendAttempts(attempts, defaultAttempts(providers, model)), nil +} + +// PlanAgentRoute is the same for a request that arrived on an agent's own +// endpoint. The chain the agent is bound to is the pool an open rung is +// resolved against; a rung naming a provider outright is an explicit +// instruction, taken as written and still held to the agent's permissions by +// the caller. +func PlanAgentRoute(agentId string, model string, chain []*Provider) ([]RouteAttempt, error) { + route, err := MatchModelRoute(agentId, model) + if err != nil { + return nil, err + } + + attempts := []RouteAttempt{} + if route != nil { + attempts = planRouteTargets(route, model, chain) + } + return appendAttempts(attempts, defaultAttempts(chain, model)), nil +} + +// defaultAttempts is the plan with no rule in it: every provider in turn, each +// asked for the closest model it serves to the one the client named. +func defaultAttempts(providers []*Provider, model string) []RouteAttempt { + attempts := make([]RouteAttempt, 0, len(providers)) + for _, provider := range providers { + attempts = append(attempts, RouteAttempt{Provider: provider, Model: ProviderModel(provider, model)}) + } + return attempts +} + +// planRouteTargets walks a rule's ladder into attempts. A rung nothing can +// serve is skipped rather than failing the request: the rungs below it are what +// it is there for. +func planRouteTargets(route *ModelRoute, requested string, pool []*Provider) []RouteAttempt { + attempts := []RouteAttempt{} + for _, target := range route.Targets { + model := target.Model + if model == "" { + model = requested + } + + for _, provider := range routeTargetProviders(target, model, pool) { + attempts = appendAttempts(attempts, []RouteAttempt{{ + Provider: provider, + Model: ProviderModel(provider, model), + Route: route.Name, + }}) + } + } + return attempts +} + +// routeTargetProviders is who may answer one rung. A named provider is taken as +// written, wherever it sits; an unnamed one has to name the model itself, and +// is looked for in the pool the plan was built against. +func routeTargetProviders(target RouteTarget, model string, pool []*Provider) []*Provider { + if target.Provider != "" { + provider, err := getProviderById(target.Provider) + if err != nil || provider == nil || provider.Status != "enabled" { + // A rule outliving the provider it names is why the ladder has + // rungs below this one. + return nil + } + return []*Provider{provider} + } + + serving := []*Provider{} + for _, provider := range pool { + if ProviderServes(provider, model) { + serving = append(serving, provider) + } + } + return serving +} + +// appendAttempts adds the attempts that are not in the list yet. The same +// provider asked for the same model twice is one attempt, however many rules +// and chains put it there. +func appendAttempts(attempts []RouteAttempt, more []RouteAttempt) []RouteAttempt { + seen := map[string]bool{} + for _, attempt := range attempts { + seen[attempt.providerId()+"|"+attempt.Model] = true + } + + for _, attempt := range more { + key := attempt.providerId() + "|" + attempt.Model + if seen[key] || len(attempts) >= maxRouteAttempts { + continue + } + seen[key] = true + attempts = append(attempts, attempt) + } + return attempts +} + +// SortAttemptsByHealth puts the attempts whose provider is inside its failure +// cooldown last, keeping the ladder's order among the rest. A dead upstream +// stops costing every request the time it takes to time out, and a downgrade is +// still reached only after what is above it has been tried. +func SortAttemptsByHealth(attempts []RouteAttempt) []RouteAttempt { + ready := make([]RouteAttempt, 0, len(attempts)) + suspended := []RouteAttempt{} + for _, attempt := range attempts { + if IsProviderSuspended(attempt.providerId()) { + suspended = append(suspended, attempt) + continue + } + ready = append(ready, attempt) + } + return append(ready, suspended...) +} + +// ModelsWithRoutes adds the exact model names the rules answer for to the ones +// the providers list. They belong on the list a client fills its picker from: a +// name the gateway would happily route is one the client can ask for, whether +// or not any provider lists it. A wildcard rule adds nothing, since it names no +// model of its own. +func ModelsWithRoutes(models []string, agentId string) []string { + routes, err := GetModelRoutes() + if err != nil { + return models + } + + seen := map[string]bool{} + for _, model := range models { + seen[strings.ToLower(model)] = true + } + for _, route := range routes { + if !route.Enabled || strings.Contains(route.Match, "*") { + continue + } + if route.Agent != "" && route.Agent != agentId { + continue + } + if !seen[strings.ToLower(route.Match)] { + seen[strings.ToLower(route.Match)] = true + models = append(models, route.Match) + } + } + return models +} diff --git a/object/model_tier.go b/object/model_tier.go new file mode 100644 index 0000000..5c27459 --- /dev/null +++ b/object/model_tier.go @@ -0,0 +1,115 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package object + +import "strings" + +// The size class a model name says it belongs to. It is what an automatic +// downgrade is decided on: an agent asks for the small model of one vendor for +// its background work — Claude Code sends claude-haiku-4-5 for that — and +// sending it whichever model the provider happens to list first bills the +// cheapest traffic at the most expensive rate. +const ( + TierSmall = "small" + TierStandard = "standard" + TierLarge = "large" +) + +// The words the vendors spell their size classes with. They are matched as +// whole tokens of the model name rather than as substrings, so MiniMax-M1 is +// not read as a "mini" and gpt-4o-mini is. +var ( + smallTierWords = []string{"haiku", "mini", "flash", "lite", "small", "tiny", "nano", "micro", "instant", "air"} + largeTierWords = []string{"opus", "ultra", "max", "pro", "plus", "large", "heavy", "reasoner", "thinking"} +) + +// ModelTier is the size class a model name reads as. It is a guess made from +// the name alone, which is all a gateway has: the vendors publish no field for +// it and a relay renames their models anyway. +func ModelTier(model string) string { + for _, token := range modelNameTokens(model) { + if containsString(smallTierWords, token) { + return TierSmall + } + if containsString(largeTierWords, token) { + return TierLarge + } + } + return TierStandard +} + +// modelNameTokens splits a model name on everything that is not a letter or a +// digit, which is how every vendor separates the parts of one. +func modelNameTokens(model string) []string { + return strings.FieldsFunc(strings.ToLower(model), func(r rune) bool { + return !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') + }) +} + +// tierPreference is the order the tiers are searched in for a model to send +// instead of the one that was asked for. A request for the small model settles +// for a bigger one only after nothing small is offered, so background traffic +// stays cheap; a request for a big one steps down rather than up. +var tierPreference = map[string][]string{ + TierSmall: {TierSmall, TierStandard, TierLarge}, + TierStandard: {TierStandard, TierLarge, TierSmall}, + TierLarge: {TierLarge, TierStandard, TierSmall}, +} + +// PickProviderModel is the model to ask a provider for when the client named +// one. An exact match is always it; otherwise the closest size class the +// provider offers is taken, and the provider's own first model is the last +// resort. The provider's order is kept inside a class, since a vendor lists its +// newest model first. +func PickProviderModel(models []string, requested string) string { + if len(models) == 0 { + return requested + } + + for _, candidate := range models { + if candidate == requested { + return requested + } + } + for _, candidate := range models { + if strings.EqualFold(candidate, requested) { + return candidate + } + } + + for _, tier := range tierPreference[ModelTier(requested)] { + for _, candidate := range models { + if ModelTier(candidate) == tier { + return candidate + } + } + } + return models[0] +} + +// ProviderServes reports whether a provider names the model itself. A provider +// that names none serves whatever arrives, which is the client-auth case: the +// account behind the caller's own credentials decides, not this table. +func ProviderServes(provider *Provider, model string) bool { + if len(provider.Models) == 0 { + return UsesClientAuth(provider) + } + for _, candidate := range provider.Models { + if strings.EqualFold(candidate, model) { + return true + } + } + return false +} diff --git a/object/ormer.go b/object/ormer.go index 683bd6c..4514eb1 100644 --- a/object/ormer.go +++ b/object/ormer.go @@ -330,4 +330,11 @@ func (a *Ormer) createTable() { if err != nil { panic(err) } + + // The rules that decide which model a request is actually sent, and what it + // steps down to when that cannot answer. + err = a.Engine.Sync2(new(ModelRoute)) + if err != nil { + panic(err) + } } diff --git a/object/permission_casbin.go b/object/permission_casbin.go index caed617..e058074 100644 --- a/object/permission_casbin.go +++ b/object/permission_casbin.go @@ -369,22 +369,29 @@ func (guard *AgentGuard) AllowTool(name string) bool { return guard.allow(objectTool + entry) } -// FilterProviders drops the providers this agent may not be sent to, and says -// which was the last one dropped so that an empty chain can report why. -func (guard *AgentGuard) FilterProviders(providers []*Provider) ([]*Provider, string) { - allowed := []*Provider{} - denied := "" - for _, provider := range providers { - if !guard.AllowProvider(provider.GetId()) { - denied = provider.GetId() +// FilterAttempts drops the attempts this agent may not make. Both halves are +// checked: a routing rule that steps down to another model, or across to +// another provider, must not be a way around what the agent is allowed to ask +// for. The last thing dropped is named so an empty plan can report why. +func (guard *AgentGuard) FilterAttempts(attempts []RouteAttempt) ([]RouteAttempt, string) { + allowed := []RouteAttempt{} + reason := "" + for _, attempt := range attempts { + if !guard.AllowProvider(attempt.Provider.GetId()) { + reason = fmt.Sprintf("the permissions of agent %s do not allow the provider %s", + guard.agentId, attempt.Provider.GetId()) + continue + } + if !guard.AllowModel(attempt.Model) { + reason = fmt.Sprintf("the permissions of agent %s do not allow the model %s", + guard.agentId, attempt.Model) continue } - allowed = append(allowed, provider) + allowed = append(allowed, attempt) } - reason := "" - if len(allowed) == 0 && denied != "" { - reason = fmt.Sprintf("the permissions of agent %s do not allow the provider %s", guard.agentId, denied) + if len(allowed) > 0 { + reason = "" } return allowed, reason } diff --git a/object/provider.go b/object/provider.go index b870393..ea6661a 100644 --- a/object/provider.go +++ b/object/provider.go @@ -910,10 +910,9 @@ func probeDetail(body []byte) string { // rejecting them leaves the client unable to talk to any third-party provider // at all. func GetProvidersByModel(model string) ([]*Provider, error) { - providers := []*Provider{} - err := ormer.Engine.Where("status = ?", "enabled").Asc("priority").Find(&providers) + providers, err := listEnabledProviders() if err != nil { - return nil, fmt.Errorf("provider query failed: %w", err) + return nil, err } // The models are JSON-serialized into a single column, so the match cannot @@ -952,16 +951,26 @@ func GetProvidersByModel(model string) ([]*Provider, error) { } // ListEnabledModels is every model the enabled providers serve, in priority -// order and without duplicates. It is what the models endpoint answers with, so -// that a client fills its model picker with what this gateway can actually -// reach. +// order and without duplicates, plus the ones the routing rules answer for. It +// is what the models endpoint answers with, so that a client fills its model +// picker with what this gateway can actually reach. func ListEnabledModels() ([]string, error) { - providers := []*Provider{} - err := ormer.Engine.Where("status = ?", "enabled").Asc("priority").Find(&providers) + providers, err := listEnabledProviders() if err != nil { + return nil, err + } + return ModelsWithRoutes(ModelsOfProviders(providers), ""), nil +} + +// listEnabledProviders is every provider in rotation, lowest priority value +// first, which is the order requests are tried against them. The keys are still +// ciphertext: only the providers a request actually reaches are decrypted. +func listEnabledProviders() ([]*Provider, error) { + providers := []*Provider{} + if err := ormer.Engine.Where("status = ?", "enabled").Asc("priority").Find(&providers); err != nil { return nil, fmt.Errorf("provider query failed: %w", err) } - return ModelsOfProviders(providers), nil + return providers, nil } // ModelsOfProviders flattens the model lists of providers into one, keeping the @@ -985,17 +994,12 @@ func ModelsOfProviders(providers []*Provider) []string { // asked for. An agent picks its model on its own - Codex Desktop remembers the // one chosen in its own state, whatever the config file says - so a provider // that never heard of it answers with an error naming the models it does serve, -// rather than with a completion. That agent is sent the first model of the -// provider it is bound to instead. A provider that serves no named model, which -// is the client-auth case, takes whatever arrives. +// rather than with a completion. That agent is sent the closest model the +// provider does serve instead, see PickProviderModel. A provider that serves no +// named model, which is the client-auth case, takes whatever arrives. func ProviderModel(provider *Provider, model string) string { if len(provider.Models) == 0 { return model } - for _, candidate := range provider.Models { - if candidate == model { - return model - } - } - return provider.Models[0] + return PickProviderModel(provider.Models, model) } diff --git a/object/provider_health.go b/object/provider_health.go index 36644ec..fc8101c 100644 --- a/object/provider_health.go +++ b/object/provider_health.go @@ -142,21 +142,6 @@ func GetProviderHealth() []*ProviderHealth { return result } -// SortProvidersByHealth puts the providers that are inside a cooldown window last, -// keeping the configured order among the rest. -func SortProvidersByHealth(providers []*Provider) []*Provider { - ready := make([]*Provider, 0, len(providers)) - suspended := []*Provider{} - for _, provider := range providers { - if IsProviderSuspended(provider.GetId()) { - suspended = append(suspended, provider) - continue - } - ready = append(ready, provider) - } - return append(ready, suspended...) -} - func healthOf(providerId string) *providerHealth { health, ok := providerHealthMap[providerId] if !ok { diff --git a/routers/router.go b/routers/router.go index 9795283..1045042 100644 --- a/routers/router.go +++ b/routers/router.go @@ -175,6 +175,14 @@ func initAPI() { beego.Router("/api/delete-probe-case", &controllers.ApiController{}, "POST:DeleteProbeCase") beego.Router("/api/reset-probe-cases", &controllers.ApiController{}, "POST:ResetProbeCases") + // The rules deciding which model a request is sent, and what it steps down + // to when that cannot answer. + beego.Router("/api/get-model-routes", &controllers.ApiController{}, "GET:GetModelRoutes") + beego.Router("/api/add-model-route", &controllers.ApiController{}, "POST:AddModelRoute") + beego.Router("/api/update-model-route", &controllers.ApiController{}, "POST:UpdateModelRoute") + beego.Router("/api/delete-model-route", &controllers.ApiController{}, "POST:DeleteModelRoute") + beego.Router("/api/preview-model-route", &controllers.ApiController{}, "GET:PreviewModelRoute") + beego.Router("/api/get-llm-records", &controllers.ApiController{}, "GET:GetLlmRecords") beego.Router("/api/get-llm-record", &controllers.ApiController{}, "GET:GetLlmRecord") beego.Router("/api/get-llm-record-status", &controllers.ApiController{}, "GET:GetLlmRecordStatus") diff --git a/web/src/App.tsx b/web/src/App.tsx index 2fa4472..16dc9e4 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -45,6 +45,7 @@ import PermissionsPage from "@/pages/PermissionsPage"; import ProviderEditPage from "@/pages/ProviderEditPage"; import ProviderListPage from "@/pages/ProviderListPage"; import LlmRecordsPage from "@/pages/LlmRecordsPage"; +import ModelRoutesPage from "@/pages/ModelRoutesPage"; import UsagePage from "@/pages/UsagePage"; import PricingPage from "@/pages/PricingPage"; import SettingPage from "@/pages/SettingPage"; @@ -335,6 +336,10 @@ export default function App() { /> <Route path="/usage" element={requireSignin(user => <UsagePage account={user} />)} /> <Route path="/pricing" element={requireSignin(user => <PricingPage account={user} />)} /> + <Route + path="/model-routes" + element={requireSignin(user => <ModelRoutesPage account={user} />)} + /> </Routes> </React.Suspense> </main> diff --git a/web/src/backend/ModelRouteBackend.ts b/web/src/backend/ModelRouteBackend.ts new file mode 100644 index 0000000..7d228a5 --- /dev/null +++ b/web/src/backend/ModelRouteBackend.ts @@ -0,0 +1,39 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {query, request} from "@/backend/request"; +import type {ModelRoute, RoutePreviewStep} from "@/types"; + +/** Every routing rule, disabled ones included, in the order they are read in. */ +export function getModelRoutes() { + return request<ModelRoute[]>("/api/get-model-routes"); +} + +export function addModelRoute(route: ModelRoute) { + return request<ModelRoute>("/api/add-model-route", "POST", route); +} + +/** Writes an edited rule back. The name identifies the row and is not renamed. */ +export function updateModelRoute(name: string, route: ModelRoute) { + return request<ModelRoute>(`/api/update-model-route${query({name: name})}`, "POST", route); +} + +export function deleteModelRoute(name: string) { + return request<boolean>(`/api/delete-model-route${query({name: name})}`, "POST"); +} + +/** What a request naming this model would actually be sent, without sending it. */ +export function previewModelRoute(model: string, agent = "") { + return request<RoutePreviewStep[]>(`/api/preview-model-route${query({model: model, agent: agent})}`); +} diff --git a/web/src/components/llm/route-edit-dialog.tsx b/web/src/components/llm/route-edit-dialog.tsx new file mode 100644 index 0000000..3a6008b --- /dev/null +++ b/web/src/components/llm/route-edit-dialog.tsx @@ -0,0 +1,294 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as React from "react"; +import {ArrowDown, ArrowUp, Plus, Trash2} from "lucide-react"; +import i18next from "i18next"; + +import {Field, FormDialog} from "@/components/shared/form-dialog"; +import {SearchSelect, type SelectOption} from "@/components/shared/simple-select"; +import {ProviderIcon} from "@/components/ProviderIcon"; +import {Button} from "@/components/ui/button"; +import {Input} from "@/components/ui/input"; +import {Label} from "@/components/ui/label"; +import {Switch} from "@/components/ui/switch"; +import {SimpleTooltip} from "@/components/ui/tooltip"; +import {providerIdOf} from "@/lib/providers"; +import type {Agent, ModelRoute, Provider, RouteTarget} from "@/types"; + +const maxTargets = 8; + +function emptyRoute(): ModelRoute { + return { + name: "", + createdTime: "", + updatedTime: "", + displayName: "", + match: "", + agent: "", + targets: [{provider: "", model: ""}], + sort: 0, + enabled: true, + }; +} + +/** A rule with at least one rung to edit, however it arrived. */ +function draftOf(route: ModelRoute | null): ModelRoute { + if (route === null) { + return emptyRoute(); + } + return {...route, targets: route.targets.length > 0 ? [...route.targets] : [{provider: "", model: ""}]}; +} + +/** One rung of the ladder: which provider answers it, asked for which model. */ +function TargetRow({ + target, + index, + count, + providers, + onChange, + onMove, + onRemove, +}: { + target: RouteTarget; + index: number; + count: number; + providers: SelectOption[]; + onChange: (target: RouteTarget) => void; + onMove: (offset: number) => void; + onRemove: () => void; +}) { + return ( + <div className="flex items-end gap-2"> + <span className="text-muted-foreground w-16 shrink-0 pb-2.5 text-xs"> + {index === 0 + ? i18next.t("llm:First choice") + : i18next.t("llm:Step {n}").replace("{n}", String(index + 1))} + </span> + <div className="grid min-w-0 flex-1 gap-2 sm:grid-cols-2"> + <Input + className="font-mono" + placeholder={i18next.t("llm:Keep the requested model")} + value={target.model} + onChange={event => onChange({...target, model: event.target.value})} + /> + <SearchSelect + allowClear + value={target.provider} + onChange={provider => onChange({...target, provider: provider})} + options={providers} + placeholder={i18next.t("llm:Any provider serving it")} + /> + </div> + <div className="flex shrink-0 items-center"> + <SimpleTooltip title={i18next.t("llm:Move up")}> + <Button type="button" size="icon" variant="ghost" disabled={index === 0} onClick={() => onMove(-1)}> + <ArrowUp /> + </Button> + </SimpleTooltip> + <SimpleTooltip title={i18next.t("llm:Move down")}> + <Button + type="button" + size="icon" + variant="ghost" + disabled={index === count - 1} + onClick={() => onMove(1)} + > + <ArrowDown /> + </Button> + </SimpleTooltip> + <SimpleTooltip title={i18next.t("general:Delete")}> + <Button type="button" size="icon" variant="ghost" disabled={count === 1} onClick={onRemove}> + <Trash2 /> + </Button> + </SimpleTooltip> + </div> + </div> + ); +} + +/** + * Edits one routing rule. The pattern says which requests it covers and the + * ladder says where they go, top rung first: everything below it is what the + * request steps down to when the rung above is rate-limited, out of quota or + * down. + */ +export function RouteEditDialog({ + open, + onOpenChange, + route, + providers, + agents, + onSave, + saving, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + /** Null adds a rule rather than editing one. */ + route: ModelRoute | null; + providers: Provider[]; + agents: Agent[]; + onSave: (route: ModelRoute) => void; + saving?: boolean; +}) { + const [draft, setDraft] = React.useState<ModelRoute>(() => draftOf(route)); + + // The dialog is kept mounted between rows, so the draft is reset whenever it + // is opened on a different rule rather than on first render only. + React.useEffect(() => { + if (open) { + setDraft(draftOf(route)); + } + }, [open, route]); + + const adding = route === null; + + const providerOptions: SelectOption[] = providers.map(provider => ({ + value: providerIdOf(provider), + text: `${provider.displayName} ${provider.name}`, + label: ( + <span className="flex min-w-0 items-center gap-2"> + <ProviderIcon icon={provider.icon} baseUrl={provider.baseUrl} alt={provider.name} size={16} /> + <span className="truncate">{provider.displayName || provider.name}</span> + </span> + ), + })); + + const agentOptions: SelectOption[] = Array.from(new Set(agents.map(agent => agent.agentId || agent.name))) + .filter(id => id !== "") + .map(id => ({value: id, label: id})); + + const setTargets = (targets: RouteTarget[]) => setDraft(current => ({...current, targets: targets})); + + const changeTarget = (index: number, target: RouteTarget) => + setTargets(draft.targets.map((current, at) => (at === index ? target : current))); + + const moveTarget = (index: number, offset: number) => { + const targets = [...draft.targets]; + const [moved] = targets.splice(index, 1); + targets.splice(index + offset, 0, moved); + setTargets(targets); + }; + + const named = draft.targets.filter(target => target.provider !== "" || target.model !== ""); + + return ( + <FormDialog + open={open} + onOpenChange={onOpenChange} + size="lg" + title={adding ? i18next.t("llm:Add a routing rule") : i18next.t("llm:Edit routing rule")} + description={i18next.t("llm:Routing rule dialog description")} + onSubmit={() => onSave({...draft, targets: named})} + submitting={saving} + submitDisabled={draft.name.trim() === "" || draft.match.trim() === "" || named.length === 0} + > + <div className="flex flex-col gap-4"> + <div className="grid gap-4 sm:grid-cols-2"> + <Field label={i18next.t("general:Name")} htmlFor="route-name" required hint={i18next.t("llm:Rule name hint")}> + <Input + id="route-name" + // The name is the row's key, so an existing one is not editable: + // changing it would leave the old rule in place beside the new. + disabled={!adding} + className="font-mono" + placeholder="haiku-to-cheap" + value={draft.name} + onChange={event => setDraft(current => ({...current, name: event.target.value}))} + /> + </Field> + <Field label={i18next.t("general:Display name")} htmlFor="route-display-name"> + <Input + id="route-display-name" + placeholder={i18next.t("llm:Background work on the cheap model")} + value={draft.displayName} + onChange={event => setDraft(current => ({...current, displayName: event.target.value}))} + /> + </Field> + </div> + + <div className="grid gap-4 sm:grid-cols-2"> + <Field + label={i18next.t("llm:When the client asks for")} + htmlFor="route-match" + required + hint={i18next.t("llm:Model pattern hint")} + > + <Input + id="route-match" + className="font-mono" + placeholder="*haiku*" + value={draft.match} + onChange={event => setDraft(current => ({...current, match: event.target.value}))} + /> + </Field> + <Field label={i18next.t("agent:Agent")} htmlFor="route-agent" hint={i18next.t("llm:Rule agent hint")}> + <SearchSelect + id="route-agent" + allowClear + allowCustomValue + value={draft.agent} + onChange={agent => setDraft(current => ({...current, agent: agent}))} + options={agentOptions} + placeholder={i18next.t("llm:Every agent")} + /> + </Field> + </div> + + <div className="flex flex-col gap-2"> + <Label>{i18next.t("llm:Send it to")}</Label> + <p className="text-muted-foreground text-xs">{i18next.t("llm:Ladder hint")}</p> + <div className="flex flex-col gap-2"> + {draft.targets.map((target, index) => ( + <TargetRow + key={index} + target={target} + index={index} + count={draft.targets.length} + providers={providerOptions} + onChange={next => changeTarget(index, next)} + onMove={offset => moveTarget(index, offset)} + onRemove={() => setTargets(draft.targets.filter((_, at) => at !== index))} + /> + ))} + </div> + <div> + <Button + type="button" + size="sm" + variant="outline" + disabled={draft.targets.length >= maxTargets} + onClick={() => setTargets([...draft.targets, {provider: "", model: ""}])} + > + <Plus /> + {i18next.t("llm:Add a fallback step")} + </Button> + </div> + </div> + + <div className="flex items-center justify-between gap-4 rounded-md border px-3 py-2"> + <div className="flex flex-col"> + <Label htmlFor="route-enabled">{i18next.t("provider:Enabled")}</Label> + <span className="text-muted-foreground text-xs">{i18next.t("llm:Rule enabled hint")}</span> + </div> + <Switch + id="route-enabled" + checked={draft.enabled} + onCheckedChange={enabled => setDraft(current => ({...current, enabled: enabled}))} + /> + </div> + </div> + </FormDialog> + ); +} diff --git a/web/src/components/llm/route-preview-card.tsx b/web/src/components/llm/route-preview-card.tsx new file mode 100644 index 0000000..fb7de8a --- /dev/null +++ b/web/src/components/llm/route-preview-card.tsx @@ -0,0 +1,143 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as React from "react"; +import {Play} from "lucide-react"; +import i18next from "i18next"; + +import * as ModelRouteBackend from "@/backend/ModelRouteBackend"; +import {SearchSelect, type SelectOption} from "@/components/shared/simple-select"; +import {Badge} from "@/components/ui/badge"; +import {Button} from "@/components/ui/button"; +import {Card, CardContent, CardDescription, CardHeader, CardTitle} from "@/components/ui/card"; +import {Input} from "@/components/ui/input"; +import {Label} from "@/components/ui/label"; +import type {Agent, RoutePreviewStep} from "@/types"; + +/** + * Asks the gateway where a model name would actually go, without sending a + * request. The first line is what answers it today; the lines below are the + * downgrade, which is the part nobody sees until an upstream is already failing. + */ +export function RoutePreviewCard({agents}: {agents: Agent[]}) { + const [model, setModel] = React.useState(""); + const [agent, setAgent] = React.useState(""); + const [steps, setSteps] = React.useState<RoutePreviewStep[] | null>(null); + const [error, setError] = React.useState(""); + const [loading, setLoading] = React.useState(false); + + const agentOptions: SelectOption[] = Array.from(new Set(agents.map(entry => entry.agentId || entry.name))) + .filter(id => id !== "") + .map(id => ({value: id, label: id})); + + const run = () => { + if (model.trim() === "") { + return; + } + setLoading(true); + ModelRouteBackend.previewModelRoute(model.trim(), agent) + .then(res => { + if (res.status === "ok") { + setSteps(res.data ?? []); + setError(""); + } else { + setSteps(null); + setError(res.msg || i18next.t("general:Failed to get data")); + } + }) + .catch(failure => { + setSteps(null); + setError(failure.message || String(failure)); + }) + .then(() => setLoading(false)); + }; + + return ( + <Card> + <CardHeader> + <CardTitle>{i18next.t("llm:Where would this go?")}</CardTitle> + <CardDescription>{i18next.t("llm:Route preview description")}</CardDescription> + </CardHeader> + <CardContent className="flex flex-col gap-4"> + <form + className="grid gap-3 sm:grid-cols-[1fr_1fr_auto] sm:items-end" + onSubmit={event => { + event.preventDefault(); + run(); + }} + > + <div className="grid gap-2"> + <Label htmlFor="preview-model">{i18next.t("agent:Model")}</Label> + <Input + id="preview-model" + className="font-mono" + placeholder="claude-haiku-4-5" + value={model} + onChange={event => setModel(event.target.value)} + /> + </div> + <div className="grid gap-2"> + <Label htmlFor="preview-agent">{i18next.t("agent:Agent")}</Label> + <SearchSelect + id="preview-agent" + allowClear + allowCustomValue + value={agent} + onChange={setAgent} + options={agentOptions} + placeholder={i18next.t("llm:No agent, by model name")} + /> + </div> + <Button type="submit" loading={loading} disabled={model.trim() === ""}> + <Play /> + {i18next.t("llm:Trace it")} + </Button> + </form> + + {error ? <p className="text-destructive text-sm">{error}</p> : null} + + {steps !== null && error === "" ? ( + steps.length === 0 ? ( + <p className="text-muted-foreground text-sm">{i18next.t("llm:Nothing would answer this")}</p> + ) : ( + <ol className="flex flex-col gap-1.5"> + {steps.map((step, index) => ( + <li + key={`${step.provider}|${step.model}`} + className="flex flex-wrap items-center gap-2 rounded-md border px-3 py-2 text-sm" + > + <span className="text-muted-foreground w-6 shrink-0 tabular-nums">{index + 1}</span> + <span className="font-mono text-xs">{step.model}</span> + <span className="text-muted-foreground text-xs">@</span> + <span className="truncate">{step.displayName || step.provider}</span> + <span className="ml-auto flex shrink-0 items-center gap-1"> + {step.route ? <Badge variant="info">{step.route}</Badge> : null} + {index === 0 ? ( + <Badge variant="success">{i18next.t("llm:Answers it now")}</Badge> + ) : ( + <Badge variant="muted">{i18next.t("llm:Fallback")}</Badge> + )} + {step.suspended ? ( + <Badge variant="warning">{i18next.t("llm:In cooldown")}</Badge> + ) : null} + </span> + </li> + ))} + </ol> + ) + ) : null} + </CardContent> + </Card> + ); +} diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json index 8aed0fc..0feb9eb 100644 --- a/web/src/locales/en/data.json +++ b/web/src/locales/en/data.json @@ -1063,9 +1063,14 @@ "{count} skill sources": "{count} skill sources" }, "llm": { + "Add a fallback step": "Add a fallback step", + "Add a routing rule": "Add a routing rule", "All time": "All time", + "Answers it now": "Answers it now", "Any outcome": "Any outcome", + "Any provider serving it": "Any provider serving it", "Attempts": "Attempts", + "Background work on the cheap model": "Background work on the cheap model", "Bodies are not stored": "Request bodies are not stored", "Bodies are not stored detail": "llmRecordMode is \"metadata\", so the prompts, tool schemas and messages behind these rows were never kept. Store them from now on to inspect them.", "Bodies are stored": "Prompts and responses are being stored", @@ -1083,15 +1088,24 @@ "Clear": "Clear", "Cost": "Cost", "Delete every stored record?": "Delete every stored record?", + "Delete this routing rule detail": "The requests it covers go back to being matched by model name against every provider.", + "Delete this routing rule?": "Delete this routing rule?", + "Edit routing rule": "Edit routing rule", "Empty message": "This message carried no content.", "Endpoint": "Endpoint", "Error": "Error", + "Every agent": "Every agent", "Failed": "Failed", + "Fallback": "Fallback", "Feed connected": "Live feed connected", "Feed connecting": "Live feed reconnecting", + "First choice": "First", "Fresh input": "Fresh input", + "In cooldown": "In cooldown", "Input tokens": "Input tokens", + "Keep the requested model": "Keep the requested model", "LLM Records": "LLM Records", + "Ladder hint": "The first line is where the request goes. Every line below it is what it steps down to when the one above is rate-limited, out of quota or down.", "Last 24 hours": "Last 24 hours", "Last 7 days": "Last 7 days", "Last hour": "Last hour", @@ -1102,14 +1116,22 @@ "Mode metadata": "metadata only", "Mode off": "off", "Mode {mode}, kept for {days} days, up to {max} records": "Mode {mode}, kept for {days} days, up to {max} records", + "Model pattern hint": "The name the client sends. \"*\" stands for any run of characters, so *haiku* covers every version of it.", + "Model routing": "Model routing", + "Model routing description": "An agent picks its own model names - Claude Code asks for a small model for its background work - so these rules decide which model each request is actually sent, and what it falls back to when that one cannot answer.", + "Move down": "Move down", + "Move up": "Move up", + "No agent, by model name": "No agent, by model name", "No error message": "The upstream gave no error message", "No messages": "This request carried no messages.", "No parameters": "This request carried no parameters of its own.", "No price for this model": "No price is configured for this model", "No records yet": "No records yet - send a request through the LLM gateway.", "No response": "no response", + "No routing rules yet": "No routing rules yet", "No system prompt": "This request carried no system prompt.", "No tools": "The model was offered no tools.", + "Nothing would answer this": "Nothing would answer this model.", "Output": "Output", "Output tokens": "Output tokens", "Parameters": "Parameters", @@ -1128,9 +1150,19 @@ "Request failed": "Request failed", "Request size": "Request size", "Requests": "Requests", + "Route preview description": "Ask where a model name would go, without sending a request.", + "Routed by": "Routed by", + "Routed to": "Routed to", + "Routing rule dialog description": "A rule says which requests it covers and where they go. It decides routing, not permission: what an agent may ask for is set on the Permissions page.", + "Rule agent hint": "Leave it empty to cover every caller.", + "Rule enabled hint": "A disabled rule is kept but not read.", + "Rule name hint": "Identifies the rule and cannot be changed later.", + "Same model": "same model", + "Send it to": "Send it to", "Show all {chars}": "Show all {chars}", "Show less": "Show less", "Status": "Status", + "Step {n}": "Step {n}", "Succeeded": "Succeeded", "System prompt": "System prompt", "Thinking": "thinking", @@ -1138,8 +1170,11 @@ "Tool error": "tool error", "Tool result": "tool result", "Tools": "Tools", + "Trace it": "Trace it", "Unknown role": "unknown role", "Upstream response": "Upstream response", + "When the client asks for": "When the client asks for", + "Where would this go?": "Where would this go?", "all succeeded": "all succeeded", "blocks": "blocks", "chars": "chars", diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json index 8534184..5c45504 100644 --- a/web/src/locales/zh/data.json +++ b/web/src/locales/zh/data.json @@ -1132,9 +1132,14 @@ "{count} skill sources": "{count}个技能来源" }, "llm": { + "Add a fallback step": "添加降级步骤", + "Add a routing rule": "添加路由规则", "All time": "全部时间", + "Answers it now": "当前由它响应", "Any outcome": "全部结果", + "Any provider serving it": "任意提供该模型的渠道", "Attempts": "尝试次数", + "Background work on the cheap model": "后台任务走便宜模型", "Bodies are not stored": "未保存请求体", "Bodies are not stored detail": "llmRecordMode为\"metadata\"，因此这些记录背后的提示词、工具schema和消息都没有保存。开启保存后即可查看。", "Bodies are stored": "正在保存请求内容", @@ -1152,15 +1157,24 @@ "Clear": "清空", "Cost": "费用", "Delete every stored record?": "确定删除全部已保存的记录？", + "Delete this routing rule detail": "它覆盖的请求会重新按模型名去匹配所有渠道。", + "Delete this routing rule?": "删除这条路由规则？", + "Edit routing rule": "编辑路由规则", "Empty message": "该消息没有内容。", "Endpoint": "接口", "Error": "错误", + "Every agent": "所有Agent", "Failed": "失败", + "Fallback": "降级", "Feed connected": "实时推送已连接", "Feed connecting": "实时推送重连中", + "First choice": "首选", "Fresh input": "新输入", + "In cooldown": "冷却中", "Input tokens": "输入Token", + "Keep the requested model": "保持请求里的模型名", "LLM Records": "LLM记录", + "Ladder hint": "第一行是请求实际去的地方，下面每一行是上一行被限流、余额耗尽或不可用时降级到的目标。", "Last 24 hours": "最近24小时", "Last 7 days": "最近7天", "Last hour": "最近1小时", @@ -1171,14 +1185,22 @@ "Mode metadata": "仅元数据", "Mode off": "关闭", "Mode {mode}, kept for {days} days, up to {max} records": "模式：{mode}，保留{days}天，最多{max}条", + "Model pattern hint": "客户端发来的模型名，`*`匹配任意长度的字符，所以`*haiku*`能覆盖它的所有版本。", + "Model routing": "模型路由", + "Model routing description": "Agent会自己决定用哪个模型名——Claude Code的后台任务固定要小模型——所以这些规则决定每个请求真正被发往哪个模型，以及它响应不了时降级到什么。", + "Move down": "下移", + "Move up": "上移", + "No agent, by model name": "不指定Agent，按模型名", "No error message": "上游没有给出错误信息", "No messages": "该请求没有消息。", "No parameters": "该请求没有额外参数。", "No price for this model": "该模型没有配置价格", "No records yet": "暂无记录——请通过LLM网关发送一次请求。", "No response": "无响应", + "No routing rules yet": "还没有路由规则", "No system prompt": "该请求没有system prompt。", "No tools": "该请求没有向模型提供工具。", + "Nothing would answer this": "没有任何渠道会响应这个模型。", "Output": "输出", "Output tokens": "输出Token", "Parameters": "参数", @@ -1197,9 +1219,19 @@ "Request failed": "请求失败", "Request size": "请求大小", "Requests": "请求数", + "Route preview description": "不发真实请求，直接查一个模型名会被路由到哪里。", + "Routed by": "命中规则", + "Routed to": "实际发往", + "Routing rule dialog description": "一条规则说明它覆盖哪些请求、这些请求去哪里。它管的是路由而不是授权：Agent能用哪些模型在权限页面设置。", + "Rule agent hint": "留空则覆盖所有调用方。", + "Rule enabled hint": "停用的规则会保留，但不会被读取。", + "Rule name hint": "规则的标识，创建后不能修改。", + "Same model": "同名模型", + "Send it to": "路由到", "Show all {chars}": "展开全部{chars}", "Show less": "收起", "Status": "状态", + "Step {n}": "第{n}级", "Succeeded": "成功", "System prompt": "System prompt", "Thinking": "思考", @@ -1207,8 +1239,11 @@ "Tool error": "工具错误", "Tool result": "工具结果", "Tools": "工具", + "Trace it": "查看路径", "Unknown role": "未知角色", "Upstream response": "上游返回的内容", + "When the client asks for": "当客户端请求", + "Where would this go?": "这个模型会走到哪里？", "all succeeded": "全部成功", "blocks": "块", "chars": "字符", diff --git a/web/src/nav.ts b/web/src/nav.ts index 5fe35f2..2bc0b2f 100644 --- a/web/src/nav.ts +++ b/web/src/nav.ts @@ -24,6 +24,7 @@ import { MessageSquare, PackageCheck, Plug, + Route, Settings, ShieldCheck, ShieldHalf, @@ -80,6 +81,9 @@ export const navGroups: NavGroup[] = [ // somebody arriving with one gets their providers, and it is not a page they // would think to look for behind the add-provider dialog. {key: "/import", label: "link:Import settings", icon: ArrowDownToLine, path: "/import", adminOnly: true}, + // Beside Providers, because a rule here is what decides which of them answers + // a model name, and what the request steps down to when none of them can. + {key: "/model-routes", label: "llm:Model routing", icon: Route, path: "/model-routes", adminOnly: true}, { key: "/agent-configs", label: "agentConfig:Skills, MCP & Prompts", diff --git a/web/src/pages/LlmRecordsPage.tsx b/web/src/pages/LlmRecordsPage.tsx index d921b6b..f65fc8d 100644 --- a/web/src/pages/LlmRecordsPage.tsx +++ b/web/src/pages/LlmRecordsPage.tsx @@ -97,6 +97,7 @@ function RecordFailure({record, errorBody}: {record: LlmRecord; errorBody: strin <div key={index} className="flex min-w-0 flex-wrap items-center gap-2 text-xs"> <Badge variant={statusVariant(failure.status)}>{failure.status || i18next.t("llm:No response")}</Badge> <CodeText>{failure.provider}</CodeText> + {failure.model ? <CodeText>{failure.model}</CodeText> : null} <span className="text-muted-foreground min-w-0 break-all">{failure.error}</span> </div> ))} @@ -147,6 +148,11 @@ function RecordDetail({record, onDelete}: {record: LlmRecord; onDelete: () => vo {label: i18next.t("general:Created time"), value: Setting.getFormattedDate(record.createdTime)}, {label: i18next.t("llm:Endpoint"), value: <CodeText>{`${record.protocol} ${record.endpoint}`}</CodeText>}, record.provider && {label: i18next.t("llm:Provider"), value: <CodeText>{record.provider}</CodeText>}, + record.upstreamModel && { + label: i18next.t("llm:Routed to"), + value: <CodeText>{record.upstreamModel}</CodeText>, + }, + record.route && {label: i18next.t("llm:Routed by"), value: <CodeText>{record.route}</CodeText>}, record.agent && {label: i18next.t("agent:Agent"), value: <CodeText>{record.agent}</CodeText>}, {label: i18next.t("llm:Attempts"), value: record.attempts}, { diff --git a/web/src/pages/ModelRoutesPage.tsx b/web/src/pages/ModelRoutesPage.tsx new file mode 100644 index 0000000..1dddf6b --- /dev/null +++ b/web/src/pages/ModelRoutesPage.tsx @@ -0,0 +1,243 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import * as React from "react"; +import {ArrowRight, Pencil, Plus, Route, Trash2} from "lucide-react"; +import i18next from "i18next"; + +import * as ModelRouteBackend from "@/backend/ModelRouteBackend"; +import * as ProviderBackend from "@/backend/ProviderBackend"; +import * as Setting from "@/Setting"; +import {RouteEditDialog} from "@/components/llm/route-edit-dialog"; +import {RoutePreviewCard} from "@/components/llm/route-preview-card"; +import {ConfirmDialog} from "@/components/shared/confirm-dialog"; +import {DataTable, type Column} from "@/components/shared/data-table"; +import {PageContainer, PageHeader} from "@/components/shared/page-header"; +import {UnauthorizedResult} from "@/components/shared/misc"; +import {MessageAlert} from "@/components/ui/alert"; +import {Badge} from "@/components/ui/badge"; +import {Button} from "@/components/ui/button"; +import {SimpleTooltip} from "@/components/ui/tooltip"; +import {useAgents} from "@/lib/agents"; +import type {Account, ModelRoute, Provider, RouteTarget} from "@/types"; + +/** One rung as the table shows it: the model asked for, and who has to answer. */ +function TargetChip({target, providers}: {target: RouteTarget; providers: Provider[]}) { + const provider = providers.find(candidate => `${candidate.owner}/${candidate.name}` === target.provider); + const on = provider ? provider.displayName || provider.name : target.provider; + + return ( + <span className="bg-muted/50 inline-flex max-w-full items-center gap-1 rounded-md border px-1.5 py-0.5"> + <span className="truncate font-mono text-xs">{target.model || i18next.t("llm:Same model")}</span> + {target.provider ? <span className="text-muted-foreground truncate text-xs">@{on}</span> : null} + </span> + ); +} + +/** + * The rules that decide which model a request is actually sent, and what it + * steps down to when that cannot answer. The preview beside them is what makes + * a ladder readable: the rungs below the first are the part nobody sees until + * an upstream is already failing. + */ +export default function ModelRoutesPage({account}: {account: Account}) { + const isAdmin = Setting.isAdminUser(account); + const [routes, setRoutes] = React.useState<ModelRoute[]>([]); + const [providers, setProviders] = React.useState<Provider[]>([]); + const [loading, setLoading] = React.useState(true); + const [error, setError] = React.useState(""); + const [editing, setEditing] = React.useState<ModelRoute | null>(null); + const [dialogOpen, setDialogOpen] = React.useState(false); + const [saving, setSaving] = React.useState(false); + const {agents} = useAgents(isAdmin); + + const load = React.useCallback(() => { + if (!isAdmin) { + return; + } + ModelRouteBackend.getModelRoutes() + .then(res => { + if (res.status === "ok") { + setRoutes(res.data ?? []); + setError(""); + } else { + setError(res.msg || i18next.t("general:Failed to get data")); + } + }) + .catch(failure => setError(failure.message || String(failure))) + .then(() => setLoading(false)); + }, [isAdmin]); + + React.useEffect(() => { + load(); + if (isAdmin) { + ProviderBackend.getProviders(account.name) + .then(res => setProviders(res.status === "ok" ? (res.data ?? []) : [])) + .catch(() => undefined); + } + }, [load, isAdmin, account]); + + if (!isAdmin) { + return <UnauthorizedResult />; + } + + const save = (route: ModelRoute) => { + setSaving(true); + const call = editing + ? ModelRouteBackend.updateModelRoute(editing.name, route) + : ModelRouteBackend.addModelRoute(route); + call + .then(res => { + if (res.status === "ok") { + Setting.showMessage("success", i18next.t("general:Successfully saved")); + setDialogOpen(false); + load(); + } else { + Setting.showMessage("error", `${i18next.t("general:Failed to save")}: ${res.msg}`); + } + }) + .catch(failure => Setting.showMessage("error", `${i18next.t("general:Failed to save")}: ${failure}`)) + .then(() => setSaving(false)); + }; + + const remove = (name: string) => + ModelRouteBackend.deleteModelRoute(name) + .then(res => { + if (res.status === "ok") { + Setting.showMessage("success", i18next.t("general:Deleted successfully")); + load(); + } else { + Setting.showMessage("error", `${i18next.t("general:Failed to delete")}: ${res.msg}`); + } + }) + .catch(failure => Setting.showMessage("error", `${i18next.t("general:Failed to delete")}: ${failure}`)); + + const open = (route: ModelRoute | null) => { + setEditing(route); + setDialogOpen(true); + }; + + const columns: Column<ModelRoute>[] = [ + { + key: "match", + title: i18next.t("llm:When the client asks for"), + render: (_value, route) => ( + <div className="flex min-w-0 flex-col"> + <span className="truncate font-mono text-xs">{route.match}</span> + {route.displayName && route.displayName !== route.name ? ( + <span className="text-muted-foreground truncate text-xs">{route.displayName}</span> + ) : null} + </div> + ), + }, + { + key: "targets", + title: i18next.t("llm:Send it to"), + render: (_value, route) => ( + <div className="flex min-w-0 flex-wrap items-center gap-1"> + {route.targets.map((target, index) => ( + <React.Fragment key={index}> + {index > 0 ? <ArrowRight className="text-muted-foreground size-3 shrink-0" /> : null} + <TargetChip target={target} providers={providers} /> + </React.Fragment> + ))} + </div> + ), + }, + { + key: "agent", + title: i18next.t("agent:Agent"), + width: "140px", + render: (_value, route) => + route.agent ? ( + <span className="truncate font-mono text-xs">{route.agent}</span> + ) : ( + <span className="text-muted-foreground text-xs">{i18next.t("llm:Every agent")}</span> + ), + }, + { + key: "enabled", + title: i18next.t("provider:Status"), + width: "110px", + render: (_value, route) => ( + <Badge variant={route.enabled ? "success" : "muted"}> + {i18next.t(route.enabled ? "provider:Enabled" : "provider:Disabled")} + </Badge> + ), + }, + { + key: "actions", + title: "", + width: "96px", + align: "right", + render: (_value, route) => ( + <div className="flex items-center justify-end gap-1"> + <SimpleTooltip title={i18next.t("general:Edit")}> + <Button size="icon" variant="ghost" onClick={() => open(route)}> + <Pencil /> + </Button> + </SimpleTooltip> + <ConfirmDialog + title={i18next.t("llm:Delete this routing rule?")} + description={i18next.t("llm:Delete this routing rule detail")} + onConfirm={() => remove(route.name)} + > + <Button size="icon" variant="ghost"> + <Trash2 /> + </Button> + </ConfirmDialog> + </div> + ), + }, + ]; + + return ( + <PageContainer> + <PageHeader + title={i18next.t("llm:Model routing")} + description={i18next.t("llm:Model routing description")} + actions={ + <Button onClick={() => open(null)}> + <Plus /> + {i18next.t("llm:Add a routing rule")} + </Button> + } + /> + + {error ? <MessageAlert title={error} /> : null} + + <DataTable + columns={columns} + dataSource={routes} + rowKey={route => route.name} + loading={loading} + pageSize={25} + emptyIcon={Route} + emptyText={i18next.t("llm:No routing rules yet")} + /> + + <RoutePreviewCard agents={agents} /> + + <RouteEditDialog + open={dialogOpen} + onOpenChange={setDialogOpen} + route={editing} + providers={providers} + agents={agents} + onSave={save} + saving={saving} + /> + </PageContainer> + ); +} diff --git a/web/src/types.ts b/web/src/types.ts index c78f18b..9fba496 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -525,6 +525,8 @@ export interface AgentTranscript { export interface LlmFailure { provider: string; + /** What that attempt asked for, which is not always what the client named. */ + model: string; status: number; error: string; } @@ -535,6 +537,11 @@ export interface LlmRecord { protocol: string; endpoint: string; model: string; + /** What the provider was actually asked for, set only when a rule or a + * downgrade put another name on the wire. The cost is priced against it. */ + upstreamModel: string; + /** The routing rule that chose the provider and model, empty when none did. */ + route: string; provider: string; agent: string; clientIp: string; @@ -1691,3 +1698,39 @@ export interface ConnectRequest { credentials: Record<string, string>; agents: string[]; } + +/** One rung of a route: the model to ask for, and optionally the only provider + * allowed to answer it. An empty model keeps the name the client sent. */ +export interface RouteTarget { + provider: string; + model: string; +} + +/** + * One rule for what the gateway actually asks for when a client names a model. + * `targets` is a ladder, most preferred first: the rungs below the first are + * the automatic downgrade, taken when the rung above cannot answer. + */ +export interface ModelRoute { + name: string; + createdTime: string; + updatedTime: string; + displayName: string; + /** The model the client asks for, with "*" standing for any run of characters. */ + match: string; + /** Limits the rule to one agent id. Empty applies it to every caller. */ + agent: string; + targets: RouteTarget[]; + sort: number; + enabled: boolean; +} + +/** One attempt of the plan a request would be relayed by, as the preview shows it. */ +export interface RoutePreviewStep { + provider: string; + displayName: string; + model: string; + /** The rule that put this step in the plan, empty when none did. */ + route: string; + suspended: boolean; +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
