This is an automated email from the ASF dual-hosted git repository. hsluoyz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
commit a39a88962d9eccd944a198876ea356746b7a2e9d Author: Yang Luo <[email protected]> AuthorDate: Sun Sep 6 01:20:36 2026 +0800 feat(proxy): speak the Responses API to a provider, not just to a client --- README.md | 2 + README_zh.md | 2 + controllers/agent_provider.go | 4 +- controllers/proxy_record.go | 20 +- object/provider.go | 32 ++- object/provider_probe.go | 2 +- object/provider_probe_answer.go | 2 +- object/provider_probe_checks.go | 6 +- object/provider_probe_run.go | 4 +- protocol/protocol.go | 10 +- protocol/responses.go | 380 +++++++++++++++++++++++++++- web/src/components/shared/brand-options.tsx | 11 + web/src/lib/providers.ts | 31 ++- web/src/locales/en/data.json | 3 + web/src/locales/zh/data.json | 3 + web/src/pages/ProviderEditPage.tsx | 25 +- web/src/pages/ProviderListPage.tsx | 9 +- web/src/types.ts | 2 + 18 files changed, 505 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index e1b22ff..73756cb 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,8 @@ The token is Gateway's own relay token, not a vendor key: the agent refuses to s One base URL answers whichever API the agent speaks: `/chat/completions` for an OpenAI client, `/v1/messages` for an Anthropic one, `/responses` for Codex, which speaks nothing else since it dropped the chat completions wire format, and `/v1beta/models/<model>:generateContent` for the Gemini CLI, which speaks only Google's own API. The API the provider serves need not be the same one: Gateway translates between all four, in both directions and for streamed answers too, so Codex runs on D [...] +Upstream, which API Gateway calls is the provider's own setting: **Upstream API** defaults to the one the provider type implies, and can be set to Responses for OpenAI itself or for a relay that accounts cached input on `/responses` more accurately than on `/chat/completions`. A Codex request then meets a Responses provider on both sides and passes through byte for byte, cache counters included. + ### No API key: keep the sign-in the agent already has An agent signed in with a ChatGPT or Claude subscription has no API key to paste. Set the provider's **Authentication** to **the caller's own login** and it needs none: the base URL points at the vendor, and every request is forwarded with the credentials the agent itself sent, so it keeps its own sign-in. Leave **Models** empty and the provider accepts any model name. diff --git a/README_zh.md b/README_zh.md index 8b988de..bb4db03 100644 --- a/README_zh.md +++ b/README_zh.md @@ -259,6 +259,8 @@ export ANTHROPIC_AUTH_TOKEN="cg-..." 同一个 base URL 会按 Agent 说的那套 API 应答：OpenAI 客户端走 `/chat/completions`，Anthropic 客户端走 `/v1/messages`，Codex 走 `/responses` —— 它去掉了 chat completions 这套线格式，只剩这一种，Gemini CLI 走 `/v1beta/models/<model>:generateContent` —— 它只会说 Google 自己那套 API。Provider 用的是哪套 API 不必与之相同：Gateway 会在这四套之间双向转换，流式回复也一样，所以 Codex 能跑在 DeepSeek、Kimi、Qwen 上，Claude Code 同样能跑在它们上面。如果 Provider 正好就说请求进来的那套 API，就原样转发，一个字节都不改。唯一由 Gateway 自己应答的是 Anthropic 和 Gemini 客户端每轮之前问的 token 数：绑定的 Provider 没有对应接口时，由 Gateway 估算。 +往上游发哪套接口由Provider自己定：**上游接口**默认按Provider类型推断，也可以设成Responses——OpenAI自己就服务这套，一部分中转站在`/responses`上算缓存也比在`/chat/completions`上准。这时Codex的请求两头都是Responses，原样透传，缓存计数一并保留。 + ### 没有 API Key：沿用 Agent 已有的登录 用 ChatGPT 或 Claude 订阅登录的 Agent 根本没有 API Key 可填。把 Provider 的**认证方式**设成**调用方自己的登录**，它就不需要 Key：base URL 指向厂商，每个请求都带着 Agent 自己发来的凭据转发上游，Agent 继续用它已有的登录。**Models** 留空，这个 Provider 就接受任何模型名。 diff --git a/controllers/agent_provider.go b/controllers/agent_provider.go index db46ec4..95d3cd9 100644 --- a/controllers/agent_provider.go +++ b/controllers/agent_provider.go @@ -129,7 +129,7 @@ func checkAgentProtocol(agentId string, mode string, providerIds []string) error if provider == nil { continue } - if spoken := object.ProviderProtocol(provider); spoken != spokenByAgent { + if spoken := object.ProviderApiFamily(provider); spoken != spokenByAgent { return fmt.Errorf("%s speaks the %s API, but provider %s speaks %s: bind a provider that speaks %s, or route this agent through the gateway", agentId, spokenByAgent, id, spoken, spokenByAgent) } @@ -157,7 +157,7 @@ func agentEndpoint(agentId string) (agentprovider.Endpoint, error) { } endpoint.Provider = provider.GetId() - endpoint.Protocol = object.ProviderProtocol(provider) + endpoint.Protocol = object.ProviderApiFamily(provider) if len(provider.Models) > 0 { endpoint.Model = provider.Models[0] endpoint.Models = provider.Models diff --git a/controllers/proxy_record.go b/controllers/proxy_record.go index 0774d2c..7c89eda 100644 --- a/controllers/proxy_record.go +++ b/controllers/proxy_record.go @@ -106,11 +106,12 @@ func (route *proxyRoute) recordUsage(tail []byte) { record.CacheWriteTokens = usage.CacheWriteTokens record.CacheReadTokens = higher(usage.CacheReadInputTokens, usage.CachedTokens) - // OpenAI counts cached input inside prompt_tokens, Anthropic reports it - // beside input_tokens. Taking it out where it is included leaves the four - // counters disjoint, so each can be priced at its own rate. + // OpenAI counts cached input inside its prompt total, Anthropic reports it + // beside one. Only the OpenAI spellings fill CachedTokens, so taking it out + // where that is set leaves the four counters disjoint, each priced at its + // own rate. record.PromptTokens = higher(usage.PromptTokens, usage.InputTokens) - if usage.PromptTokens > 0 && usage.CachedTokens > 0 { + if usage.CachedTokens > 0 { record.PromptTokens = higher(record.PromptTokens-usage.CachedTokens, 0) } @@ -171,6 +172,13 @@ type llmUsage struct { CompletionTokensDetails struct { ReasoningTokens int `json:"reasoning_tokens"` } `json:"completion_tokens_details"` + // The Responses API spells the same two details. + InputTokensDetails struct { + CachedTokens int `json:"cached_tokens"` + } `json:"input_tokens_details"` + OutputTokensDetails struct { + ReasoningTokens int `json:"reasoning_tokens"` + } `json:"output_tokens_details"` // Flattened out of the detail objects above by normalize. CachedTokens int `json:"-"` @@ -179,8 +187,8 @@ type llmUsage struct { // normalize lifts the nested counters up onto one flat set of fields. func (usage *llmUsage) normalize() { - usage.CachedTokens = usage.PromptTokensDetails.CachedTokens - usage.ReasoningTokens = usage.CompletionTokensDetails.ReasoningTokens + usage.CachedTokens = higher(usage.PromptTokensDetails.CachedTokens, usage.InputTokensDetails.CachedTokens) + usage.ReasoningTokens = higher(usage.CompletionTokensDetails.ReasoningTokens, usage.OutputTokensDetails.ReasoningTokens) if usage.CacheWriteTokens == 0 { usage.CacheWriteTokens = usage.CacheCreation.Ephemeral5m + usage.CacheCreation.Ephemeral1h } diff --git a/object/provider.go b/object/provider.go index 27e1081..b870393 100644 --- a/object/provider.go +++ b/object/provider.go @@ -203,6 +203,7 @@ func UsesClientAuth(provider *Provider) bool { const ( ProtocolOpenAi = protocol.OpenAi ProtocolAnthropic = protocol.Anthropic + ProtocolResponses = protocol.Responses ) // IsProviderTypeSupported reports whether the gateway can talk to the provider's @@ -211,21 +212,34 @@ func IsProviderTypeSupported(provider *Provider) bool { return containsString(providerTypes, provider.Type) } -// ProviderProtocol is the wire format a provider's upstream speaks. Everything -// that is not Anthropic is reached with an OpenAI-formatted request. +// ProviderProtocol is the wire format a provider's upstream is talked to in. +// The provider names it when its upstream serves more than its type implies; +// otherwise everything that is not Anthropic is sent an OpenAI request. func ProviderProtocol(provider *Provider) string { + if protocol.IsUpstream(provider.Protocol) { + return provider.Protocol + } if provider.Type == "anthropic" { return ProtocolAnthropic } return ProtocolOpenAi } +// ProviderApiFamily is the family a provider's base URL belongs to, which is +// what an agent configuration is written against: an agent pointed straight at +// a Responses provider still writes an OpenAI base URL. +func ProviderApiFamily(provider *Provider) string { + if spoken := ProviderProtocol(provider); spoken != ProtocolResponses { + return spoken + } + return ProtocolOpenAi +} + // ServesResponsesApi reports whether the provider's own upstream answers on the -// OpenAI Responses API. Only OpenAI itself does; the OpenAI-compatible vendors -// stop at chat completions, which the gateway translates for the clients that -// speak nothing else. +// OpenAI Responses API. The other vendors stop at chat completions, which the +// gateway translates for the clients that speak nothing else. func ServesResponsesApi(provider *Provider) bool { - return provider.Type == "openai" + return provider.Type == "openai" || ProviderProtocol(provider) == ProtocolResponses } func containsString(values []string, value string) bool { @@ -247,6 +261,9 @@ type Provider struct { DisplayName string `xorm:"varchar(100)" json:"displayName"` Type string `xorm:"varchar(100)" json:"type"` BaseUrl string `xorm:"varchar(255)" json:"baseUrl"` + // Protocol is the wire format the upstream is talked to in, when it is not + // the one Type implies. + Protocol string `xorm:"varchar(100)" json:"protocol"` // ApiKey holds base64 ciphertext, not the bare key, when // "apiKeyEncryptionKey" is set in app.conf, hence the wider column. ApiKey string `xorm:"varchar(1000)" json:"apiKey"` @@ -358,6 +375,9 @@ func validateProvider(provider *Provider) error { if !containsString(providerTypes, provider.Type) { return fmt.Errorf("invalid provider type: %s", provider.Type) } + if provider.Protocol != "" && !protocol.IsUpstream(provider.Protocol) { + return fmt.Errorf("the gateway cannot speak the %s API to a provider", provider.Protocol) + } if !containsString(providerStatuses, provider.Status) { return fmt.Errorf("invalid provider status: %s", provider.Status) } diff --git a/object/provider_probe.go b/object/provider_probe.go index fd3bca1..0d23a58 100644 --- a/object/provider_probe.go +++ b/object/provider_probe.go @@ -447,7 +447,7 @@ func newProviderProbe(provider *Provider, trigger string) *ProviderProbe { Provider: provider.GetId(), CreatedTime: util.GetCurrentTime(), Trigger: trigger, - Protocol: ProviderProtocol(provider), + Protocol: ProviderApiFamily(provider), VendorHeaders: []string{}, Checks: []ProbeCheck{}, } diff --git a/object/provider_probe_answer.go b/object/provider_probe_answer.go index 54b6693..5a08186 100644 --- a/object/provider_probe_answer.go +++ b/object/provider_probe_answer.go @@ -60,7 +60,7 @@ type probeReply struct { // finding here: the engines below decide what an unanswered question means, // which is not the same for all of them. func probeAsk(provider *Provider, model string, probe *ProviderProbe, probeCase *ProbeCase) *probeReply { - upstream := ProviderProtocol(provider) + upstream := ProviderApiFamily(provider) answer, err := sendProbe(provider, probeAskBody(upstream, model, probeCase), false) if err != nil { return &probeReply{failure: auditutil.BoundString(err.Error(), probeErrorChars)} diff --git a/object/provider_probe_checks.go b/object/provider_probe_checks.go index cb0fb04..cd397ba 100644 --- a/object/provider_probe_checks.go +++ b/object/provider_probe_checks.go @@ -297,7 +297,7 @@ func probeToolCase( probeCase *ProbeCase, opening bool, ) (*probeBody, http.Header) { - upstream := ProviderProtocol(provider) + upstream := ProviderApiFamily(provider) answer, err := sendProbe(provider, probeToolBody(upstream, model, probeCase), false) if err != nil || !answer.ok() { reason := "" @@ -626,7 +626,7 @@ func probeStreamBody(upstream string, model string, probeCase *ProbeCase) map[st } func probeStreamShape(provider *Provider, model string, probe *ProviderProbe, probeCase *ProbeCase) { - upstream := ProviderProtocol(provider) + upstream := ProviderApiFamily(provider) answer, err := sendProbe(provider, probeStreamBody(upstream, model, probeCase), true) if err != nil || !answer.ok() { probe.addCheck(checkOf(probeCase, LlmAuditUnknown)) @@ -861,7 +861,7 @@ func probeCachePairOf( probe *ProviderProbe, probeCase *ProbeCase, ) probeCachePair { - upstream := ProviderProtocol(provider) + upstream := ProviderApiFamily(provider) filler := probeFiller(probeCase.Params.FillerChars) prompt := firstString(probeCase.Params.Prompt, probeCachePrompt) limit := firstInt(probeCase.Params.MaxTokens, probeCacheMaxTokens) diff --git a/object/provider_probe_run.go b/object/provider_probe_run.go index 515bd4a..2ee2d53 100644 --- a/object/provider_probe_run.go +++ b/object/provider_probe_run.go @@ -72,7 +72,7 @@ func runProviderProbe(provider *Provider, trigger string) *ProviderProbe { } probe.Model = model - cases := probeCasesFor(ProviderProtocol(provider), model) + cases := probeCasesFor(ProviderApiFamily(provider), model) if len(cases) == 0 { probe.Error = "no test case is enabled for this upstream API" return probe @@ -171,7 +171,7 @@ func sendProbe(provider *Provider, body map[string]any, stream bool) (*probeAnsw } func postProbe(provider *Provider, body map[string]any, stream bool) (*probeAnswer, error) { - upstream := ProviderProtocol(provider) + upstream := ProviderApiFamily(provider) endpoint := "/chat/completions" if upstream == ProtocolAnthropic { endpoint = "/v1/messages" diff --git a/protocol/protocol.go b/protocol/protocol.go index 9ad725e..2b049df 100644 --- a/protocol/protocol.go +++ b/protocol/protocol.go @@ -29,8 +29,8 @@ import ( const ( OpenAi = "openai" Anthropic = "anthropic" - // Responses is the OpenAI Responses API, which recent Codex versions speak. - // No provider type serves it, so it is a client format only. + // Responses is the OpenAI Responses API, which recent Codex versions speak + // and which OpenAI itself and some relays serve upstream. Responses = "responses" // Gemini is Google's generateContent API, which the Gemini CLI speaks. Like // Responses it is a client format only: a Gemini provider is reached through @@ -94,6 +94,12 @@ func Of(name string) Codec { return codecs[name] } +// IsUpstream reports whether a provider can be talked to in this format. +func IsUpstream(name string) bool { + _, ok := upstreams[name] + return ok +} + // UpstreamOf is the codec for a provider format. func UpstreamOf(name string) (Upstream, error) { upstream, ok := upstreams[name] diff --git a/protocol/responses.go b/protocol/responses.go index 4c76100..0520a5a 100644 --- a/protocol/responses.go +++ b/protocol/responses.go @@ -13,9 +13,8 @@ // limitations under the License. // This file speaks the OpenAI Responses API, the only wire format recent Codex -// versions know. It is a client format only: the gateway reads requests in it -// and writes answers back in it, but no provider type serves it, so a request -// that arrives here is always translated for whichever upstream answers it. +// versions know. It works both ways: as a client format, and as the upstream of +// a provider that answers on /responses rather than on chat completions. package protocol @@ -36,6 +35,8 @@ func init() { func (responsesCodec) Name() string { return Responses } +func (responsesCodec) Endpoint() string { return "/responses" } + // --------------------------------------------------------------------------- // Request // --------------------------------------------------------------------------- @@ -430,11 +431,7 @@ func (answer *responsesAnswer) response(status string, output []any) map[string] "status": status, "model": answer.model, "output": output, } if !answer.usage.IsZero() { - response["usage"] = map[string]any{ - "input_tokens": answer.usage.InputTokens, - "output_tokens": answer.usage.OutputTokens, - "total_tokens": answer.usage.InputTokens + answer.usage.OutputTokens, - } + response["usage"] = responsesUsageOf(answer.usage) } if answer.failure != nil { response["error"] = map[string]any{ @@ -619,3 +616,370 @@ func (writer *responsesWriter) Close() { func (responsesCodec) EncodeError(kind string, message string) []byte { return openAiCodec{}.EncodeError(kind, message) } + +// --------------------------------------------------------------------------- +// Upstream +// --------------------------------------------------------------------------- + +// responsesBody is the request written for an upstream serving this API. store +// is false because the gateway never sends a previous_response_id, so a copy +// kept upstream would never be read. +type responsesBody struct { + Model string `json:"model"` + Stream bool `json:"stream,omitempty"` + Store bool `json:"store"` + Instructions string `json:"instructions,omitempty"` + Input []any `json:"input"` + Tools []responsesTool `json:"tools,omitempty"` + ToolChoice json.RawMessage `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + MaxOutputTokens *int `json:"max_output_tokens,omitempty"` + Text json.RawMessage `json:"text,omitempty"` + Reasoning json.RawMessage `json:"reasoning,omitempty"` +} + +func (responsesCodec) EncodeRequest(request *Request) ([]byte, error) { + body := responsesBody{ + Model: request.Model, + Stream: request.Stream, + Instructions: strings.Join(request.System, "\n\n"), + Input: []any{}, + ParallelToolCalls: request.ParallelToolCalls, + Temperature: request.Temperature, + TopP: request.TopP, + MaxOutputTokens: request.MaxTokens, + } + for _, message := range request.Messages { + body.Input = append(body.Input, responsesInputOf(message)...) + } + if len(body.Input) == 0 { + return nil, errors.New("the request carries no message") + } + + for _, tool := range request.Tools { + parameters := tool.Parameters + if len(parameters) == 0 { + parameters = rawValue(map[string]any{"type": "object", "properties": map[string]any{}}) + } + body.Tools = append(body.Tools, responsesTool{ + Type: "function", Name: tool.Name, Description: tool.Description, Parameters: parameters, + }) + } + if choice := request.ToolChoice; choice != nil { + switch choice.Mode { + case ChoiceTool: + body.ToolChoice = rawValue(map[string]any{"type": "function", "name": choice.Name}) + case ChoiceAuto, ChoiceNone, ChoiceRequired: + body.ToolChoice = rawString(choice.Mode) + } + } + if format := request.Format; format != nil && format.Kind != "" { + // This API keeps the name and the schema beside the type rather than + // one level down, as chat completions does. + written := map[string]any{"type": format.Kind} + if format.Kind == "json_schema" { + written["name"] = emptyAs(format.Name, "response") + written["schema"] = format.Schema + if format.Strict != nil { + written["strict"] = *format.Strict + } + } + body.Text = rawValue(map[string]any{"format": written}) + } + if effort := request.Reasoning.EffortOf(); effort != "" { + body.Reasoning = rawValue(map[string]any{"effort": effort}) + } + return json.Marshal(body) +} + +// responsesInputOf writes one canonical message out. A tool call and a tool +// result are items of their own here rather than content of a message, so a +// message carrying either turns into several. +func responsesInputOf(message Message) []any { + items := []any{} + parts := []any{} + partType := "input_text" + if message.Role == RoleAssistant { + partType = "output_text" + } + + flush := func() { + if len(parts) == 0 { + return + } + items = append(items, map[string]any{ + "type": "message", "role": message.Role, "content": parts, + }) + parts = []any{} + } + + for _, content := range message.Content { + switch content.Kind { + case KindText: + if content.Text == "" { + continue + } + parts = append(parts, map[string]any{"type": partType, "text": content.Text}) + case KindImage: + if content.Image == nil { + continue + } + parts = append(parts, map[string]any{"type": "input_image", "image_url": content.Image.Link()}) + case KindToolUse: + if content.ToolUse == nil { + continue + } + flush() + items = append(items, map[string]any{ + "type": "function_call", "call_id": content.ToolUse.Id, + "name": content.ToolUse.Name, "arguments": emptyAsObject(content.ToolUse.Arguments), + }) + case KindToolResult: + if content.ToolResult == nil { + continue + } + flush() + items = append(items, map[string]any{ + "type": "function_call_output", "call_id": content.ToolResult.Id, + "output": content.ToolResult.Text, + }) + } + // A thinking block is left out: this API only takes back reasoning + // items of its own, which carry an id no other upstream signed. + } + + flush() + return items +} + +type responsesResult struct { + Id string `json:"id"` + Model string `json:"model"` + Status string `json:"status"` + Output []responsesOutputItem `json:"output"` + Usage *responsesUsage `json:"usage"` + IncompleteDetails *struct { + Reason string `json:"reason"` + } `json:"incomplete_details"` + Error *responsesFailure `json:"error"` +} + +type responsesFailure struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (failure *responsesFailure) canonical() *Failure { + if failure == nil || failure.Message == "" { + return nil + } + return &Failure{Kind: emptyAs(failure.Code, "server_error"), Message: failure.Message} +} + +// responsesOutputItem is one item of the output list: the assistant message, +// the reasoning, or one function call. +type responsesOutputItem struct { + Type string `json:"type"` + Role string `json:"role"` + Content []responsesOutputPart `json:"content"` + Summary []responsesOutputPart `json:"summary"` + Id string `json:"id"` + CallId string `json:"call_id"` + Name string `json:"name"` + Arguments string `json:"arguments"` +} + +type responsesOutputPart struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type responsesUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + TotalTokens int `json:"total_tokens"` + InputTokensDetails *struct { + CachedTokens int `json:"cached_tokens"` + } `json:"input_tokens_details"` + OutputTokensDetails *struct { + ReasoningTokens int `json:"reasoning_tokens"` + } `json:"output_tokens_details"` +} + +func (usage *responsesUsage) canonical() Usage { + canonical := Usage{InputTokens: usage.InputTokens, OutputTokens: usage.OutputTokens} + if usage.InputTokensDetails != nil { + canonical.CacheReadTokens = usage.InputTokensDetails.CachedTokens + } + if usage.OutputTokensDetails != nil { + canonical.ReasoningTokens = usage.OutputTokensDetails.ReasoningTokens + } + return canonical +} + +// responsesUsageOf writes the counters back out. This format counts the cached +// part inside the input total, which is where the canonical form keeps it too. +func responsesUsageOf(usage Usage) map[string]any { + input := usage.InputTokens + usage.CacheWriteTokens + written := map[string]any{ + "input_tokens": input, + "output_tokens": usage.OutputTokens, + "total_tokens": input + usage.OutputTokens, + } + if usage.CacheReadTokens > 0 { + written["input_tokens_details"] = map[string]any{"cached_tokens": usage.CacheReadTokens} + } + if usage.ReasoningTokens > 0 { + written["output_tokens_details"] = map[string]any{"reasoning_tokens": usage.ReasoningTokens} + } + return written +} + +func (responsesCodec) DecodeResponse(raw []byte) (*Response, error) { + var body responsesResult + if err := json.Unmarshal(raw, &body); err != nil { + return nil, errors.New("the upstream answered with an unreadable body") + } + + response := &Response{Id: body.Id, Model: body.Model, StopReason: StopEnd} + if body.Usage != nil { + response.Usage = body.Usage.canonical() + } + response.Failure = body.Error.canonical() + + for _, item := range body.Output { + switch item.Type { + case "message": + for _, part := range item.Content { + if part.Text != "" { + response.Content = append(response.Content, Content{Kind: KindText, Text: part.Text}) + } + } + case "reasoning": + // A model that hides its reasoning sends only the summary. + for _, part := range append(append([]responsesOutputPart{}, item.Content...), item.Summary...) { + if part.Text != "" { + response.Content = append(response.Content, Content{Kind: KindThinking, Text: part.Text}) + } + } + case "function_call": + response.StopReason = StopToolUse + response.Content = append(response.Content, Content{Kind: KindToolUse, ToolUse: &ToolUse{ + Id: emptyAs(item.CallId, item.Id), Name: item.Name, Arguments: emptyAsObject(item.Arguments), + }}) + } + } + if body.Status == "incomplete" && body.IncompleteDetails != nil && + body.IncompleteDetails.Reason == "max_output_tokens" { + response.StopReason = StopMaxTokens + } + return response, nil +} + +// responsesEvent is one event of this API's stream, which names itself in its +// own payload. +type responsesEvent struct { + Type string `json:"type"` + Delta string `json:"delta"` + OutputIndex int `json:"output_index"` + Item *responsesOutputItem `json:"item"` + Response *responsesResult `json:"response"` + Error *responsesFailure `json:"error"` +} + +func (responsesCodec) DecodeStream(reader io.Reader, fn func(Event) bool) error { + stop := "" + usage := Usage{} + model := "" + running := true + // A call's output index is what its argument deltas name it by, so it is + // the call's index too. + opened := map[int]bool{} + + err := forEachEvent(reader, func(data []byte) bool { + var event responsesEvent + if err := json.Unmarshal(data, &event); err != nil { + return true + } + if event.Response != nil && event.Response.Model != "" { + model = event.Response.Model + } + + switch event.Type { + case "response.output_text.delta": + if event.Delta != "" { + running = fn(Event{Kind: EventText, Text: event.Delta, Model: model}) + } + + case "response.reasoning_summary_text.delta", "response.reasoning_text.delta": + if event.Delta != "" { + running = fn(Event{Kind: EventThinking, Text: event.Delta, Model: model}) + } + + case "response.output_item.added": + // A call's name and id arrive here once; only its arguments stream. + if event.Item == nil || event.Item.Type != "function_call" { + return true + } + opened[event.OutputIndex] = true + stop = StopToolUse + running = fn(Event{Kind: EventToolUse, Model: model, Tool: &ToolDelta{ + Index: event.OutputIndex, + Id: emptyAs(event.Item.CallId, event.Item.Id), + Name: event.Item.Name, + }}) + + case "response.function_call_arguments.delta": + if event.Delta == "" { + return true + } + running = fn(Event{Kind: EventToolUse, Model: model, Tool: &ToolDelta{ + Index: event.OutputIndex, Arguments: event.Delta, + }}) + + case "response.output_item.done": + // An upstream that sent no delta reports the call only here. + if event.Item == nil || event.Item.Type != "function_call" || opened[event.OutputIndex] { + return true + } + opened[event.OutputIndex] = true + stop = StopToolUse + running = fn(Event{Kind: EventToolUse, Model: model, Tool: &ToolDelta{ + Index: event.OutputIndex, + Id: emptyAs(event.Item.CallId, event.Item.Id), + Name: event.Item.Name, + Arguments: event.Item.Arguments, + }}) + + case "response.completed", "response.incomplete": + if event.Response == nil { + return true + } + if event.Response.Usage != nil { + usage = event.Response.Usage.canonical() + } + if event.Response.IncompleteDetails != nil && + event.Response.IncompleteDetails.Reason == "max_output_tokens" { + stop = StopMaxTokens + } + + case "response.failed", "error": + failure := event.Error.canonical() + if failure == nil && event.Response != nil { + failure = event.Response.Error.canonical() + } + if failure == nil { + failure = &Failure{Kind: "server_error", Message: "the upstream stream failed"} + } + running = fn(Event{Kind: EventFailure, Failure: failure}) + } + return running + }) + + if running { + fn(Event{Kind: EventDone, StopReason: emptyAs(stop, StopEnd), Usage: &usage}) + } + return err +} diff --git a/web/src/components/shared/brand-options.tsx b/web/src/components/shared/brand-options.tsx index 906137e..3b94a29 100644 --- a/web/src/components/shared/brand-options.tsx +++ b/web/src/components/shared/brand-options.tsx @@ -54,6 +54,17 @@ export const protocolOptions: SelectOption[] = [ vendorOption("anthropic", "Anthropic", "anthropic.com"), ]; +/** The Upstream API field of both provider forms, each API under the mark of + * the vendor that defined it. */ +export function wireApiOptions(fromType: string): SelectOption[] { + return [ + brandOption("", fromType, <Settings2 className="size-4 opacity-50" />), + vendorOption("openai", "Chat Completions", "openai.com"), + vendorOption("responses", "Responses", "openai.com"), + vendorOption("anthropic", "Messages", "anthropic.com"), + ]; +} + /** The Type field of both provider forms: two vendors and everything else. */ export const providerTypeOptions: SelectOption[] = [ ...protocolOptions, diff --git a/web/src/lib/providers.ts b/web/src/lib/providers.ts index 5cc96d7..69f88dc 100644 --- a/web/src/lib/providers.ts +++ b/web/src/lib/providers.ts @@ -20,18 +20,41 @@ export function providerIdOf(provider: Pick<Provider, "owner" | "name">) { return `${provider.owner}/${provider.name}`; } -/** Mirrors object.ProviderProtocol on the server. */ +/** Mirrors object.ProviderApiFamily: the family a provider's base URL belongs to. */ export function providerProtocol(type: string) { return type === "anthropic" ? "anthropic" : "openai"; } +/** The wire formats the gateway can talk to an upstream in. */ +export const wireApis = ["openai", "responses", "anthropic"] as const; + +/** Mirrors object.ProviderProtocol: the API the gateway posts to. */ +export function providerWireApi(provider: {type?: string; protocol?: string} | undefined) { + const named = provider?.protocol ?? ""; + return wireApis.includes(named as (typeof wireApis)[number]) + ? named + : providerProtocol(provider?.type ?? ""); +} + +/** The path one wire format is appended to the base URL as. */ +export function wireApiEndpoint(wireApi: string) { + switch (wireApi) { + case "anthropic": + return "/v1/messages"; + case "responses": + return "/responses"; + default: + return "/chat/completions"; + } +} + /** Mirrors object.ProviderAuthProvider and object.ProviderAuthClient on the server. */ export const authProvider = "provider"; export const authClient = "client"; -/** Mirrors object.ServesResponsesApi: only OpenAI itself serves that API. */ -export function servesResponsesApi(provider: {type?: string} | undefined) { - return provider?.type === "openai"; +/** Mirrors object.ServesResponsesApi: OpenAI itself, and anything pointed at it. */ +export function servesResponsesApi(provider: {type?: string; protocol?: string} | undefined) { + return provider?.type === "openai" || providerWireApi(provider) === "responses"; } /** Mirrors object.UsesClientAuth: whose credentials reach the upstream. */ diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json index 8050e6b..bf0eb8d 100644 --- a/web/src/locales/en/data.json +++ b/web/src/locales/en/data.json @@ -1206,6 +1206,7 @@ "Fetch models": "Fetch models", "Fetched {count} models": "Fetched {count} models", "Fill in from": "Fill in from a preset", + "From the type": "From the type", "Get a key from {vendor}": "Get a key from {vendor}", "Granted": "granted", "Granted path": "Granted path", @@ -1275,6 +1276,8 @@ "This link cannot be read": "This link cannot be read", "Type": "Type", "Unfinished draft": "Unfinished draft", + "Upstream API": "Upstream API", + "Upstream API hint": "Which API the gateway calls on this provider. Chat Completions suits nearly every vendor; Responses is what OpenAI and some relays serve as well, and it is the one a Codex client passes through untranslated, cache counters included.", "Usage": "Usage", "Usage hint": "Point any client that speaks this provider's API at Gateway with these variables.", "Used": "used", diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json index 0f2238f..1b7af3b 100644 --- a/web/src/locales/zh/data.json +++ b/web/src/locales/zh/data.json @@ -1276,6 +1276,7 @@ "Fetch models": "自动获取", "Fetched {count} models": "共获取到{count}个模型", "Fill in from": "从预设填入", + "From the type": "按类型推断", "Get a key from {vendor}": "去{vendor}获取密钥", "Granted": "总额", "Granted path": "总额字段路径", @@ -1345,6 +1346,8 @@ "This link cannot be read": "这个链接读不出来", "Type": "类型", "Unfinished draft": "有未完成的草稿", + "Upstream API": "上游接口", + "Upstream API hint": "网关调用该供应商时使用的接口。Chat Completions适用于几乎所有厂商；Responses是OpenAI和部分中转站同样提供的接口，Codex客户端走这条路时请求原样透传，缓存计数也一并保留。", "Usage": "使用方法", "Usage hint": "任何说这个供应商所用API的客户端，配上下面的变量就能走网关。", "Used": "已用", diff --git a/web/src/pages/ProviderEditPage.tsx b/web/src/pages/ProviderEditPage.tsx index e175e43..70bd297 100644 --- a/web/src/pages/ProviderEditPage.tsx +++ b/web/src/pages/ProviderEditPage.tsx @@ -30,7 +30,7 @@ import {CodeText, ResultScreen} from "@/components/shared/misc"; import {NumberInput} from "@/components/shared/number-input"; import {PageContainer, PageHeader, Section} from "@/components/shared/page-header"; import {PasswordInput} from "@/components/shared/password-input"; -import {baseUrlOptions, providerTypeOptions} from "@/components/shared/brand-options"; +import {baseUrlOptions, providerTypeOptions, wireApiOptions} from "@/components/shared/brand-options"; import {SearchSelect, SimpleSelect} from "@/components/shared/simple-select"; import {Button} from "@/components/ui/button"; import {Input} from "@/components/ui/input"; @@ -39,9 +39,11 @@ import { authClient, baseUrlPlaceholder, providerProtocol, + providerWireApi, gatewayBaseUrl, localShell, usesClientAuth, + wireApiEndpoint, } from "@/lib/providers"; import type {Provider, QuotaConfig} from "@/types"; @@ -97,11 +99,13 @@ function buildAnthropicUrl(baseUrl: string, endpoint: string) { return url.toString(); } -/** The upstream URL requests to a provider of this type end up at. */ -function buildUpstreamUrl(baseUrl: string, type: string) { - return providerProtocol(type) === "anthropic" - ? buildAnthropicUrl(baseUrl, "/v1/messages") - : buildOpenAiUrl(baseUrl, "/chat/completions"); +/** The upstream URL requests to this provider end up at. */ +function buildUpstreamUrl(provider: Provider) { + const wireApi = providerWireApi(provider); + const endpoint = wireApiEndpoint(wireApi); + return wireApi === "anthropic" + ? buildAnthropicUrl(provider.baseUrl, endpoint) + : buildOpenAiUrl(provider.baseUrl, endpoint); } export default function ProviderEditPage() { @@ -164,7 +168,7 @@ export default function ProviderEditPage() { ); } - const upstreamUrl = buildUpstreamUrl(provider.baseUrl, provider.type); + const upstreamUrl = buildUpstreamUrl(provider); return ( <PageContainer> @@ -217,6 +221,13 @@ export default function ProviderEditPage() { onChange={value => setField("baseUrl", value)} /> </Field> + <Field label={i18next.t("provider:Upstream API")} hint={i18next.t("provider:Upstream API hint")}> + <SimpleSelect + value={provider.protocol ?? ""} + onChange={value => setField("protocol", value)} + options={wireApiOptions(i18next.t("provider:From the type"))} + /> + </Field> <Field label={i18next.t("provider:Authentication")} hint={usesClientAuth(provider) ? i18next.t("provider:Client auth hint") : undefined} diff --git a/web/src/pages/ProviderListPage.tsx b/web/src/pages/ProviderListPage.tsx index 2d6440f..419d824 100644 --- a/web/src/pages/ProviderListPage.tsx +++ b/web/src/pages/ProviderListPage.tsx @@ -39,7 +39,7 @@ import {ErrorState} from "@/components/shared/empty-state"; import {Field, FormDialog} from "@/components/shared/form-dialog"; import {PageContainer, PageHeader} from "@/components/shared/page-header"; import {PasswordInput} from "@/components/shared/password-input"; -import {baseUrlOptions, providerTypeOptions} from "@/components/shared/brand-options"; +import {baseUrlOptions, providerTypeOptions, wireApiOptions} from "@/components/shared/brand-options"; import {SearchSelect, SimpleSelect} from "@/components/shared/simple-select"; import {MessageAlert} from "@/components/ui/alert"; import {Badge} from "@/components/ui/badge"; @@ -685,6 +685,13 @@ export default function ProviderListPage({account}: {account: Account}) { /> </Field> )} + <Field label={i18next.t("provider:Upstream API")} hint={i18next.t("provider:Upstream API hint")}> + <SimpleSelect + value={form.protocol ?? ""} + onChange={value => setFormField("protocol", value)} + options={wireApiOptions(i18next.t("provider:From the type"))} + /> + </Field> <Field label={i18next.t("provider:Authentication")} hint={usesClientAuth(form) ? i18next.t("provider:Client auth hint") : undefined} diff --git a/web/src/types.ts b/web/src/types.ts index bb2aae1..8514594 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -61,6 +61,8 @@ export interface Provider { models: string[]; priority: number; baseUrl: string; + /** The wire format the upstream is talked to in, empty for the one the type implies. */ + protocol?: string; apiKey: string; authMode: string; /** A site the vendor's icon is taken from, or an image URL. Empty derives it --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
