This is an automated email from the ASF dual-hosted git repository.
hsluoyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
The following commit(s) were added to refs/heads/master by this push:
new 712cd5d feat: add unified LLM API proxy endpoint /v1/chat/completions
(#190)
712cd5d is described below
commit 712cd5dd9e4247ea431900cf42ec1102ab825f25
Author: Ziyang Dong <[email protected]>
AuthorDate: Thu Aug 13 13:41:25 2026 +0800
feat: add unified LLM API proxy endpoint /v1/chat/completions (#190)
---
controllers/proxy.go | 192 +++++++++++++++++++++++++++++++++++++++++++++++++++
object/channel.go | 26 +++++++
routers/filter.go | 2 +-
routers/router.go | 3 +
4 files changed, 222 insertions(+), 1 deletion(-)
diff --git a/controllers/proxy.go b/controllers/proxy.go
new file mode 100644
index 0000000..db96979
--- /dev/null
+++ b/controllers/proxy.go
@@ -0,0 +1,192 @@
+// Copyright 2023 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controllers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/apache/casbin-gateway/object"
+ "github.com/beego/beego"
+)
+
+// chatCompletionsRequest contains only the fields needed for routing.
+// All other fields (messages, temperature, etc.) are forwarded as-is to the
upstream.
+// (PM section 3.3.1)
+type chatCompletionsRequest struct {
+ Model string `json:"model"`
+ Stream bool `json:"stream"`
+}
+
+// openaiErrorResponse follows the OpenAI API error format.
+// (PM section 3.3.7)
+type openaiErrorResponse struct {
+ Error openaiErrorDetail `json:"error"`
+}
+
+type openaiErrorDetail struct {
+ Message string `json:"message"`
+ Type string `json:"type"`
+}
+
+// proxyClient is a shared HTTP client for upstream requests.
+// Reusing a single instance allows TCP connection pooling across requests.
+var proxyClient = &http.Client{
+ CheckRedirect: func(*http.Request, []*http.Request) error { return
http.ErrUseLastResponse },
+}
+
+// ChatCompletions is the OpenAI-compatible chat completions proxy endpoint.
+// It matches an upstream channel by model name and forwards the request
+// and response body as-is (pass-through).
+// Supports SSE streaming when stream=true in the request body.
+// This endpoint does NOT require Casdoor authentication (auth deferred to
milestone 1.3).
+// (PM section 3.3)
+func (c *ApiController) ChatCompletions() {
+ rawBody := c.Ctx.Input.RequestBody
+
+ // 3.3.1 — Parse only model and stream; forward everything else as-is.
+ var req chatCompletionsRequest
+ if err := json.Unmarshal(rawBody, &req); err != nil {
+ c.writeOpenAIError(http.StatusBadRequest,
"invalid_request_error", "invalid request body")
+ return
+ }
+ if req.Model == "" {
+ c.writeOpenAIError(http.StatusBadRequest,
"invalid_request_error", "model is required")
+ return
+ }
+
+ // 3.3.2 — Match channel globally (no owner filter; Judge Q1 → Plan A).
+ channel, err := object.GetChannelByModel(req.Model)
+ if err != nil {
+ if errors.Is(err, object.ErrNoChannelAvailable) {
+ c.writeOpenAIError(http.StatusBadRequest,
"invalid_request_error", err.Error())
+ } else {
+ beego.Error("channel lookup failed:", err)
+ c.writeOpenAIError(http.StatusBadGateway,
"server_error", "channel lookup failed")
+ }
+ return
+ }
+
+ // Guard against misconfigured channels with empty base URL.
+ if channel.BaseUrl == "" {
+ beego.Error("channel has empty base URL:", channel.GetId())
+ c.writeOpenAIError(http.StatusBadGateway, "server_error",
"channel base URL is not configured")
+ return
+ }
+
+ // 3.3.4 — Build upstream request. Path is hardcoded to
OpenAI-compatible /v1/chat/completions.
+ upstreamURL := strings.TrimRight(channel.BaseUrl, "/") +
"/v1/chat/completions"
+ upstreamReq, err := http.NewRequestWithContext(c.Ctx.Request.Context(),
"POST", upstreamURL, bytes.NewReader(rawBody))
+ if err != nil {
+ c.writeOpenAIError(http.StatusBadGateway, "server_error",
"upstream connection failed")
+ return
+ }
+ upstreamReq.Header.Set("Content-Type", "application/json")
+ upstreamReq.Header.Set("Authorization", "Bearer "+channel.ApiKey)
+
+ // 3.3.5 — Per-request timeout via context.
+ ctx, cancel := context.WithTimeout(c.Ctx.Request.Context(),
120*time.Second)
+ defer cancel()
+ upstreamReq = upstreamReq.WithContext(ctx)
+
+ // Execute upstream request.
+ upstreamResp, err := proxyClient.Do(upstreamReq)
+ if err != nil {
+ if ctx.Err() == context.DeadlineExceeded {
+ c.writeOpenAIError(http.StatusGatewayTimeout,
"server_error", "upstream timeout")
+ } else {
+ c.writeOpenAIError(http.StatusBadGateway,
"server_error", "upstream connection failed")
+ }
+ return
+ }
+ defer upstreamResp.Body.Close()
+
+ // 3.3.6 / 3.3.7 — Stream SSE or copy non-stream response as
pass-through.
+ if req.Stream {
+ c.proxySSE(upstreamResp)
+ } else {
+ c.proxyNonStream(upstreamResp)
+ }
+}
+
+// proxyNonStream copies the upstream response headers, status code, and body
+// directly to the client without any transformation (pass-through mode).
+// (PM section 3.3.7)
+func (c *ApiController) proxyNonStream(upstreamResp *http.Response) {
+ for k, vs := range upstreamResp.Header {
+ for _, v := range vs {
+ c.Ctx.ResponseWriter.Header().Add(k, v)
+ }
+ }
+ c.Ctx.ResponseWriter.WriteHeader(upstreamResp.StatusCode)
+ if _, err := io.Copy(c.Ctx.ResponseWriter, upstreamResp.Body); err !=
nil {
+ beego.Error("proxy non-stream response copy failed:", err)
+ }
+}
+
+// proxySSE forwards an SSE (Server-Sent Events) stream from the upstream
+// to the client. Each chunk is written and flushed immediately so that
+// the client receives events in real time.
+// Monitors client disconnection via c.Ctx.Request.Context().Done().
+// (PM section 3.3.6)
+func (c *ApiController) proxySSE(upstreamResp *http.Response) {
+ c.Ctx.ResponseWriter.Header().Set("Content-Type", "text/event-stream")
+ c.Ctx.ResponseWriter.Header().Set("Cache-Control", "no-cache")
+ c.Ctx.ResponseWriter.Header().Set("Connection", "keep-alive")
+ c.Ctx.ResponseWriter.WriteHeader(upstreamResp.StatusCode)
+
+ buf := make([]byte, 4096)
+ for {
+ // Respect client disconnection to avoid goroutine leak.
+ select {
+ case <-c.Ctx.Request.Context().Done():
+ return
+ default:
+ }
+
+ n, err := upstreamResp.Body.Read(buf)
+ if n > 0 {
+ c.Ctx.ResponseWriter.Write(buf[:n])
+ c.Ctx.ResponseWriter.Flush()
+ }
+ if err != nil {
+ if err != io.EOF {
+ beego.Error("SSE upstream read error:", err)
+ }
+ return
+ }
+ }
+}
+
+// writeOpenAIError writes an OpenAI-compatible JSON error response
+// with the given HTTP status code, error type, and message.
+// (PM section 3.3.7)
+func (c *ApiController) writeOpenAIError(statusCode int, errType, message
string) {
+ resp := openaiErrorResponse{
+ Error: openaiErrorDetail{
+ Message: message,
+ Type: errType,
+ },
+ }
+ c.Ctx.ResponseWriter.Header().Set("Content-Type", "application/json")
+ c.Ctx.ResponseWriter.WriteHeader(statusCode)
+ json.NewEncoder(c.Ctx.ResponseWriter).Encode(resp)
+}
diff --git a/object/channel.go b/object/channel.go
index 8dffa1f..99bee30 100644
--- a/object/channel.go
+++ b/object/channel.go
@@ -27,6 +27,12 @@ import (
"github.com/xorm-io/core"
)
+// ErrNoChannelAvailable is returned by GetChannelByModel when no enabled
+// channel matches the requested model name. It is a sentinel error so
+// callers can distinguish "no match" (client error, HTTP 400) from
+// database failures (server error, HTTP 502).
+var ErrNoChannelAvailable = errors.New("no available channel")
+
// ApiKeyMask is what the API returns in place of a stored API key. Sending it
// back in an update means "keep the existing key"; sending anything else
// (including an empty string) overwrites the stored key.
@@ -291,3 +297,23 @@ func TestChannelConnectivity(channel *Channel) (bool, int,
string) {
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
return resp.StatusCode >= 200 && resp.StatusCode < 300,
resp.StatusCode, resp.Status
}
+
+// GetChannelByModel returns the highest-priority enabled channel that supports
+// the given model name. It queries all channels globally (no owner filter)
+// because /v1/chat/completions is an unauthenticated public endpoint.
+// Corresponds to PM section 3.4.
+func GetChannelByModel(model string) (*Channel, error) {
+ channels := []*Channel{}
+ err := ormer.Engine.Where("status = ?",
"enabled").Asc("priority").Find(&channels)
+ if err != nil {
+ return nil, fmt.Errorf("channel query failed: %w", err)
+ }
+ for _, ch := range channels {
+ for _, m := range ch.Models {
+ if m == model {
+ return ch, nil
+ }
+ }
+ }
+ return nil, fmt.Errorf("%w: %s", ErrNoChannelAvailable, model)
+}
diff --git a/routers/filter.go b/routers/filter.go
index d092a7d..2b9fc91 100644
--- a/routers/filter.go
+++ b/routers/filter.go
@@ -25,7 +25,7 @@ import (
func TransparentStatic(ctx *context.Context) {
urlPath := ctx.Request.URL.Path
- if strings.HasPrefix(urlPath, "/api/") {
+ if strings.HasPrefix(urlPath, "/api/") || strings.HasPrefix(urlPath,
"/v1/") {
return
}
diff --git a/routers/router.go b/routers/router.go
index 9e6e03f..d42c2c3 100644
--- a/routers/router.go
+++ b/routers/router.go
@@ -83,4 +83,7 @@ func initAPI() {
beego.Router("/api/update-channel", &controllers.ApiController{},
"POST:UpdateChannel")
beego.Router("/api/delete-channel", &controllers.ApiController{},
"POST:DeleteChannel")
beego.Router("/api/test-channel", &controllers.ApiController{},
"POST:TestChannel")
+
+ // OpenAI-compatible chat completions endpoint for LLM gateway
milestone 1.2.
+ beego.Router("/v1/chat/completions", &controllers.ApiController{},
"POST:ChatCompletions")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
