This is an automated email from the ASF dual-hosted git repository.
hsluoyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
The following commit(s) were added to refs/heads/master by this push:
new 87f1803 feat(records): keep and show the response each request got
back
87f1803 is described below
commit 87f180306d6db9ddabbcd1cddef4bf9cc890427e
Author: Yang Luo <[email protected]>
AuthorDate: Thu Sep 10 22:40:19 2026 +0800
feat(records): keep and show the response each request got back
---
controllers/proxy.go | 6 +-
controllers/proxy_record.go | 28 +++++++--
object/llm_record.go | 86 +++++++++++++++++++++++++---
web/src/components/llm/request-inspector.tsx | 51 ++++++++++++++++-
web/src/lib/llm-request.ts | 38 ++++++++++++
web/src/locales/en/data.json | 9 ++-
web/src/locales/zh/data.json | 11 ++--
web/src/pages/LlmRecordsPage.tsx | 7 ++-
web/src/types.ts | 2 +
9 files changed, 209 insertions(+), 29 deletions(-)
diff --git a/controllers/proxy.go b/controllers/proxy.go
index 723787c..45840db 100644
--- a/controllers/proxy.go
+++ b/controllers/proxy.go
@@ -101,7 +101,8 @@ type proxyRoute struct {
start time.Time
// record accumulates what is written to the LLM record of this
request. It
// is nil while recording is off.
- record *object.LlmRecord
+ record *object.LlmRecord
+ response *object.LlmResponseCapture
// request is the body in canonical form, decoded on first use: a
provider
// speaking the client's own format never needs one.
request *protocol.Request
@@ -604,10 +605,11 @@ func (c *ApiController) forwardToProvider(attempt
object.RouteAttempt, route *pr
// The tap reads the counters out of the upstream answer as it passes,
so it
// sees them in the provider's own spelling, translated or not. On a
failure
// the same bytes are the error the client was handed.
- tap := &usageTap{reader: body}
+ tap := &usageTap{reader: body, headLimit:
object.LlmResponseCaptureBytes()}
c.relayResponse(route, upstream, upstreamResp, tap)
if isSuccessStatus(upstreamResp.StatusCode) {
route.recordUsage(tap.tail)
+ route.recordResponse(upstream, isEventStream(upstreamResp),
tap.head)
} else {
route.recordErrorBody(tap.tail)
}
diff --git a/controllers/proxy_record.go b/controllers/proxy_record.go
index a45f973..ae2ef12 100644
--- a/controllers/proxy_record.go
+++ b/controllers/proxy_record.go
@@ -47,7 +47,16 @@ func (c *ApiController) finishLlmRecord(route *proxyRoute) {
record := route.record
route.record = nil
record.DurationMs = time.Since(route.start).Milliseconds()
- object.AddLlmRecord(record, route.body)
+ object.AddLlmRecord(record, route.body, route.response)
+}
+
+// recordResponse keeps the answer the client was given, in the upstream's own
+// spelling. The record writer decodes it.
+func (route *proxyRoute) recordResponse(upstream protocol.Upstream, stream
bool, raw []byte) {
+ if route.record == nil || len(raw) == 0 {
+ return
+ }
+ route.response = &object.LlmResponseCapture{Protocol: upstream.Name(),
Stream: stream, Raw: raw}
}
// recordAttempt names the provider and the model this try is being made
@@ -131,17 +140,26 @@ func (route *proxyRoute) recordUsage(tail []byte) {
}
}
-// usageTap keeps the tail of a relayed response. It never changes a byte of
-// what the client receives, and never holds more than usageTailBytes.
+// usageTap keeps the tail of a relayed response, and its head up to headLimit
+// when the record stores the answer. It never changes a byte of what the
client
+// receives.
type usageTap struct {
- reader io.Reader
- tail []byte
+ reader io.Reader
+ tail []byte
+ head []byte
+ headLimit int
}
func (tap *usageTap) Read(p []byte) (int, error) {
n, err := tap.reader.Read(p)
if n > 0 {
tap.keep(p[:n])
+ if room := tap.headLimit - len(tap.head); room > 0 {
+ if room > n {
+ room = n
+ }
+ tap.head = append(tap.head, p[:room]...)
+ }
}
return n, err
}
diff --git a/object/llm_record.go b/object/llm_record.go
index 9dab17f..9469a04 100644
--- a/object/llm_record.go
+++ b/object/llm_record.go
@@ -15,6 +15,7 @@
package object
import (
+ "bytes"
"encoding/json"
"fmt"
"strings"
@@ -24,6 +25,7 @@ import (
"github.com/apache/casbin-gateway/auditutil"
"github.com/apache/casbin-gateway/conf"
+ "github.com/apache/casbin-gateway/protocol"
"github.com/apache/casbin-gateway/util"
"github.com/beego/beego"
"github.com/xorm-io/xorm"
@@ -51,8 +53,8 @@ type LlmFailure struct {
}
// LlmRecord is one client request relayed by the LLM proxy, together with the
-// outcome it ended in. Payload is filled only in "full" mode, and only after
-// auditutil has removed the credentials the body may carry.
+// outcome it ended in. Payload and Response are filled only in "full" mode,
and
+// only after auditutil has removed the credentials the bodies may carry.
type LlmRecord struct {
Id int64 `xorm:"int notnull pk autoincr" json:"id"`
CreatedTime string `xorm:"varchar(100) notnull index"
json:"createdTime"`
@@ -105,8 +107,11 @@ type LlmRecord struct {
MessageCount int `xorm:"int" json:"messageCount"`
ToolCount int `xorm:"int" json:"toolCount"`
- Summary string `xorm:"varchar(500)" json:"summary"`
- Payload string `xorm:"mediumtext" json:"payload"`
+ Summary string `xorm:"varchar(500)" json:"summary"`
+ Payload string `xorm:"mediumtext" json:"payload"`
+ // Response is the answer as an Anthropic message, whatever the upstream
+ // spoke, with a stream folded into one.
+ Response string `xorm:"mediumtext" json:"response"`
Redactions int `xorm:"int" json:"redactions"`
Truncated bool `xorm:"bool" json:"truncated"`
Bytes int `xorm:"int" json:"bytes"`
@@ -195,9 +200,18 @@ type LlmRecordStatus struct {
Count int64 `json:"count"`
}
+// LlmResponseCapture is an upstream answer as it came off the wire. It is
+// decoded on the writer goroutine, not on the request's.
+type LlmResponseCapture struct {
+ Protocol string
+ Stream bool
+ Raw []byte
+}
+
type llmRecordTask struct {
- record *LlmRecord
- rawBody []byte
+ record *LlmRecord
+ rawBody []byte
+ response *LlmResponseCapture
}
type llmRecordWriter struct {
@@ -248,9 +262,18 @@ func IsLlmRecording() bool {
return conf.GetLlmRecordMode() != conf.LlmRecordOff
}
+// LlmResponseCaptureBytes is how much of an answer the proxy should keep for
its
+// record, zero when the record would not store it.
+func LlmResponseCaptureBytes() int {
+ if conf.GetLlmRecordMode() != conf.LlmRecordFull {
+ return 0
+ }
+ return conf.GetLlmRecordMaxPayloadBytes() * llmOversizeFactor
+}
+
// AddLlmRecord queues one finished request. Recording is best effort: a full
// queue drops the record rather than holding the request up.
-func AddLlmRecord(record *LlmRecord, rawBody []byte) {
+func AddLlmRecord(record *LlmRecord, rawBody []byte, response
*LlmResponseCapture) {
if record == nil {
return
}
@@ -258,6 +281,7 @@ func AddLlmRecord(record *LlmRecord, rawBody []byte) {
record.Bytes = len(rawBody)
if conf.GetLlmRecordMode() != conf.LlmRecordFull {
rawBody = nil
+ response = nil
} else if len(rawBody) >
conf.GetLlmRecordMaxPayloadBytes()*llmOversizeFactor {
record.Truncated = true
rawBody = nil
@@ -269,7 +293,7 @@ func AddLlmRecord(record *LlmRecord, rawBody []byte) {
return
}
select {
- case llmWriter.queue <- llmRecordTask{record: record, rawBody: rawBody}:
+ case llmWriter.queue <- llmRecordTask{record: record, rawBody: rawBody,
response: response}:
default:
llmWriter.noteDrop()
}
@@ -311,6 +335,9 @@ func (writer *llmRecordWriter) write(task llmRecordTask) {
if len(task.rawBody) > 0 {
fillLlmRecordBody(task.record, task.rawBody)
}
+ if task.response != nil && len(task.response.Raw) > 0 {
+ fillLlmRecordResponse(task.record, task.response)
+ }
fillLlmRecordCost(task.record)
if _, err := ormer.Engine.Insert(task.record); err != nil {
beego.Error("LLM record write failed:", err)
@@ -376,6 +403,46 @@ func fillLlmRecordBody(record *LlmRecord, rawBody []byte) {
record.Summary = llmRecordSummary(sanitized)
}
+// fillLlmRecordResponse stores the answer in one shape for every upstream, so
+// the page reads a single format. What cannot be decoded, such as a body cut
+// off by the capture limit, is kept as it came.
+func fillLlmRecordResponse(record *LlmRecord, capture *LlmResponseCapture) {
+ maximum := conf.GetLlmRecordMaxPayloadBytes()
+ var decoded any
+ if response, err := decodeLlmResponse(record.BilledModel(), capture);
err == nil {
+ if encoded, err :=
protocol.Of(protocol.Anthropic).EncodeResponse(response); err == nil {
+ _ = json.Unmarshal(encoded, &decoded)
+ }
+ }
+ if decoded == nil {
+ record.Response = auditutil.SanitizeJSON(string(capture.Raw),
maximum)
+ return
+ }
+ record.Response, _ = encodeLlmBody(auditutil.SanitizeValue("",
decoded), maximum)
+}
+
+func decodeLlmResponse(model string, capture *LlmResponseCapture)
(*protocol.Response, error) {
+ upstream, err := protocol.UpstreamOf(capture.Protocol)
+ if err != nil {
+ return nil, err
+ }
+ if !capture.Stream {
+ return upstream.DecodeResponse(capture.Raw)
+ }
+
+ collector := protocol.NewCollector(model)
+ err = upstream.DecodeStream(bytes.NewReader(capture.Raw), func(event
protocol.Event) bool {
+ collector.Add(event)
+ return true
+ })
+ // A stream cut short still shows what arrived before the cut.
+ response := collector.Response()
+ if err != nil && len(response.Content) == 0 {
+ return nil, err
+ }
+ return response, nil
+}
+
// fillLlmRecordCost prices a record from its token counters, against the model
// that was actually billed rather than the one the client asked for.
func fillLlmRecordCost(record *LlmRecord) {
@@ -576,7 +643,7 @@ func GetLlmRecords(filter LlmRecordFilter, offset int,
limit int) ([]*LlmRecord,
session := llmRecordSession(filter)
defer session.Close()
records := []*LlmRecord{}
- err = session.Omit("payload", "error_body").Desc("id").Limit(limit,
offset).Find(&records)
+ err = session.Omit("payload", "response",
"error_body").Desc("id").Limit(limit, offset).Find(&records)
return records, count, err
}
@@ -917,6 +984,7 @@ func publishLlmRecord(record *LlmRecord) {
}
summary := *record
summary.Payload = ""
+ summary.Response = ""
summary.ErrorBody = ""
for _, feed := range llmHub.subscribers {
select {
diff --git a/web/src/components/llm/request-inspector.tsx
b/web/src/components/llm/request-inspector.tsx
index 8857cb3..c05350f 100644
--- a/web/src/components/llm/request-inspector.tsx
+++ b/web/src/components/llm/request-inspector.tsx
@@ -21,7 +21,14 @@ import {Badge, type BadgeVariant} from
"@/components/ui/badge";
import {Button} from "@/components/ui/button";
import {Tabs, TabsContent, TabsList, TabsTrigger} from "@/components/ui/tabs";
import {cn} from "@/lib/utils";
-import {parseRequest, type ParsedRequest, type RequestBlock, type
RequestMessage, type ToolSchema} from "@/lib/llm-request";
+import {
+ parseRequest,
+ parseResponse,
+ type ParsedRequest,
+ type RequestBlock,
+ type RequestMessage,
+ type ToolSchema,
+} from "@/lib/llm-request";
const COLLAPSED_CHARS = 1200;
@@ -225,11 +232,44 @@ function SystemView({request}: {request: ParsedRequest}) {
);
}
+function ResponseView({response}: {response: string}) {
+ const parsed = React.useMemo(() => parseResponse(response), [response]);
+
+ if (!response) {
+ return <span className="text-muted-foreground text-xs">{i18next.t("llm:No
response stored")}</span>;
+ }
+ if (parsed.invalid || !parsed.message) {
+ return (
+ <div className="grid gap-2">
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:Body
is not JSON")}</span>
+ <CodeBlock copyable maxHeight="24rem">
+ {parsed.raw}
+ </CodeBlock>
+ </div>
+ );
+ }
+
+ return (
+ <div className="grid gap-2">
+ <div className="flex items-center gap-2">
+ {parsed.stopReason ? (
+ <span className="text-muted-foreground text-xs">
+ {i18next.t("llm:Stop reason")}:
<CodeText>{parsed.stopReason}</CodeText>
+ </span>
+ ) : null}
+ <CopyButton value={parsed.raw} />
+ </div>
+ <MessageView message={parsed.message} />
+ </div>
+ );
+}
+
/**
* Everything the client sent this turn: the system prompt, every message in
- * order, and the schema of every tool the model was offered.
+ * order, and the schema of every tool the model was offered. The answer it got
+ * back has a tab of its own.
*/
-export function RequestInspector({payload}: {payload: string}) {
+export function RequestInspector({payload, response = ""}: {payload: string;
response?: string}) {
const request = React.useMemo(() => parseRequest(payload), [payload]);
const [tab, setTab] = React.useState("messages");
@@ -251,6 +291,7 @@ export function RequestInspector({payload}: {payload:
string}) {
{i18next.t("llm:Messages")}
<Badge variant="muted">{request.messages.length}</Badge>
</TabsTrigger>
+ <TabsTrigger value="response">{i18next.t("llm:Response")}</TabsTrigger>
<TabsTrigger value="system">
{i18next.t("llm:System prompt")}
{request.systemChars > 0 ? <Badge
variant="muted">{formatChars(request.systemChars)}</Badge> : null}
@@ -271,6 +312,10 @@ export function RequestInspector({payload}: {payload:
string}) {
)}
</TabsContent>
+ <TabsContent value="response">
+ <ResponseView response={response} />
+ </TabsContent>
+
<TabsContent value="system">
<SystemView request={request} />
</TabsContent>
diff --git a/web/src/lib/llm-request.ts b/web/src/lib/llm-request.ts
index b25214e..c467b2e 100644
--- a/web/src/lib/llm-request.ts
+++ b/web/src/lib/llm-request.ts
@@ -230,6 +230,44 @@ function typeOfProperty(property: any): string {
return property.enum ? "enum" : "";
}
+export interface ParsedResponse {
+ message: RequestMessage | null;
+ stopReason: string;
+ raw: string;
+ invalid: boolean;
+}
+
+/** Reads a recorded answer, which is stored as an Anthropic message whatever
the upstream spoke. */
+export function parseResponse(payload: string): ParsedResponse {
+ const empty: ParsedResponse = {message: null, stopReason: "", raw: payload,
invalid: false};
+ if (!payload) {
+ return empty;
+ }
+
+ let body: Record<string, any>;
+ try {
+ body = JSON.parse(payload);
+ } catch {
+ return {...empty, invalid: true};
+ }
+ if (!body || typeof body !== "object" || !Array.isArray(body.content)) {
+ return {...empty, raw: JSON.stringify(body, null, 2), invalid: true};
+ }
+
+ const blocks = readContent(body.content);
+ return {
+ message: {
+ index: 0,
+ role: String(body.role ?? "assistant"),
+ blocks: blocks,
+ chars: blocks.reduce((total, block) => total + block.chars, 0),
+ },
+ stopReason: String(body.stop_reason ?? ""),
+ raw: JSON.stringify(body, null, 2),
+ invalid: false,
+ };
+}
+
export function parseRequest(payload: string): ParsedRequest {
const empty: ParsedRequest = {
system: [],
diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json
index ed8af3b..74d83fd 100644
--- a/web/src/locales/en/data.json
+++ b/web/src/locales/en/data.json
@@ -1119,9 +1119,9 @@
"Bodies are not stored": "Request bodies are not stored",
"Bodies are not stored detail": "llmRecordMode is \"metadata\", so the
prompts, tool schemas and messages behind these rows were never kept. Store
them from now on to inspect them.",
"Bodies are stored": "Prompts and responses are being stored",
- "Bodies are stored detail": "llmRecordMode is \"full\", so the prompts,
tool schemas and messages of every relayed request are kept, with the
credentials removed. Record metadata only, or stop recording, to keep less.",
+ "Bodies are stored detail": "llmRecordMode is \"full\", so the prompts,
tool schemas and messages of every relayed request, and the answer it got back,
are kept with the credentials removed. Record metadata only, or stop recording,
to keep less.",
"Body is not JSON": "The stored body is not valid JSON, so it is shown as
it was captured.",
- "Body not stored": "The request body was not stored: llmRecordMode is
\"metadata\".",
+ "Body not stored": "The request and response bodies were not stored:
llmRecordMode is \"metadata\".",
"Body too large to store": "The request body was larger than the capture
limit and was not stored.",
"Body was shortened": "The body was over the capture limit, so its longest
strings were cut. Every message, tool and parameter is still listed. Raise
llmRecordMaxPayloadBytes to keep more.",
"Cache breakpoint": "cached",
@@ -1157,7 +1157,7 @@
"List prices, see llmPricingFile": "List prices - override them with
llmPricingFile.",
"Live": "Live",
"Messages": "Messages",
- "Mode full": "full - metadata and sanitized request body",
+ "Mode full": "full - metadata and sanitized request and response bodies",
"Mode metadata": "metadata only",
"Mode off": "off",
"Mode {mode}, kept for {days} days, up to {max} records": "Mode {mode},
kept for {days} days, up to {max} records",
@@ -1173,6 +1173,7 @@
"No price for this model": "No price is configured for this model",
"No records yet": "No records yet - send a request through the LLM
gateway.",
"No response": "no response",
+ "No response stored": "No response was stored for this request. Only a
successful answer relayed while llmRecordMode is \"full\" is kept; a failure is
shown above.",
"No routing rules yet": "No routing rules yet",
"No system prompt": "This request carried no system prompt.",
"No tools": "The model was offered no tools.",
@@ -1195,6 +1196,7 @@
"Request failed": "Request failed",
"Request size": "Request size",
"Requests": "Requests",
+ "Response": "Response",
"Route preview description": "Ask where a model name would go, without
sending a request.",
"Routed by": "Routed by",
"Routed to": "Routed to",
@@ -1208,6 +1210,7 @@
"Show less": "Show less",
"Status": "Status",
"Step {n}": "Step {n}",
+ "Stop reason": "Stop reason",
"Succeeded": "Succeeded",
"System prompt": "System prompt",
"Thinking": "thinking",
diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json
index 2b4f21f..ed63795 100644
--- a/web/src/locales/zh/data.json
+++ b/web/src/locales/zh/data.json
@@ -1304,10 +1304,10 @@
"Background work on the cheap model": "后台任务走便宜模型",
"Bodies are not stored": "未保存请求体",
"Bodies are not stored detail":
"llmRecordMode为\"metadata\"，因此这些记录背后的提示词、工具schema和消息都没有保存。开启保存后即可查看。",
- "Bodies are stored": "正在保存请求内容",
- "Bodies are stored detail":
"llmRecordMode为\"full\"，转发的每一条请求，其提示词、工具schema和消息都会在去除凭据后保存下来。可改为只记录元数据，或直接关闭记录。",
+ "Bodies are stored": "正在保存请求和响应内容",
+ "Bodies are stored detail":
"llmRecordMode为\"full\"，转发的每一条请求，其提示词、工具schema、消息以及返回的结果都会在去除凭据后保存下来。可改为只记录元数据，或直接关闭记录。",
"Body is not JSON": "保存的请求体不是合法JSON，这里按捕获时的原样显示。",
- "Body not stored": "未保存请求正文：llmRecordMode为\"metadata\"。",
+ "Body not stored": "未保存请求和响应正文：llmRecordMode为\"metadata\"。",
"Body too large to store": "请求正文超过采集上限，未予保存。",
"Body was shortened":
"请求体超出了捕获上限，其中最长的字符串被截断，但每条消息、每个工具和每个参数都仍然完整列出。调大llmRecordMaxPayloadBytes可保留更多内容。",
"Cache breakpoint": "缓存点",
@@ -1343,7 +1343,7 @@
"List prices, see llmPricingFile": "按官方标价估算，可用llmPricingFile覆盖。",
"Live": "实时",
"Messages": "消息",
- "Mode full": "完整（元数据+脱敏后的请求正文）",
+ "Mode full": "完整（元数据+脱敏后的请求和响应正文）",
"Mode metadata": "仅元数据",
"Mode off": "关闭",
"Mode {mode}, kept for {days} days, up to {max} records":
"模式：{mode}，保留{days}天，最多{max}条",
@@ -1359,6 +1359,7 @@
"No price for this model": "该模型没有配置价格",
"No records yet": "暂无记录——请通过LLM网关发送一次请求。",
"No response": "无响应",
+ "No response stored":
"这条请求没有保存响应：只有llmRecordMode为\"full\"时成功返回的结果才会保存，失败原因见上方。",
"No routing rules yet": "还没有路由规则",
"No system prompt": "该请求没有system prompt。",
"No tools": "该请求没有向模型提供工具。",
@@ -1381,6 +1382,7 @@
"Request failed": "请求失败",
"Request size": "请求大小",
"Requests": "请求数",
+ "Response": "响应",
"Route preview description": "不发真实请求，直接查一个模型名会被路由到哪里。",
"Routed by": "命中规则",
"Routed to": "实际发往",
@@ -1394,6 +1396,7 @@
"Show less": "收起",
"Status": "状态",
"Step {n}": "第{n}级",
+ "Stop reason": "停止原因",
"Succeeded": "成功",
"System prompt": "System prompt",
"Thinking": "思考",
diff --git a/web/src/pages/LlmRecordsPage.tsx b/web/src/pages/LlmRecordsPage.tsx
index f65fc8d..3954c49 100644
--- a/web/src/pages/LlmRecordsPage.tsx
+++ b/web/src/pages/LlmRecordsPage.tsx
@@ -135,6 +135,7 @@ function RecordDetail({record, onDelete}: {record:
LlmRecord; onDelete: () => vo
}, [record.id]);
const payload = detail?.payload ?? "";
+ const response = detail?.response ?? "";
const rate = price
? `${formatCost(price.input)} / ${formatCost(price.output)}
${i18next.t("llm:per million")}`
: i18next.t("llm:No price for this model");
@@ -186,9 +187,9 @@ function RecordDetail({record, onDelete}: {record:
LlmRecord; onDelete: () => vo
{error ? <MessageAlert title={error} /> : null}
{record.truncated ? <MessageAlert variant="warning"
title={i18next.t("llm:Body was shortened")} /> : null}
- {payload ? <RequestInspector payload={payload} /> : null}
- {!payload && detail === null && error === "" ? <Loading type="small" />
: null}
- {!payload && detail !== null ? (
+ {payload || response ? <RequestInspector payload={payload}
response={response} /> : null}
+ {!payload && !response && detail === null && error === "" ? <Loading
type="small" /> : null}
+ {!payload && !response && detail !== null ? (
<span className="text-muted-foreground text-xs">{i18next.t("llm:Body
not stored")}</span>
) : null}
diff --git a/web/src/types.ts b/web/src/types.ts
index cdf4441..af9be77 100644
--- a/web/src/types.ts
+++ b/web/src/types.ts
@@ -674,6 +674,8 @@ export interface LlmRecord {
summary: string;
/** Only returned by getLlmRecord, the list endpoint leaves it out. */
payload: string;
+ /** The answer as an Anthropic message, whatever the upstream spoke. Only
returned by getLlmRecord. */
+ response: string;
redactions: number;
truncated: boolean;
bytes: number;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
