This is an automated email from the ASF dual-hosted git repository.
hsluoyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
The following commit(s) were added to refs/heads/master by this push:
new 98052d3 feat(llm): inspect every prompt, tool schema and token cost
live
98052d3 is described below
commit 98052d320d450aecbbbea175169b8129ccb1b588
Author: Yang Luo <[email protected]>
AuthorDate: Fri Aug 21 15:29:44 2026 +0800
feat(llm): inspect every prompt, tool schema and token cost live
---
README.md | 22 ++
conf/app.conf | 6 +
conf/conf.go | 20 ++
controllers/llm_record.go | 102 ++++++++-
controllers/proxy_record.go | 63 ++++-
object/llm_pricing.go | 123 ++++++++++
object/llm_record.go | 281 ++++++++++++++++++++++-
routers/router.go | 2 +
web/src/backend/LlmRecordBackend.ts | 36 ++-
web/src/components/llm/request-inspector.tsx | 312 +++++++++++++++++++++++++
web/src/lib/llm-request.ts | 317 +++++++++++++++++++++++++
web/src/locales/en/data.json | 61 ++++-
web/src/locales/zh/data.json | 61 ++++-
web/src/pages/LlmRecordsPage.tsx | 331 +++++++++++++++++++++++----
web/src/types.ts | 39 ++++
15 files changed, 1703 insertions(+), 73 deletions(-)
diff --git a/README.md b/README.md
index 07c50c2..43220c1 100644
--- a/README.md
+++ b/README.md
@@ -85,6 +85,28 @@ Gateway reads and writes these files in place. It stages a
replacement and renam
Configuration is found whether or not the agent itself was discovered, so an
agent installed some way Gateway does not recognize still shows the skills it
has on disk. Like agent monitoring, this only ever sees the host Gateway runs
on.
+### What was sent to the model
+
+Agents point at Gateway instead of at the model vendor, so every request they
make passes through it. The **LLM Records** page is where that traffic is read
back: one row per relayed request, and behind each row the whole body that went
upstream - the full system prompt, every message in order with its tool calls
and tool results, and the schema of every tool the model was offered. Both wire
formats are shown the same way, so an OpenAI request and an Anthropic one read
alike.
+
+Each row also carries what the turn cost: input tokens billed as fresh, tokens
read from and written to the prompt cache, output tokens, and the price of all
four. Above the table the same window is totalled, with a cache hit rate and a
per-model breakdown. Prices are built-in list prices, which vendors change and
resellers do not follow; point `llmPricingFile` at a JSON file of your own
rates to correct them.
+
+Turn **Live** on and the page stops polling: new requests appear at the top as
Gateway writes them.
+
+Nothing is recorded until you ask for it, because a prompt can carry anything
the user pasted into it. In `conf/app.conf`:
+
+```ini
+; "off" keeps nothing, "metadata" records who called which model with which
+; outcome, "full" also stores the request body - which is what the page needs
+; to show prompts, messages and tool schemas.
+llmRecordMode = "full"
+llmRecordRetentionDays = 30
+llmRecordMaxRecords = 10000
+llmRecordMaxPayloadBytes = 1048576
+```
+
+Bodies are sanitized before they are stored: anything that looks like a
credential is replaced, and the count of replacements is shown with the record.
Request headers, which is where the inbound API key is, never reach a record at
all. A body over `llmRecordMaxPayloadBytes` keeps its structure and loses only
its longest strings, so a large conversation is still listed message by message.
+
### Quick start
From nothing to a request flowing through the gateway, in four steps. No
database server is needed: Gateway creates `./data/casbin-gateway.db` on first
start.
diff --git a/conf/app.conf b/conf/app.conf
index a26b290..e51360b 100644
--- a/conf/app.conf
+++ b/conf/app.conf
@@ -97,3 +97,9 @@ llmRecordMode = "off"
llmRecordQueueCapacity = 1000
llmRecordRetentionDays = 30
llmRecordMaxRecords = 10000
+; Largest request body kept per record. A body over it keeps its structure and
+; loses only its longest strings.
+llmRecordMaxPayloadBytes = 1048576
+; Optional JSON file of token prices in US dollars per million, overriding the
+; built-in list prices, e.g. {"my-model": {"input": 1, "output": 3,
"cacheWrite": 1.25, "cacheRead": 0.1}}.
+llmPricingFile = "./data/llm-pricing.json"
diff --git a/conf/conf.go b/conf/conf.go
index fb30e26..fc09bef 100644
--- a/conf/conf.go
+++ b/conf/conf.go
@@ -265,6 +265,20 @@ func GetLlmRecordRetentionDays() int {
return res
}
+// GetLlmRecordMaxPayloadBytes bounds one retained request body. A coding agent
+// sends its whole system prompt, tool schemas and conversation every turn, so
+// the limit is far above what a chat message needs.
+func GetLlmRecordMaxPayloadBytes() int {
+ res := GetConfigIntDefault("llmRecordMaxPayloadBytes", 1024*1024)
+ if res < 64*1024 {
+ res = 64 * 1024
+ }
+ if res > 32*1024*1024 {
+ res = 32 * 1024 * 1024
+ }
+ return res
+}
+
// GetLlmRecordMaxRecords caps the table regardless of the retention window.
func GetLlmRecordMaxRecords() int {
res := GetConfigIntDefault("llmRecordMaxRecords", 10000)
@@ -274,6 +288,12 @@ func GetLlmRecordMaxRecords() int {
return res
}
+// GetLlmPricingFile is an optional JSON file of token prices overriding the
+// built-in table.
+func GetLlmPricingFile() string {
+ return strings.Trim(GetConfigString("llmPricingFile"), `"' `)
+}
+
func GetConfigRealDataSourceName(driverName string) string {
var dataSourceName string
if driverName != "mysql" {
diff --git a/controllers/llm_record.go b/controllers/llm_record.go
index 423545f..3635662 100644
--- a/controllers/llm_record.go
+++ b/controllers/llm_record.go
@@ -15,11 +15,22 @@
package controllers
import (
+ "encoding/json"
+ "fmt"
+ "net/http"
"strconv"
+ "time"
"github.com/apache/casbin-gateway/object"
+ "github.com/apache/casbin-gateway/util"
)
+// llmStreamHeartbeat keeps an idle live feed from being closed by whatever
sits
+// between the browser and Gateway.
+const llmStreamHeartbeat = 25 * time.Second
+
+const llmStatsTopModels = 8
+
// A record holds what a user asked a model, so unlike the proxy endpoints
// themselves these are limited to Gateway administrators.
@@ -41,6 +52,18 @@ func (c *ApiController) GetLlmRecords() {
limit = 100
}
+ records, count, err := object.GetLlmRecords(c.readLlmRecordFilter(),
(page-1)*limit, limit)
+ if err != nil {
+ c.ResponseError(err.Error())
+ return
+ }
+
+ c.ResponseOk(records, count)
+}
+
+// readLlmRecordFilter reads the filter the list and the stats share, so the
two
+// always describe the same set of requests.
+func (c *ApiController) readLlmRecordFilter() object.LlmRecordFilter {
filter := object.LlmRecordFilter{
Model: c.Input().Get("model"),
Channel: c.Input().Get("channel"),
@@ -48,13 +71,75 @@ func (c *ApiController) GetLlmRecords() {
ClientIp: c.Input().Get("clientIp"),
Outcome: c.Input().Get("outcome"),
}
- records, count, err := object.GetLlmRecords(filter, (page-1)*limit,
limit)
+ if hours, err := strconv.Atoi(c.Input().Get("windowHours")); err == nil
&& hours > 0 {
+ filter.Since =
util.FormatTime(time.Now().Add(-time.Duration(hours) * time.Hour))
+ }
+ return filter
+}
+
+// GetLlmRecordStats totals the window the list is showing.
+func (c *ApiController) GetLlmRecordStats() {
+ if c.RequireAdmin() {
+ return
+ }
+
+ stats, err := object.GetLlmRecordStats(c.readLlmRecordFilter(),
llmStatsTopModels)
if err != nil {
c.ResponseError(err.Error())
return
}
- c.ResponseOk(records, count)
+ c.ResponseOk(stats)
+}
+
+// StreamLlmRecords pushes records to the page as they are written, so what a
+// model is being sent is visible while it happens rather than on the next
+// refresh.
+func (c *ApiController) StreamLlmRecords() {
+ if c.RequireAdmin() {
+ return
+ }
+
+ c.EnableRender = false
+ writer := c.Ctx.ResponseWriter
+ writer.Header().Set("Content-Type", "text/event-stream")
+ writer.Header().Set("Cache-Control", "no-cache")
+ writer.Header().Set("Connection", "keep-alive")
+ // An nginx in front of Gateway would otherwise hold every event back
until
+ // the response ends.
+ writer.Header().Set("X-Accel-Buffering", "no")
+ writer.WriteHeader(http.StatusOK)
+ writer.Flush()
+
+ id, feed := object.SubscribeLlmRecords()
+ defer object.UnsubscribeLlmRecords(id)
+
+ ticker := time.NewTicker(llmStreamHeartbeat)
+ defer ticker.Stop()
+ closed := c.Ctx.Request.Context().Done()
+ for {
+ select {
+ case record, ok := <-feed:
+ if !ok {
+ return
+ }
+ data, err := json.Marshal(record)
+ if err != nil {
+ continue
+ }
+ if _, err := fmt.Fprintf(writer, "event: record\ndata:
%s\n\n", data); err != nil {
+ return
+ }
+ writer.Flush()
+ case <-ticker.C:
+ if _, err := fmt.Fprint(writer, ": keep-alive\n\n");
err != nil {
+ return
+ }
+ writer.Flush()
+ case <-closed:
+ return
+ }
+ }
}
// GetLlmRecord returns one record with its stored request body.
@@ -73,8 +158,19 @@ func (c *ApiController) GetLlmRecord() {
c.ResponseError(err.Error())
return
}
+ if record == nil {
+ c.ResponseError("record not found")
+ return
+ }
- c.ResponseOk(record)
+ // The rates travel with the record, so the detail pane does not match
model
+ // names of its own.
+ price, priced := object.GetLlmPrice(record.Model)
+ if !priced {
+ c.ResponseOk(record)
+ return
+ }
+ c.ResponseOk(record, price)
}
// GetLlmRecordStatus reports the recorder's own settings and how many records
diff --git a/controllers/proxy_record.go b/controllers/proxy_record.go
index 13dd5d1..0966f52 100644
--- a/controllers/proxy_record.go
+++ b/controllers/proxy_record.go
@@ -63,11 +63,23 @@ func (route *proxyRoute) recordUsage(tail []byte) {
}
usage := readUsage(tail)
- route.record.PromptTokens = higher(usage.PromptTokens,
usage.InputTokens)
- route.record.CompletionTokens = higher(usage.CompletionTokens,
usage.OutputTokens)
- route.record.TotalTokens = usage.TotalTokens
- if route.record.TotalTokens == 0 {
- route.record.TotalTokens = route.record.PromptTokens +
route.record.CompletionTokens
+ record := route.record
+ record.CompletionTokens = higher(usage.CompletionTokens,
usage.OutputTokens)
+ record.ReasoningTokens = usage.ReasoningTokens
+ record.CacheWriteTokens = usage.CacheWriteTokens
+ record.CacheReadTokens = higher(usage.CacheReadInputTokens,
usage.CachedTokens)
+
+ // OpenAI counts cached input inside prompt_tokens, Anthropic reports it
+ // beside input_tokens. Taking it out where it is included leaves the
four
+ // counters disjoint, so each can be priced at its own rate.
+ record.PromptTokens = higher(usage.PromptTokens, usage.InputTokens)
+ if usage.PromptTokens > 0 && usage.CachedTokens > 0 {
+ record.PromptTokens =
higher(record.PromptTokens-usage.CachedTokens, 0)
+ }
+
+ record.TotalTokens = record.PromptTokens + record.CompletionTokens +
record.CacheReadTokens + record.CacheWriteTokens
+ if record.TotalTokens == 0 {
+ record.TotalTokens = usage.TotalTokens
}
}
@@ -99,14 +111,42 @@ func (tap *usageTap) keep(chunk []byte) {
tap.tail = append(tap.tail, chunk...)
}
-// llmUsage covers both spellings of the same counters: OpenAI reports
-// prompt/completion tokens, Anthropic input/output ones.
+// llmUsage covers both spellings of the same counters: OpenAI nests the cached
+// and reasoning parts in detail objects, Anthropic reports them flat.
type llmUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
TotalTokens int `json:"total_tokens"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
+
+ CacheWriteTokens int `json:"cache_creation_input_tokens"`
+ CacheReadInputTokens int `json:"cache_read_input_tokens"`
+ // CacheCreation is what cache_creation_input_tokens totals, so it is
only
+ // read when that field is absent.
+ CacheCreation struct {
+ Ephemeral5m int `json:"ephemeral_5m_input_tokens"`
+ Ephemeral1h int `json:"ephemeral_1h_input_tokens"`
+ } `json:"cache_creation"`
+ PromptTokensDetails struct {
+ CachedTokens int `json:"cached_tokens"`
+ } `json:"prompt_tokens_details"`
+ CompletionTokensDetails struct {
+ ReasoningTokens int `json:"reasoning_tokens"`
+ } `json:"completion_tokens_details"`
+
+ // Flattened out of the detail objects above by normalize.
+ CachedTokens int `json:"-"`
+ ReasoningTokens int `json:"-"`
+}
+
+// normalize lifts the nested counters up onto one flat set of fields.
+func (usage *llmUsage) normalize() {
+ usage.CachedTokens = usage.PromptTokensDetails.CachedTokens
+ usage.ReasoningTokens = usage.CompletionTokensDetails.ReasoningTokens
+ if usage.CacheWriteTokens == 0 {
+ usage.CacheWriteTokens = usage.CacheCreation.Ephemeral5m +
usage.CacheCreation.Ephemeral1h
+ }
}
func (usage *llmUsage) merge(other llmUsage) {
@@ -115,6 +155,10 @@ func (usage *llmUsage) merge(other llmUsage) {
usage.TotalTokens = higher(usage.TotalTokens, other.TotalTokens)
usage.InputTokens = higher(usage.InputTokens, other.InputTokens)
usage.OutputTokens = higher(usage.OutputTokens, other.OutputTokens)
+ usage.CacheWriteTokens = higher(usage.CacheWriteTokens,
other.CacheWriteTokens)
+ usage.CacheReadInputTokens = higher(usage.CacheReadInputTokens,
other.CacheReadInputTokens)
+ usage.CachedTokens = higher(usage.CachedTokens, other.CachedTokens)
+ usage.ReasoningTokens = higher(usage.ReasoningTokens,
other.ReasoningTokens)
}
// readUsage merges every usage object in the tail: a stream splits the
counters
@@ -131,14 +175,15 @@ func readUsage(tail []byte) llmUsage {
var parsed llmUsage
if value := balancedObject(rest); value != nil &&
json.Unmarshal(value, &parsed) == nil {
+ parsed.normalize()
usage.merge(parsed)
}
}
}
// balancedObject returns the JSON object that directly follows a key, or nil
-// when the value is not one. Usage objects hold only numbers, so counting
-// braces cannot be thrown off by one inside a string.
+// when the value is not one. Usage objects hold only numbers and objects of
+// numbers, so counting braces cannot be thrown off by one inside a string.
func balancedObject(data []byte) []byte {
start := 0
for start < len(data) && (data[start] == ':' || data[start] == ' ' ||
data[start] == '\t' || data[start] == '\r' || data[start] == '\n') {
diff --git a/object/llm_pricing.go b/object/llm_pricing.go
new file mode 100644
index 0000000..b671f6d
--- /dev/null
+++ b/object/llm_pricing.go
@@ -0,0 +1,123 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package object
+
+import (
+ "encoding/json"
+ "os"
+ "sort"
+ "strings"
+ "sync"
+
+ "github.com/apache/casbin-gateway/conf"
+ "github.com/beego/beego"
+)
+
+// LlmPrice is what one million tokens of each kind costs, in US dollars. APIs
+// that discount cached input instead of charging for the write leave
+// CacheWrite at zero.
+type LlmPrice struct {
+ Input float64 `json:"input"`
+ Output float64 `json:"output"`
+ CacheWrite float64 `json:"cacheWrite"`
+ CacheRead float64 `json:"cacheRead"`
+}
+
+// llmPrices are published list prices, keyed by the part of a model name that
+// identifies the model. They go stale, so llmPricingFile overrides them.
+var llmPrices = map[string]LlmPrice{
+ "claude-opus-4": {15, 75, 18.75, 1.5},
+ "claude-3-opus": {15, 75, 18.75, 1.5},
+ "claude-sonnet-4": {3, 15, 3.75, 0.3},
+ "claude-3-7-sonnet": {3, 15, 3.75, 0.3},
+ "claude-3-5-sonnet": {3, 15, 3.75, 0.3},
+ "claude-haiku-4-5": {1, 5, 1.25, 0.1},
+ "claude-3-5-haiku": {0.8, 4, 1, 0.08},
+ "claude-3-haiku": {0.25, 1.25, 0.3, 0.03},
+
+ "gpt-4o": {2.5, 10, 0, 1.25},
+ "gpt-4o-mini": {0.15, 0.6, 0, 0.075},
+ "gpt-4.1": {2, 8, 0, 0.5},
+ "gpt-4.1-mini": {0.4, 1.6, 0, 0.1},
+ "gpt-4.1-nano": {0.1, 0.4, 0, 0.025},
+ "o3": {2, 8, 0, 0.5},
+ "o3-mini": {1.1, 4.4, 0, 0.55},
+ "o4-mini": {1.1, 4.4, 0, 0.275},
+
+ "deepseek-chat": {0.27, 1.1, 0, 0.07},
+ "deepseek-reasoner": {0.55, 2.19, 0, 0.14},
+}
+
+var (
+ pricingOnce sync.Once
+ pricingKeys []string
+)
+
+// loadLlmPrices merges the override file over the built-in table, longest key
+// first so a specific model wins over the family it belongs to.
+func loadLlmPrices() {
+ path := conf.GetLlmPricingFile()
+ if path != "" {
+ if data, err := os.ReadFile(path); err != nil {
+ if !os.IsNotExist(err) {
+ beego.Error("LLM pricing file could not be
read:", err)
+ }
+ } else {
+ overrides := map[string]LlmPrice{}
+ if err := json.Unmarshal(data, &overrides); err != nil {
+ beego.Error("LLM pricing file is not valid
JSON:", err)
+ } else {
+ for model, price := range overrides {
+ llmPrices[strings.ToLower(model)] =
price
+ }
+ }
+ }
+ }
+
+ pricingKeys = make([]string, 0, len(llmPrices))
+ for key := range llmPrices {
+ pricingKeys = append(pricingKeys, key)
+ }
+ sort.Slice(pricingKeys, func(i, j int) bool { return
len(pricingKeys[i]) > len(pricingKeys[j]) })
+}
+
+// GetLlmPrice matches a model name in any of the shapes it arrives in, e.g.
+// "claude-sonnet-4-20250514" or "us.anthropic.claude-sonnet-4-v1:0". It
reports
+// false when nothing matches, so an unpriced model is not costed at zero.
+func GetLlmPrice(model string) (LlmPrice, bool) {
+ pricingOnce.Do(loadLlmPrices)
+
+ name := strings.ToLower(strings.TrimSpace(model))
+ for _, key := range pricingKeys {
+ if strings.Contains(name, key) {
+ return llmPrices[key], true
+ }
+ }
+ return LlmPrice{}, false
+}
+
+// GetLlmCost is what one recorded request cost, in US dollars.
+func GetLlmCost(model string, promptTokens int, completionTokens int,
cacheWriteTokens int, cacheReadTokens int) (float64, bool) {
+ price, ok := GetLlmPrice(model)
+ if !ok {
+ return 0, false
+ }
+
+ cost := float64(promptTokens)*price.Input +
+ float64(completionTokens)*price.Output +
+ float64(cacheWriteTokens)*price.CacheWrite +
+ float64(cacheReadTokens)*price.CacheRead
+ return cost / 1e6, true
+}
diff --git a/object/llm_record.go b/object/llm_record.go
index 0ee3b6c..8381886 100644
--- a/object/llm_record.go
+++ b/object/llm_record.go
@@ -30,6 +30,14 @@ import (
const llmRecordSummaryRunes = 160
+// llmOversizeFactor is how far past the payload limit a body may still be
+// worth shrinking rather than dropping.
+const llmOversizeFactor = 4
+
+// llmStringCaps are tried in turn on a body over the limit, so that every
+// message and tool stays visible where dropping the body would keep none.
+var llmStringCaps = []int{16384, 4096, 1024, 256}
+
// LlmRecord is one client request relayed by the LLM proxy, together with the
// outcome it ended in. Payload is filled only in "full" mode, and only after
// auditutil has removed the credentials the body may carry.
@@ -50,10 +58,26 @@ type LlmRecord struct {
Attempts int `xorm:"int" json:"attempts"`
Error string `xorm:"varchar(500)" json:"error"`
+ // PromptTokens counts only the input billed as fresh: the cached part
is
+ // reported separately, so the counters can be added up without
counting a
+ // token twice.
PromptTokens int `xorm:"int" json:"promptTokens"`
CompletionTokens int `xorm:"int" json:"completionTokens"`
+ CacheReadTokens int `xorm:"int" json:"cacheReadTokens"`
+ CacheWriteTokens int `xorm:"int" json:"cacheWriteTokens"`
+ ReasoningTokens int `xorm:"int" json:"reasoningTokens"`
TotalTokens int `xorm:"int" json:"totalTokens"`
+ // Cost is in US dollars, and means anything only where Priced is true.
+ Cost float64 `xorm:"double" json:"cost"`
+ Priced bool `xorm:"bool" json:"priced"`
+
+ // Counted while the body is parsed, so the list can show the shape of a
+ // request without loading it.
+ SystemBytes int `xorm:"int" json:"systemBytes"`
+ MessageCount int `xorm:"int" json:"messageCount"`
+ ToolCount int `xorm:"int" json:"toolCount"`
+
Summary string `xorm:"varchar(500)" json:"summary"`
Payload string `xorm:"mediumtext" json:"payload"`
Redactions int `xorm:"int" json:"redactions"`
@@ -68,6 +92,31 @@ type LlmRecordFilter struct {
Agent string
ClientIp string
Outcome string
+ // Since bounds the window to records created after it, formatted the
way
+ // CreatedTime is. Empty means every record still retained.
+ Since string
+}
+
+// LlmModelStat is one model's share of the window the stats cover.
+type LlmModelStat struct {
+ Model string `json:"model"`
+ Requests int64 `json:"requests"`
+ Tokens int64 `json:"tokens"`
+ Cost float64 `json:"cost"`
+}
+
+// LlmRecordStats totals the records a filter matches.
+type LlmRecordStats struct {
+ Requests int64 `json:"requests"`
+ Failed int64 `json:"failed"`
+ PromptTokens int64 `json:"promptTokens"`
+ CompletionTokens int64 `json:"completionTokens"`
+ CacheReadTokens int64 `json:"cacheReadTokens"`
+ CacheWriteTokens int64 `json:"cacheWriteTokens"`
+ TotalTokens int64 `json:"totalTokens"`
+ Cost float64 `json:"cost"`
+ Unpriced int64 `json:"unpriced"`
+ Models []LlmModelStat `json:"models"`
}
// LlmRecordStatus is what the management page shows about the recorder itself,
@@ -143,7 +192,7 @@ func AddLlmRecord(record *LlmRecord, rawBody []byte) {
record.Bytes = len(rawBody)
if conf.GetLlmRecordMode() != conf.LlmRecordFull {
rawBody = nil
- } else if len(rawBody) > auditutil.MaxPayloadBytes {
+ } else if len(rawBody) >
conf.GetLlmRecordMaxPayloadBytes()*llmOversizeFactor {
record.Truncated = true
rawBody = nil
}
@@ -196,9 +245,12 @@ func (writer *llmRecordWriter) write(task llmRecordTask) {
if len(task.rawBody) > 0 {
fillLlmRecordBody(task.record, task.rawBody)
}
+ fillLlmRecordCost(task.record)
if _, err := ormer.Engine.Insert(task.record); err != nil {
beego.Error("LLM record write failed:", err)
+ return
}
+ publishLlmRecord(task.record)
}
func (writer *llmRecordWriter) prune() {
@@ -249,18 +301,100 @@ func fillLlmRecordBody(record *LlmRecord, rawBody
[]byte) {
}
sanitized := auditutil.SanitizeValue("", decoded)
- encoded, err := json.Marshal(sanitized)
- if err != nil {
+ describeLlmRequest(record, sanitized)
+
+ payload, truncated := encodeLlmBody(sanitized,
conf.GetLlmRecordMaxPayloadBytes())
+ record.Payload = payload
+ record.Truncated = record.Truncated || truncated
+ record.Redactions = strings.Count(record.Payload, "[REDACTED")
+ record.Summary = llmRecordSummary(sanitized)
+}
+
+// fillLlmRecordCost prices a record from its token counters.
+func fillLlmRecordCost(record *LlmRecord) {
+ record.Cost, record.Priced = GetLlmCost(
+ record.Model, record.PromptTokens, record.CompletionTokens,
record.CacheWriteTokens, record.CacheReadTokens)
+}
+
+// describeLlmRequest counts what the request was made of.
+func describeLlmRequest(record *LlmRecord, value any) {
+ body, ok := value.(map[string]any)
+ if !ok {
return
}
- if len(encoded) > auditutil.MaxPayloadBytes {
- record.Truncated = true
- record.Payload = auditutil.EncodeBoundedJSON(sanitized,
auditutil.MaxPayloadBytes)
- } else {
- record.Payload = string(encoded)
+ record.SystemBytes = len(llmSystemText(body))
+ if messages, ok := body["messages"].([]any); ok {
+ record.MessageCount = len(messages)
}
- record.Redactions = strings.Count(record.Payload, "[REDACTED")
- record.Summary = llmRecordSummary(sanitized)
+ if tools, ok := body["tools"].([]any); ok {
+ record.ToolCount = len(tools)
+ }
+}
+
+// llmSystemText is the system prompt however the API spells it: a top-level
+// field for Anthropic, the first messages for OpenAI.
+func llmSystemText(body map[string]any) string {
+ if text := llmMessageText(body["system"]); text != "" {
+ return text
+ }
+
+ messages, _ := body["messages"].([]any)
+ parts := []string{}
+ for _, item := range messages {
+ message, ok := item.(map[string]any)
+ if !ok {
+ continue
+ }
+ if role, _ := message["role"].(string); role != "system" &&
role != "developer" {
+ continue
+ }
+ parts = append(parts, llmMessageText(message["content"]))
+ }
+ return strings.Join(parts, "\n")
+}
+
+// encodeLlmBody serializes a body within the limit, capping its strings a step
+// at a time until it fits. It reports whether anything was cut.
+func encodeLlmBody(value any, maximum int) (string, bool) {
+ encoded, err := json.Marshal(value)
+ if err != nil {
+ return "", false
+ }
+ if len(encoded) <= maximum {
+ return string(encoded), false
+ }
+
+ for _, cap := range llmStringCaps {
+ encoded, err = json.Marshal(boundStrings(value, cap))
+ if err != nil {
+ return "", true
+ }
+ if len(encoded) <= maximum {
+ return string(encoded), true
+ }
+ }
+ return auditutil.EncodeBoundedJSON(value, maximum), true
+}
+
+// boundStrings caps every string in a decoded body, leaving its structure
alone.
+func boundStrings(value any, maximum int) any {
+ switch typed := value.(type) {
+ case string:
+ return auditutil.BoundString(typed, maximum)
+ case []any:
+ bounded := make([]any, len(typed))
+ for i, item := range typed {
+ bounded[i] = boundStrings(item, maximum)
+ }
+ return bounded
+ case map[string]any:
+ bounded := make(map[string]any, len(typed))
+ for key, item := range typed {
+ bounded[key] = boundStrings(item, maximum)
+ }
+ return bounded
+ }
+ return value
}
// llmRecordSummary is the last thing the user asked for, which is what makes a
@@ -332,11 +466,15 @@ func llmRecordSession(filter LlmRecordFilter)
*xorm.Session {
if filter.ClientIp != "" {
session = session.And("client_ip = ?", filter.ClientIp)
}
+ if filter.Since != "" {
+ session = session.And("created_time >= ?", filter.Since)
+ }
switch filter.Outcome {
case "ok":
session = session.And("status >= 200").And("status < 300")
case "error":
- session = session.And("status = 0 or status < 200 or status >=
300")
+ // Parenthesized because it sits next to the AND clauses above.
+ session = session.And("(status = 0 or status < 200 or status >=
300)")
}
return session
}
@@ -391,3 +529,124 @@ func GetLlmRecordStatus() (*LlmRecordStatus, error) {
Count: count,
}, nil
}
+
+// GetLlmRecordStats totals the same records GetLlmRecords lists.
+func GetLlmRecordStats(filter LlmRecordFilter, topModels int)
(*LlmRecordStats, error) {
+ countSession := llmRecordSession(filter)
+ defer countSession.Close()
+ requests, err := countSession.Count(&LlmRecord{})
+ if err != nil {
+ return nil, err
+ }
+
+ failedSession := llmRecordSession(filter)
+ defer failedSession.Close()
+ failed, err := failedSession.And("(status = 0 or status < 200 or status
>= 300)").Count(&LlmRecord{})
+ if err != nil {
+ return nil, err
+ }
+
+ sumSession := llmRecordSession(filter)
+ defer sumSession.Close()
+ sums, err := sumSession.SumsInt(&LlmRecord{},
+ "prompt_tokens", "completion_tokens", "cache_read_tokens",
"cache_write_tokens", "total_tokens")
+ if err != nil {
+ return nil, err
+ }
+
+ costSession := llmRecordSession(filter)
+ defer costSession.Close()
+ cost, err := costSession.Sum(&LlmRecord{}, "cost")
+ if err != nil {
+ return nil, err
+ }
+
+ unpricedSession := llmRecordSession(filter)
+ defer unpricedSession.Close()
+ unpriced, err := unpricedSession.And("priced = ?",
false).And("total_tokens > 0").Count(&LlmRecord{})
+ if err != nil {
+ return nil, err
+ }
+
+ stats := &LlmRecordStats{
+ Requests: requests,
+ Failed: failed,
+ PromptTokens: sums[0],
+ CompletionTokens: sums[1],
+ CacheReadTokens: sums[2],
+ CacheWriteTokens: sums[3],
+ TotalTokens: sums[4],
+ Cost: cost,
+ Unpriced: unpriced,
+ Models: []LlmModelStat{},
+ }
+
+ modelSession := llmRecordSession(filter)
+ defer modelSession.Close()
+ err = modelSession.Table("llm_record").
+ Select("model as model, COUNT(*) as requests, SUM(total_tokens)
as tokens, SUM(cost) as cost").
+ GroupBy("model").
+ Desc("requests").
+ Limit(topModels).
+ Find(&stats.Models)
+ if err != nil {
+ return nil, err
+ }
+ return stats, nil
+}
+
+// llmSubscriberBuffer is how far a live watcher may fall behind before it
+// starts losing records.
+const llmSubscriberBuffer = 64
+
+// llmRecordHub fans finished records out to the dashboards watching live. A
+// watcher that cannot keep up loses records rather than holding the writer up.
+type llmRecordHub struct {
+ mutex sync.Mutex
+ subscribers map[int64]chan *LlmRecord
+ nextId int64
+}
+
+var llmHub = llmRecordHub{subscribers: map[int64]chan *LlmRecord{}}
+
+// SubscribeLlmRecords opens a live feed of records as they are written. The
+// caller must pass the id back to UnsubscribeLlmRecords when it is done.
+func SubscribeLlmRecords() (int64, <-chan *LlmRecord) {
+ llmHub.mutex.Lock()
+ defer llmHub.mutex.Unlock()
+
+ llmHub.nextId++
+ id := llmHub.nextId
+ feed := make(chan *LlmRecord, llmSubscriberBuffer)
+ llmHub.subscribers[id] = feed
+ return id, feed
+}
+
+func UnsubscribeLlmRecords(id int64) {
+ llmHub.mutex.Lock()
+ defer llmHub.mutex.Unlock()
+
+ if feed, ok := llmHub.subscribers[id]; ok {
+ delete(llmHub.subscribers, id)
+ close(feed)
+ }
+}
+
+// publishLlmRecord hands a record to every live feed, without its body: the
+// feed keeps a table up to date, and a row is opened one at a time.
+func publishLlmRecord(record *LlmRecord) {
+ llmHub.mutex.Lock()
+ defer llmHub.mutex.Unlock()
+
+ if len(llmHub.subscribers) == 0 {
+ return
+ }
+ summary := *record
+ summary.Payload = ""
+ for _, feed := range llmHub.subscribers {
+ select {
+ case feed <- &summary:
+ default:
+ }
+ }
+}
diff --git a/routers/router.go b/routers/router.go
index 4932a4e..95c036d 100644
--- a/routers/router.go
+++ b/routers/router.go
@@ -102,6 +102,8 @@ func initAPI() {
beego.Router("/api/get-llm-records", &controllers.ApiController{},
"GET:GetLlmRecords")
beego.Router("/api/get-llm-record", &controllers.ApiController{},
"GET:GetLlmRecord")
beego.Router("/api/get-llm-record-status",
&controllers.ApiController{}, "GET:GetLlmRecordStatus")
+ beego.Router("/api/get-llm-record-stats", &controllers.ApiController{},
"GET:GetLlmRecordStats")
+ beego.Router("/api/stream-llm-records", &controllers.ApiController{},
"GET:StreamLlmRecords")
beego.Router("/api/delete-llm-record", &controllers.ApiController{},
"POST:DeleteLlmRecord")
beego.Router("/api/clear-llm-records", &controllers.ApiController{},
"POST:ClearLlmRecords")
diff --git a/web/src/backend/LlmRecordBackend.ts
b/web/src/backend/LlmRecordBackend.ts
index d0d5b38..31a6304 100644
--- a/web/src/backend/LlmRecordBackend.ts
+++ b/web/src/backend/LlmRecordBackend.ts
@@ -13,7 +13,8 @@
// limitations under the License.
import {query, request} from "@/backend/request";
-import type {LlmRecord, LlmRecordStatus} from "@/types";
+import {ServerUrl} from "@/Setting";
+import type {LlmPrice, LlmRecord, LlmRecordStats, LlmRecordStatus} from
"@/types";
export interface LlmRecordFilter {
model?: string;
@@ -21,6 +22,8 @@ export interface LlmRecordFilter {
agent?: string;
clientIp?: string;
outcome?: string;
+ /** How far back the list and the totals reach. 0 is everything. */
+ windowHours?: number;
}
export function getLlmRecords(page = 1, pageSize = 25, filter: LlmRecordFilter
= {}) {
@@ -29,8 +32,37 @@ export function getLlmRecords(page = 1, pageSize = 25,
filter: LlmRecordFilter =
);
}
+/** The second payload is the rates the record was costed at, when it has any.
*/
export function getLlmRecord(id: number) {
- return request<LlmRecord>(`/api/get-llm-record${query({id: id})}`);
+ return request<LlmRecord, LlmPrice>(`/api/get-llm-record${query({id: id})}`);
+}
+
+export function getLlmRecordStats(filter: LlmRecordFilter = {}) {
+ return
request<LlmRecordStats>(`/api/get-llm-record-stats${query({...filter})}`);
+}
+
+/**
+ * Opens the live feed of records; the browser reconnects on its own. Returns
+ * the function that closes it.
+ */
+export function streamLlmRecords(handlers: {
+ onRecord: (record: LlmRecord) => void;
+ onOpen?: () => void;
+ onError?: () => void;
+}) {
+ const source = new EventSource(`${ServerUrl}/api/stream-llm-records`,
{withCredentials: true});
+
+ source.addEventListener("record", event => {
+ try {
+ handlers.onRecord(JSON.parse((event as MessageEvent).data) as LlmRecord);
+ } catch {
+ // A malformed event is not worth tearing the feed down for.
+ }
+ });
+ source.onopen = () => handlers.onOpen?.();
+ source.onerror = () => handlers.onError?.();
+
+ return () => source.close();
}
export function getLlmRecordStatus() {
diff --git a/web/src/components/llm/request-inspector.tsx
b/web/src/components/llm/request-inspector.tsx
new file mode 100644
index 0000000..8857cb3
--- /dev/null
+++ b/web/src/components/llm/request-inspector.tsx
@@ -0,0 +1,312 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import * as React from "react";
+import {ChevronDown, ChevronRight, Database, Image, Wrench} from
"lucide-react";
+import i18next from "i18next";
+
+import {CodeBlock, CodeText, CopyButton} from "@/components/shared/misc";
+import {Badge, type BadgeVariant} from "@/components/ui/badge";
+import {Button} from "@/components/ui/button";
+import {Tabs, TabsContent, TabsList, TabsTrigger} from "@/components/ui/tabs";
+import {cn} from "@/lib/utils";
+import {parseRequest, type ParsedRequest, type RequestBlock, type
RequestMessage, type ToolSchema} from "@/lib/llm-request";
+
+const COLLAPSED_CHARS = 1200;
+
+function formatChars(chars: number) {
+ if (chars < 1000) {
+ return `${chars} ${i18next.t("llm:chars")}`;
+ }
+ return `${(chars / 1000).toFixed(1)}k ${i18next.t("llm:chars")}`;
+}
+
+function formatJson(value: unknown) {
+ try {
+ return JSON.stringify(value, null, 2);
+ } catch {
+ return String(value);
+ }
+}
+
+const ROLE_VARIANTS: Record<string, BadgeVariant> = {
+ user: "info",
+ assistant: "success",
+ system: "warning",
+ developer: "warning",
+ tool: "muted",
+};
+
+/** Text that opens up when it is longer than a screenful. */
+function ExpandableText({text, className}: {text: string; className?: string})
{
+ const [expanded, setExpanded] = React.useState(false);
+ const isLong = text.length > COLLAPSED_CHARS;
+ const shown = expanded || !isLong ? text : text.slice(0, COLLAPSED_CHARS);
+
+ return (
+ <div className="relative">
+ <pre className={cn("scrollbar-thin overflow-x-auto text-xs
leading-relaxed whitespace-pre-wrap", className)}>
+ {shown}
+ {isLong && !expanded ? <span
className="text-muted-foreground">…</span> : null}
+ </pre>
+ {isLong ? (
+ <Button size="sm" variant="ghost" className="mt-1 h-6 px-2"
onClick={() => setExpanded(!expanded)}>
+ {expanded
+ ? i18next.t("llm:Show less")
+ : i18next.t("llm:Show all {chars}").replace("{chars}",
formatChars(text.length))}
+ </Button>
+ ) : null}
+ </div>
+ );
+}
+
+function CacheBadge() {
+ return (
+ <Badge variant="info" title={i18next.t("llm:Cache breakpoint detail")}>
+ <Database />
+ {i18next.t("llm:Cache breakpoint")}
+ </Badge>
+ );
+}
+
+function BlockView({block}: {block: RequestBlock}) {
+ switch (block.kind) {
+ case "text":
+ return <ExpandableText text={block.text ?? ""} />;
+ case "thinking":
+ return (
+ <div className="grid gap-1">
+ <Badge variant="muted">{i18next.t("llm:Thinking")}</Badge>
+ <ExpandableText className="text-muted-foreground italic"
text={block.text ?? ""} />
+ </div>
+ );
+ case "image":
+ return (
+ <Badge variant="muted">
+ <Image />
+ {block.mediaType} · {formatChars(block.chars)}
+ </Badge>
+ );
+ case "toolUse":
+ return (
+ <div className="grid gap-1">
+ <span className="flex items-center gap-2">
+ <Badge variant="warning">
+ <Wrench />
+ {block.toolName}
+ </Badge>
+ {block.toolUseId ? <span className="text-muted-foreground font-mono
text-xs">{block.toolUseId}</span> : null}
+ </span>
+ <CodeBlock copyable maxHeight="16rem">
+ {formatJson(block.data)}
+ </CodeBlock>
+ </div>
+ );
+ case "toolResult":
+ return (
+ <div className="grid gap-1">
+ <span className="flex items-center gap-2">
+ <Badge variant={block.isError ? "danger" : "muted"}>
+ {block.isError ? i18next.t("llm:Tool error") : i18next.t("llm:Tool
result")}
+ </Badge>
+ {block.toolUseId ? <span className="text-muted-foreground font-mono
text-xs">{block.toolUseId}</span> : null}
+ </span>
+ {block.text ? <ExpandableText text={block.text} /> : <CodeBlock
maxHeight="16rem">{formatJson(block.data)}</CodeBlock>}
+ </div>
+ );
+ default:
+ return <CodeBlock maxHeight="16rem">{formatJson(block.data)}</CodeBlock>;
+ }
+}
+
+function MessageView({message}: {message: RequestMessage}) {
+ return (
+ <div className="rounded-lg border">
+ <div className="bg-muted/40 flex flex-wrap items-center gap-2 border-b
px-3 py-1.5">
+ <Badge variant={ROLE_VARIANTS[message.role] ??
"secondary"}>{message.role || i18next.t("llm:Unknown role")}</Badge>
+ <span className="text-muted-foreground text-xs">#{message.index +
1}</span>
+ <span className="text-muted-foreground
text-xs">{formatChars(message.chars)}</span>
+ {message.blocks.some(block => block.cached) ? <CacheBadge /> : null}
+ </div>
+ <div className="grid gap-3 px-3 py-2">
+ {message.blocks.length === 0 ? (
+ <span className="text-muted-foreground
text-xs">{i18next.t("llm:Empty message")}</span>
+ ) : (
+ message.blocks.map((block, index) => <BlockView key={index}
block={block} />)
+ )}
+ </div>
+ </div>
+ );
+}
+
+function ToolView({tool}: {tool: ToolSchema}) {
+ const [expanded, setExpanded] = React.useState(false);
+
+ return (
+ <div className="rounded-lg border">
+ <button
+ type="button"
+ className="hover:bg-muted/40 flex w-full items-center gap-2 px-3 py-2
text-left"
+ onClick={() => setExpanded(!expanded)}
+ >
+ {expanded ? <ChevronDown className="size-4 shrink-0" /> :
<ChevronRight className="size-4 shrink-0" />}
+ <code className="font-mono text-xs font-medium">{tool.name}</code>
+ <span className="text-muted-foreground min-w-0 flex-1 truncate
text-xs">{tool.description}</span>
+ {tool.cached ? <CacheBadge /> : null}
+ <Badge variant="muted">
+ {tool.parameters.length} {i18next.t("llm:params")}
+ </Badge>
+ <span className="text-muted-foreground
text-xs">{formatChars(tool.chars)}</span>
+ </button>
+
+ {expanded ? (
+ <div className="grid gap-3 border-t px-3 py-2">
+ {tool.description ? <ExpandableText
className="text-muted-foreground" text={tool.description} /> : null}
+ {tool.parameters.length > 0 ? (
+ <table className="w-full text-xs">
+ <tbody>
+ {tool.parameters.map(parameter => (
+ <tr key={parameter.name} className="border-b last:border-0">
+ <td className="py-1 pr-3 align-top font-mono
whitespace-nowrap">{parameter.name}</td>
+ <td className="text-muted-foreground py-1 pr-3 align-top
whitespace-nowrap">{parameter.type}</td>
+ <td className="py-1 pr-3 align-top whitespace-nowrap">
+ {parameter.required ? <Badge
variant="warning">{i18next.t("llm:required")}</Badge> : null}
+ </td>
+ <td className="text-muted-foreground py-1
align-top">{parameter.description}</td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ ) : null}
+ <CodeBlock copyable maxHeight="20rem">
+ {formatJson(tool.schema)}
+ </CodeBlock>
+ </div>
+ ) : null}
+ </div>
+ );
+}
+
+function SystemView({request}: {request: ParsedRequest}) {
+ if (request.system.length === 0) {
+ return <span className="text-muted-foreground text-xs">{i18next.t("llm:No
system prompt")}</span>;
+ }
+
+ return (
+ <div className="grid gap-3">
+ <div className="flex items-center gap-2">
+ <span className="text-muted-foreground text-xs">
+ {request.system.length} {i18next.t("llm:blocks")} ·
{formatChars(request.systemChars)}
+ </span>
+ <CopyButton value={request.system.map(block => block.text ??
"").join("\n\n")} />
+ </div>
+ {request.system.map((block, index) => (
+ <div key={index} className="rounded-lg border px-3 py-2">
+ {block.cached ? (
+ <div className="mb-1">
+ <CacheBadge />
+ </div>
+ ) : null}
+ <BlockView block={block} />
+ </div>
+ ))}
+ </div>
+ );
+}
+
+/**
+ * Everything the client sent this turn: the system prompt, every message in
+ * order, and the schema of every tool the model was offered.
+ */
+export function RequestInspector({payload}: {payload: string}) {
+ const request = React.useMemo(() => parseRequest(payload), [payload]);
+ const [tab, setTab] = React.useState("messages");
+
+ if (request.invalid) {
+ return (
+ <div className="grid gap-2">
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:Body
is not JSON")}</span>
+ <CodeBlock copyable maxHeight="24rem">
+ {request.raw}
+ </CodeBlock>
+ </div>
+ );
+ }
+
+ return (
+ <Tabs value={tab} onValueChange={setTab} className="gap-3">
+ <TabsList>
+ <TabsTrigger value="messages">
+ {i18next.t("llm:Messages")}
+ <Badge variant="muted">{request.messages.length}</Badge>
+ </TabsTrigger>
+ <TabsTrigger value="system">
+ {i18next.t("llm:System prompt")}
+ {request.systemChars > 0 ? <Badge
variant="muted">{formatChars(request.systemChars)}</Badge> : null}
+ </TabsTrigger>
+ <TabsTrigger value="tools">
+ {i18next.t("llm:Tools")}
+ <Badge variant="muted">{request.tools.length}</Badge>
+ </TabsTrigger>
+ <TabsTrigger value="params">{i18next.t("llm:Parameters")}</TabsTrigger>
+ <TabsTrigger value="raw">{i18next.t("llm:Raw")}</TabsTrigger>
+ </TabsList>
+
+ <TabsContent value="messages" className="grid gap-2">
+ {request.messages.length === 0 ? (
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:No
messages")}</span>
+ ) : (
+ request.messages.map(message => <MessageView key={message.index}
message={message} />)
+ )}
+ </TabsContent>
+
+ <TabsContent value="system">
+ <SystemView request={request} />
+ </TabsContent>
+
+ <TabsContent value="tools" className="grid gap-2">
+ {request.tools.length === 0 ? (
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:No
tools")}</span>
+ ) : (
+ request.tools.map(tool => <ToolView key={tool.name} tool={tool} />)
+ )}
+ </TabsContent>
+
+ <TabsContent value="params">
+ {request.params.length === 0 ? (
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:No
parameters")}</span>
+ ) : (
+ <table className="w-full text-sm">
+ <tbody>
+ {request.params.map(parameter => (
+ <tr key={parameter.key} className="border-b last:border-0">
+ <td className="text-muted-foreground w-[220px] py-1.5 pr-3
align-top text-xs">{parameter.key}</td>
+ <td className="py-1.5 align-top break-all">
+ <CodeText>{parameter.value}</CodeText>
+ </td>
+ </tr>
+ ))}
+ </tbody>
+ </table>
+ )}
+ </TabsContent>
+
+ <TabsContent value="raw">
+ <CodeBlock copyable maxHeight="30rem">
+ {request.raw}
+ </CodeBlock>
+ </TabsContent>
+ </Tabs>
+ );
+}
diff --git a/web/src/lib/llm-request.ts b/web/src/lib/llm-request.ts
new file mode 100644
index 0000000..b25214e
--- /dev/null
+++ b/web/src/lib/llm-request.ts
@@ -0,0 +1,317 @@
+// Copyright 2026 The casbin Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/**
+ * Reads a recorded request body into the shape the inspector renders. The two
+ * APIs describe the same conversation differently, so both are normalized here
+ * and the inspector renders one shape.
+ */
+
+export type BlockKind = "text" | "thinking" | "image" | "toolUse" |
"toolResult" | "other";
+
+export interface RequestBlock {
+ kind: BlockKind;
+ text?: string;
+ toolName?: string;
+ toolUseId?: string;
+ /** A tool call's arguments, or the raw block when nothing else fits. */
+ data?: unknown;
+ isError?: boolean;
+ mediaType?: string;
+ /** The block carries a cache_control marker. */
+ cached?: boolean;
+ chars: number;
+}
+
+export interface RequestMessage {
+ index: number;
+ role: string;
+ blocks: RequestBlock[];
+ chars: number;
+}
+
+export interface ToolParameter {
+ name: string;
+ type: string;
+ required: boolean;
+ description: string;
+}
+
+export interface ToolSchema {
+ name: string;
+ description: string;
+ parameters: ToolParameter[];
+ schema: unknown;
+ cached: boolean;
+ chars: number;
+}
+
+export interface ParsedRequest {
+ system: RequestBlock[];
+ systemChars: number;
+ messages: RequestMessage[];
+ tools: ToolSchema[];
+ /** Everything else the body carried, e.g. max_tokens or tool_choice. */
+ params: {key: string; value: string}[];
+ raw: string;
+ /** The body was not JSON, so only the raw tab has anything to show. */
+ invalid: boolean;
+ /** Too large even after shrinking, so only a preview is left. */
+ previewOnly: boolean;
+}
+
+const SYSTEM_ROLES = ["system", "developer"];
+
+/** The keys the inspector renders in tabs of their own, not as parameters. */
+const OWN_TABS = ["system", "messages", "tools"];
+
+function textOf(value: unknown): string {
+ if (typeof value === "string") {
+ return value;
+ }
+ if (Array.isArray(value)) {
+ return value.map(textOf).filter(Boolean).join("\n\n");
+ }
+ if (value && typeof value === "object") {
+ const text = (value as Record<string, unknown>).text;
+ return typeof text === "string" ? text : "";
+ }
+ return "";
+}
+
+function isCached(block: unknown): boolean {
+ return Boolean(block && typeof block === "object" && (block as
Record<string, unknown>).cache_control);
+}
+
+function textBlock(text: string, kind: BlockKind = "text", cached = false):
RequestBlock {
+ return {kind: kind, text: text, cached: cached, chars: text.length};
+}
+
+/** One content block, in either API's spelling. */
+function readBlock(value: unknown): RequestBlock {
+ if (typeof value === "string") {
+ return textBlock(value);
+ }
+ if (!value || typeof value !== "object") {
+ return {kind: "other", data: value, chars: 0};
+ }
+
+ const block = value as Record<string, any>;
+ const cached = isCached(block);
+ switch (block.type) {
+ case "text":
+ case "input_text":
+ case "output_text":
+ return textBlock(String(block.text ?? ""), "text", cached);
+ case "thinking":
+ case "reasoning":
+ return textBlock(String(block.thinking ?? block.text ?? ""), "thinking",
cached);
+ case "redacted_thinking":
+ return {kind: "thinking", text: "", cached: cached, chars:
String(block.data ?? "").length};
+ case "image":
+ case "image_url":
+ case "input_image":
+ return {
+ kind: "image",
+ mediaType: String(block.source?.media_type ?? block.source?.type ??
"image"),
+ cached: cached,
+ chars: String(block.source?.data ?? block.image_url?.url ?? "").length,
+ };
+ case "tool_use":
+ case "server_tool_use":
+ return {
+ kind: "toolUse",
+ toolName: String(block.name ?? ""),
+ toolUseId: String(block.id ?? ""),
+ data: block.input,
+ cached: cached,
+ chars: JSON.stringify(block.input ?? "").length,
+ };
+ case "tool_result": {
+ const text = textOf(block.content);
+ return {
+ kind: "toolResult",
+ toolUseId: String(block.tool_use_id ?? ""),
+ text: text,
+ data: text === "" ? block.content : undefined,
+ isError: Boolean(block.is_error),
+ cached: cached,
+ chars: text.length,
+ };
+ }
+ default: {
+ const text = textOf(block);
+ if (text !== "") {
+ return textBlock(text, "text", cached);
+ }
+ return {kind: "other", data: block, cached: cached, chars:
JSON.stringify(block ?? "").length};
+ }
+ }
+}
+
+function readContent(content: unknown): RequestBlock[] {
+ if (content === undefined || content === null || content === "") {
+ return [];
+ }
+ return (Array.isArray(content) ? content : [content]).map(readBlock);
+}
+
+/** An OpenAI assistant message asks for its tools in a field of its own. */
+function readToolCalls(message: Record<string, any>): RequestBlock[] {
+ if (!Array.isArray(message.tool_calls)) {
+ return [];
+ }
+ return message.tool_calls.map((call: Record<string, any>) => {
+ const args = call.function?.arguments;
+ let parsed: unknown = args;
+ try {
+ parsed = typeof args === "string" ? JSON.parse(args) : args;
+ } catch {
+ // Cut short by the capture limit, so the text is what there is to show.
+ }
+ return {
+ kind: "toolUse" as const,
+ toolName: String(call.function?.name ?? call.name ?? ""),
+ toolUseId: String(call.id ?? ""),
+ data: parsed,
+ chars: String(args ?? "").length,
+ };
+ });
+}
+
+function readTool(value: unknown): ToolSchema | null {
+ if (!value || typeof value !== "object") {
+ return null;
+ }
+ const tool = value as Record<string, any>;
+ // OpenAI wraps the tool in a function envelope, Anthropic does not.
+ const inner = tool.function && typeof tool.function === "object" ?
tool.function : tool;
+ const schema = inner.input_schema ?? inner.parameters ?? inner.schema;
+ const properties = (schema?.properties ?? {}) as Record<string, any>;
+ const required: string[] = Array.isArray(schema?.required) ? schema.required
: [];
+
+ return {
+ name: String(inner.name ?? tool.type ?? ""),
+ description: String(inner.description ?? ""),
+ parameters: Object.keys(properties).map(name => ({
+ name: name,
+ type: typeOfProperty(properties[name]),
+ required: required.includes(name),
+ description: String(properties[name]?.description ?? ""),
+ })),
+ schema: schema ?? tool,
+ cached: isCached(tool),
+ chars: JSON.stringify(tool ?? "").length,
+ };
+}
+
+function typeOfProperty(property: any): string {
+ if (!property || typeof property !== "object") {
+ return "";
+ }
+ if (typeof property.type === "string") {
+ return property.type;
+ }
+ const alternatives = property.anyOf ?? property.oneOf;
+ if (Array.isArray(alternatives)) {
+ return alternatives.map((one: any) => one?.type).filter(Boolean).join(" |
");
+ }
+ return property.enum ? "enum" : "";
+}
+
+export function parseRequest(payload: string): ParsedRequest {
+ const empty: ParsedRequest = {
+ system: [],
+ systemChars: 0,
+ messages: [],
+ tools: [],
+ params: [],
+ raw: payload,
+ invalid: false,
+ previewOnly: false,
+ };
+ if (!payload) {
+ return empty;
+ }
+
+ let body: Record<string, any>;
+ try {
+ body = JSON.parse(payload);
+ } catch {
+ return {...empty, invalid: true};
+ }
+ if (!body || typeof body !== "object") {
+ return {...empty, invalid: true};
+ }
+
+ const raw = JSON.stringify(body, null, 2);
+ if (body.truncated === true && typeof body.preview === "string") {
+ return {...empty, raw: raw, previewOnly: true};
+ }
+
+ const system = readContent(body.system);
+ const messages: RequestMessage[] = [];
+ const rawMessages: unknown[] = Array.isArray(body.messages) ? body.messages
: [];
+ rawMessages.forEach((item, index) => {
+ if (!item || typeof item !== "object") {
+ return;
+ }
+ const message = item as Record<string, any>;
+ const role = String(message.role ?? "");
+ const blocks = [...readContent(message.content),
...readToolCalls(message)];
+
+ // Lifted out so the System tab means the same for both APIs.
+ if (SYSTEM_ROLES.includes(role) && messages.length === 0) {
+ system.push(...blocks);
+ return;
+ }
+ // An OpenAI tool reply is a message of its own rather than a block.
+ if (role === "tool" && message.tool_call_id) {
+ blocks.forEach(block => {
+ block.kind = "toolResult";
+ block.toolUseId = String(message.tool_call_id);
+ block.toolName = String(message.name ?? "");
+ });
+ }
+
+ messages.push({
+ index: index,
+ role: role,
+ blocks: blocks,
+ chars: blocks.reduce((total, block) => total + block.chars, 0),
+ });
+ });
+
+ const tools = (Array.isArray(body.tools) ? body.tools : [])
+ .map(readTool)
+ .filter((tool): tool is ToolSchema => tool !== null);
+
+ const params = Object.keys(body)
+ .filter(key => !OWN_TABS.includes(key))
+ .map(key => ({
+ key: key,
+ value: typeof body[key] === "object" ? JSON.stringify(body[key]) :
String(body[key]),
+ }));
+
+ return {
+ system: system,
+ systemChars: system.reduce((total, block) => total + block.chars, 0),
+ messages: messages,
+ tools: tools,
+ params: params,
+ raw: raw,
+ invalid: false,
+ previewOnly: false,
+ };
+}
diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json
index 076651b..5cf7830 100644
--- a/web/src/locales/en/data.json
+++ b/web/src/locales/en/data.json
@@ -289,32 +289,91 @@
"{total} in total": "{total} in total"
},
"llm": {
+ "All time": "All time",
"Any outcome": "Any outcome",
"Attempts": "Attempts",
+ "Bodies are not stored": "Request bodies are not stored",
+ "Bodies are not stored detail": "llmRecordMode is \"metadata\", so the
prompts, tool schemas and messages behind these rows were never kept. Set it to
\"full\" in conf/app.conf and restart Gateway to inspect them.",
+ "Body is not JSON": "The stored body is not valid JSON, so it is shown as
it was captured.",
"Body not stored": "The request body was not stored: llmRecordMode is
\"metadata\".",
"Body too large to store": "The request body was larger than the capture
limit and was not stored.",
+ "Body was shortened": "The body was over the capture limit, so its longest
strings were cut. Every message, tool and parameter is still listed. Raise
llmRecordMaxPayloadBytes to keep more.",
+ "Cache breakpoint": "cached",
+ "Cache breakpoint detail": "This block carries cache_control, so the
prompt cache is written up to here.",
+ "Cache hit rate": "Cache hit rate",
+ "Cache read": "Cache read",
+ "Cache write": "Cache write",
+ "Cached input": "Cached input",
"Channel": "Channel",
"Clear": "Clear",
+ "Cost": "Cost",
"Delete every stored record?": "Delete every stored record?",
+ "Empty message": "This message carried no content.",
"Endpoint": "Endpoint",
"Error": "Error",
"Failed": "Failed",
+ "Feed connected": "Live feed connected",
+ "Feed connecting": "Live feed reconnecting",
+ "Fresh input": "Fresh input",
+ "Input tokens": "Input tokens",
"LLM Records": "LLM Records",
+ "Last 24 hours": "Last 24 hours",
+ "Last 7 days": "Last 7 days",
+ "Last hour": "Last hour",
+ "List prices, see llmPricingFile": "List prices - override them with
llmPricingFile.",
+ "Live": "Live",
+ "Messages": "Messages",
"Mode full": "full - metadata and sanitized request body",
"Mode metadata": "metadata only",
"Mode off": "off",
"Mode {mode}, kept for {days} days, up to {max} records": "Mode {mode},
kept for {days} days, up to {max} records",
+ "No messages": "This request carried no messages.",
+ "No parameters": "This request carried no parameters of its own.",
+ "No price for this model": "No price is configured for this model",
"No records yet": "No records yet - send a request through the LLM
gateway.",
"No response": "no response",
+ "No system prompt": "This request carried no system prompt.",
+ "No tools": "The model was offered no tools.",
+ "Output": "Output",
+ "Output tokens": "Output tokens",
+ "Parameters": "Parameters",
+ "Raw": "Raw",
"Recording is off": "Recording is off",
"Recording is off detail": "Set llmRecordMode to \"metadata\" or \"full\"
in conf/app.conf and restart Gateway. Prompts can carry sensitive data, so
nothing is retained until you ask for it.",
"Records were dropped": "{count} records were dropped because the write
queue was full. Raise llmRecordQueueCapacity if this keeps happening.",
"Redacted values": "Redacted values",
"Request": "Request",
"Request size": "Request size",
+ "Requests": "Requests",
+ "Show all {chars}": "Show all {chars}",
+ "Show less": "Show less",
"Status": "Status",
"Succeeded": "Succeeded",
- "Tokens": "Tokens"
+ "System prompt": "System prompt",
+ "Thinking": "thinking",
+ "Tokens": "Tokens",
+ "Tool error": "tool error",
+ "Tool result": "tool result",
+ "Tools": "Tools",
+ "Unknown role": "unknown role",
+ "all succeeded": "all succeeded",
+ "blocks": "blocks",
+ "chars": "chars",
+ "failed": "failed",
+ "fresh": "fresh",
+ "in": "in",
+ "messages": "messages",
+ "out": "out",
+ "params": "params",
+ "per million": "per million",
+ "read": "read",
+ "reasoning": "reasoning",
+ "required": "required",
+ "system": "system",
+ "tools": "tools",
+ "written": "written",
+ "{count} new records": "{count} new records",
+ "{count} records have no price": "{count} records have no price"
},
"node": {
"Edit Node": "Edit Node",
diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json
index 1a8d651..b20d684 100644
--- a/web/src/locales/zh/data.json
+++ b/web/src/locales/zh/data.json
@@ -289,32 +289,91 @@
"{total} in total": "共 {total} 条"
},
"llm": {
+ "All time": "全部时间",
"Any outcome": "全部结果",
"Attempts": "尝试次数",
+ "Bodies are not stored": "未保存请求体",
+ "Bodies are not stored detail": "llmRecordMode 为
\"metadata\"，因此这些记录背后的提示词、工具 schema 和消息都没有保存。在 conf/app.conf 中改为 \"full\" 并重启
Gateway 后即可查看。",
+ "Body is not JSON": "保存的请求体不是合法 JSON，这里按捕获时的原样显示。",
"Body not stored": "未保存请求正文：llmRecordMode 为 \"metadata\"。",
"Body too large to store": "请求正文超过采集上限，未予保存。",
+ "Body was shortened": "请求体超出了捕获上限，其中最长的字符串被截断，但每条消息、每个工具和每个参数都仍然完整列出。调大
llmRecordMaxPayloadBytes 可保留更多内容。",
+ "Cache breakpoint": "缓存点",
+ "Cache breakpoint detail": "该块带有 cache_control，提示词缓存写到此处为止。",
+ "Cache hit rate": "缓存命中率",
+ "Cache read": "缓存读取",
+ "Cache write": "缓存写入",
+ "Cached input": "缓存输入",
"Channel": "渠道",
"Clear": "清空",
+ "Cost": "费用",
"Delete every stored record?": "确定删除全部已保存的记录？",
+ "Empty message": "该消息没有内容。",
"Endpoint": "接口",
"Error": "错误",
"Failed": "失败",
+ "Feed connected": "实时推送已连接",
+ "Feed connecting": "实时推送重连中",
+ "Fresh input": "新输入",
+ "Input tokens": "输入 Token",
"LLM Records": "LLM 记录",
+ "Last 24 hours": "最近 24 小时",
+ "Last 7 days": "最近 7 天",
+ "Last hour": "最近 1 小时",
+ "List prices, see llmPricingFile": "按官方标价估算，可用 llmPricingFile 覆盖。",
+ "Live": "实时",
+ "Messages": "消息",
"Mode full": "完整（元数据 + 脱敏后的请求正文）",
"Mode metadata": "仅元数据",
"Mode off": "关闭",
"Mode {mode}, kept for {days} days, up to {max} records": "模式：{mode}，保留
{days} 天，最多 {max} 条",
+ "No messages": "该请求没有消息。",
+ "No parameters": "该请求没有额外参数。",
+ "No price for this model": "该模型没有配置价格",
"No records yet": "暂无记录——请通过 LLM 网关发送一次请求。",
"No response": "无响应",
+ "No system prompt": "该请求没有 system prompt。",
+ "No tools": "该请求没有向模型提供工具。",
+ "Output": "输出",
+ "Output tokens": "输出 Token",
+ "Parameters": "参数",
+ "Raw": "原始 JSON",
"Recording is off": "记录功能已关闭",
"Recording is off detail": "请在 conf/app.conf 中把 llmRecordMode 设为
\"metadata\" 或 \"full\" 并重启 Gateway。提示词可能包含敏感数据，因此默认不保留任何内容。",
"Records were dropped": "有 {count} 条记录因写入队列已满而被丢弃。如持续出现，请调大
llmRecordQueueCapacity。",
"Redacted values": "已脱敏字段数",
"Request": "请求内容",
"Request size": "请求大小",
+ "Requests": "请求数",
+ "Show all {chars}": "展开全部 {chars}",
+ "Show less": "收起",
"Status": "状态",
"Succeeded": "成功",
- "Tokens": "Token 用量"
+ "System prompt": "System prompt",
+ "Thinking": "思考",
+ "Tokens": "Token 用量",
+ "Tool error": "工具错误",
+ "Tool result": "工具结果",
+ "Tools": "工具",
+ "Unknown role": "未知角色",
+ "all succeeded": "全部成功",
+ "blocks": "块",
+ "chars": "字符",
+ "failed": "失败",
+ "fresh": "新增",
+ "in": "输入",
+ "messages": "条消息",
+ "out": "输出",
+ "params": "参数",
+ "per million": "每百万 Token",
+ "read": "读取",
+ "reasoning": "推理",
+ "required": "必填",
+ "system": "system",
+ "tools": "个工具",
+ "written": "写入",
+ "{count} new records": "{count} 条新记录",
+ "{count} records have no price": "{count} 条记录没有价格"
},
"node": {
"Edit Node": "编辑节点",
diff --git a/web/src/pages/LlmRecordsPage.tsx b/web/src/pages/LlmRecordsPage.tsx
index 784b135..5a1a3bb 100644
--- a/web/src/pages/LlmRecordsPage.tsx
+++ b/web/src/pages/LlmRecordsPage.tsx
@@ -13,23 +13,29 @@
// limitations under the License.
import * as React from "react";
-import {Logs, RefreshCw, Trash2} from "lucide-react";
+import {Activity, CircleDollarSign, Database, Hash, Logs, RefreshCw, Trash2}
from "lucide-react";
import i18next from "i18next";
import * as LlmRecordBackend from "@/backend/LlmRecordBackend";
import * as Setting from "@/Setting";
+import {RequestInspector} from "@/components/llm/request-inspector";
import {DataTable, type Column} from "@/components/shared/data-table";
import {ConfirmDialog} from "@/components/shared/confirm-dialog";
-import {CodeBlock, CodeText, DescriptionList, UnauthorizedResult} from
"@/components/shared/misc";
+import {Loading} from "@/components/shared/loading";
+import {CodeText, DescriptionList, UnauthorizedResult} from
"@/components/shared/misc";
import {PageContainer, PageHeader} from "@/components/shared/page-header";
import {SimpleSelect} from "@/components/shared/simple-select";
+import {StatCard} from "@/components/shared/stat-card";
import {MessageAlert} from "@/components/ui/alert";
import {Badge, type BadgeVariant} from "@/components/ui/badge";
import {Button} from "@/components/ui/button";
import {Input} from "@/components/ui/input";
import {Label} from "@/components/ui/label";
import {Switch} from "@/components/ui/switch";
-import type {Account, LlmRecord, LlmRecordStatus} from "@/types";
+import type {Account, LlmPrice, LlmRecord, LlmRecordStats, LlmRecordStatus}
from "@/types";
+
+/** How often the totals are recomputed while the live feed is open. */
+const STATS_INTERVAL = 10000;
function statusVariant(status: number): BadgeVariant {
if (status >= 200 && status < 300) {
@@ -41,29 +47,70 @@ function statusVariant(status: number): BadgeVariant {
return "danger";
}
-function formatPayload(payload: string) {
- try {
- return JSON.stringify(JSON.parse(payload), null, 2);
- } catch {
- return payload;
+function formatCost(cost: number) {
+ if (cost <= 0) {
+ return "$0";
+ }
+ if (cost < 0.01) {
+ return `$${cost.toFixed(4)}`;
+ }
+ if (cost < 1) {
+ return `$${cost.toFixed(3)}`;
}
+ return `$${cost.toFixed(2)}`;
+}
+
+function formatTokens(tokens: number) {
+ if (tokens < 1000) {
+ return String(tokens);
+ }
+ if (tokens < 1000000) {
+ return `${(tokens / 1000).toFixed(1)}k`;
+ }
+ return `${(tokens / 1000000).toFixed(2)}M`;
+}
+
+/** Share of the input tokens served out of the prompt cache. */
+function cacheHitRate(stats: LlmRecordStats) {
+ const input = stats.promptTokens + stats.cacheReadTokens +
stats.cacheWriteTokens;
+ return input === 0 ? 0 : (stats.cacheReadTokens / input) * 100;
+}
+
+function tokenBreakdown(record: LlmRecord) {
+ return [
+ `${i18next.t("llm:Fresh input")}: ${record.promptTokens.toLocaleString()}`,
+ `${i18next.t("llm:Cache read")}:
${record.cacheReadTokens.toLocaleString()}`,
+ `${i18next.t("llm:Cache write")}:
${record.cacheWriteTokens.toLocaleString()}`,
+ `${i18next.t("llm:Output")}: ${record.completionTokens.toLocaleString()}`,
+ ].join("\n");
}
/** Mounted only once a row is expanded, which is when the body is fetched. */
function RecordDetail({record, onDelete}: {record: LlmRecord; onDelete: () =>
void}) {
const [detail, setDetail] = React.useState<LlmRecord | null>(null);
+ const [price, setPrice] = React.useState<LlmPrice | null>(null);
const [error, setError] = React.useState("");
React.useEffect(() => {
LlmRecordBackend.getLlmRecord(record.id)
- .then(res => (res.status === "ok" ? setDetail(res.data ?? null) :
setError(res.msg)))
+ .then(res => {
+ if (res.status === "ok") {
+ setDetail(res.data ?? null);
+ setPrice(res.data2 ?? null);
+ } else {
+ setError(res.msg);
+ }
+ })
.catch(err => setError(err.message || String(err)));
}, [record.id]);
const payload = detail?.payload ?? "";
+ const rate = price
+ ? `${formatCost(price.input)} / ${formatCost(price.output)}
${i18next.t("llm:per million")}`
+ : i18next.t("llm:No price for this model");
return (
- <div className="flex flex-col gap-3 px-4 py-3">
+ <div className="flex flex-col gap-4 px-4 py-3">
<DescriptionList
columns={3}
items={[
@@ -73,7 +120,25 @@ function RecordDetail({record, onDelete}: {record:
LlmRecord; onDelete: () => vo
record.channel && {label: i18next.t("llm:Channel"), value:
<CodeText>{record.channel}</CodeText>},
record.agent && {label: i18next.t("agent:Agent"), value:
<CodeText>{record.agent}</CodeText>},
{label: i18next.t("llm:Attempts"), value: record.attempts},
- {label: i18next.t("llm:Tokens"), value: `${record.promptTokens} +
${record.completionTokens} = ${record.totalTokens}`},
+ {
+ label: i18next.t("llm:Input tokens"),
+ value: `${record.promptTokens.toLocaleString()}
${i18next.t("llm:fresh")}`,
+ },
+ {
+ label: i18next.t("llm:Cached input"),
+ value: `${record.cacheReadTokens.toLocaleString()}
${i18next.t("llm:read")} · ${record.cacheWriteTokens.toLocaleString()}
${i18next.t("llm:written")}`,
+ },
+ {
+ label: i18next.t("llm:Output tokens"),
+ value:
+ record.reasoningTokens > 0
+ ? `${record.completionTokens.toLocaleString()} ·
${record.reasoningTokens.toLocaleString()} ${i18next.t("llm:reasoning")}`
+ : record.completionTokens.toLocaleString(),
+ },
+ {
+ label: i18next.t("llm:Cost"),
+ value: record.priced ? `${formatCost(record.cost)} · ${rate}` :
i18next.t("llm:No price for this model"),
+ },
{label: i18next.t("llm:Request size"), value:
`${record.bytes.toLocaleString()} B`},
{label: i18next.t("llm:Redacted values"), value: record.redactions},
record.error && {label: i18next.t("llm:Error"), value: record.error},
@@ -81,19 +146,13 @@ function RecordDetail({record, onDelete}: {record:
LlmRecord; onDelete: () => vo
/>
{error ? <MessageAlert title={error} /> : null}
+ {record.truncated ? <MessageAlert variant="warning"
title={i18next.t("llm:Body was shortened")} /> : null}
- {payload ? (
- <div className="grid gap-1">
- <span className="text-muted-foreground
text-xs">{i18next.t("agent:Payload")}</span>
- <CodeBlock copyable maxHeight="24rem">
- {formatPayload(payload)}
- </CodeBlock>
- </div>
- ) : (
- <span className="text-muted-foreground text-xs">
- {record.truncated ? i18next.t("llm:Body too large to store") :
i18next.t("llm:Body not stored")}
- </span>
- )}
+ {payload ? <RequestInspector payload={payload} /> : null}
+ {!payload && detail === null && error === "" ? <Loading type="small" />
: null}
+ {!payload && detail !== null ? (
+ <span className="text-muted-foreground text-xs">{i18next.t("llm:Body
not stored")}</span>
+ ) : null}
<div>
<ConfirmDialog
@@ -114,36 +173,46 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
const isAdmin = Setting.isAdminUser(account);
const [records, setRecords] = React.useState<LlmRecord[]>([]);
const [status, setStatus] = React.useState<LlmRecordStatus | null>(null);
+ const [stats, setStats] = React.useState<LlmRecordStats | null>(null);
const [total, setTotal] = React.useState(0);
const [page, setPage] = React.useState(1);
const [pageSize, setPageSize] = React.useState(25);
const [loading, setLoading] = React.useState(false);
const [error, setError] = React.useState("");
- const [autoRefresh, setAutoRefresh] = React.useState(false);
+ const [live, setLive] = React.useState(false);
+ const [liveOpen, setLiveOpen] = React.useState(false);
+ const [pending, setPending] = React.useState(0);
const [outcome, setOutcome] = React.useState("");
+ const [windowHours, setWindowHours] = React.useState("24");
const [modelDraft, setModelDraft] = React.useState("");
const [clientIpDraft, setClientIpDraft] = React.useState("");
const [filter, setFilter] =
React.useState<LlmRecordBackend.LlmRecordFilter>({});
+ const activeFilter = React.useMemo(
+ () => ({...filter, outcome: outcome, windowHours: Number(windowHours)}),
+ [filter, outcome, windowHours],
+ );
+
const load = React.useCallback(
(nextPage = page, nextPageSize = pageSize, foreground = true) => {
if (!isAdmin) {
return;
}
- // A background poll must not raise the loading state, or it would cover
- // the rows the operator is reading every few seconds.
+ // A background reload keeps the loading state down, or it would cover
the
+ // rows being read.
if (foreground) {
setLoading(true);
}
- LlmRecordBackend.getLlmRecords(nextPage, nextPageSize, {...filter,
outcome: outcome})
+ LlmRecordBackend.getLlmRecords(nextPage, nextPageSize, activeFilter)
.then(res => {
if (res.status === "ok") {
setRecords(res.data ?? []);
setTotal(res.data2 ?? 0);
setPage(nextPage);
setPageSize(nextPageSize);
+ setPending(0);
setError("");
} else {
setError(res.msg || i18next.t("general:Failed to get data"));
@@ -156,9 +225,20 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
}
});
},
- [filter, isAdmin, outcome, page, pageSize],
+ [activeFilter, isAdmin, page, pageSize],
);
+ const loadStats = React.useCallback(() => {
+ if (!isAdmin) {
+ return;
+ }
+ LlmRecordBackend.getLlmRecordStats(activeFilter).then(res => {
+ if (res.status === "ok") {
+ setStats(res.data ?? null);
+ }
+ });
+ }, [activeFilter, isAdmin]);
+
const loadStatus = React.useCallback(() => {
if (!isAdmin) {
return;
@@ -172,22 +252,78 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
React.useEffect(() => {
load(1, pageSize);
+ loadStats();
loadStatus();
// eslint-disable-next-line react-hooks/exhaustive-deps
- }, [filter, outcome]);
+ }, [activeFilter]);
+
+ // The feed carries every record Gateway writes, so the page drops the ones
+ // its own filter would not have asked for.
+ const matchesFilter = React.useCallback(
+ (record: LlmRecord) => {
+ if (activeFilter.model && !record.model.includes(activeFilter.model)) {
+ return false;
+ }
+ if (activeFilter.clientIp && record.clientIp !== activeFilter.clientIp) {
+ return false;
+ }
+ const succeeded = record.status >= 200 && record.status < 300;
+ if (outcome === "ok" && !succeeded) {
+ return false;
+ }
+ if (outcome === "error" && succeeded) {
+ return false;
+ }
+ return true;
+ },
+ [activeFilter, outcome],
+ );
React.useEffect(() => {
- if (!autoRefresh) {
+ if (!live || !isAdmin) {
+ setLiveOpen(false);
return undefined;
}
- const interval = setInterval(() => load(page, pageSize, false), 5000);
- return () => clearInterval(interval);
- }, [autoRefresh, load, page, pageSize]);
+
+ const close = LlmRecordBackend.streamLlmRecords({
+ onRecord: record => {
+ if (!matchesFilter(record)) {
+ return;
+ }
+ setTotal(current => current + 1);
+ // A page further down counts them instead of reshuffling what is
+ // being read.
+ setPage(currentPage => {
+ if (currentPage === 1) {
+ setRecords(current => [record, ...current].slice(0, pageSize));
+ } else {
+ setPending(current => current + 1);
+ }
+ return currentPage;
+ });
+ },
+ onOpen: () => setLiveOpen(true),
+ onError: () => setLiveOpen(false),
+ });
+ const interval = setInterval(loadStats, STATS_INTERVAL);
+
+ return () => {
+ close();
+ clearInterval(interval);
+ setLiveOpen(false);
+ };
+ }, [isAdmin, live, loadStats, matchesFilter, pageSize]);
if (!isAdmin) {
return <UnauthorizedResult />;
}
+ const refresh = () => {
+ load(page, pageSize);
+ loadStats();
+ loadStatus();
+ };
+
const deleteRecord = (id: number) => {
LlmRecordBackend.deleteLlmRecord(id)
.then(res => {
@@ -195,7 +331,7 @@ export default function LlmRecordsPage({account}: {account:
Account}) {
Setting.showMessage("error", `${i18next.t("general:Failed to
delete")}: ${res.msg}`);
} else {
Setting.showMessage("success", i18next.t("general:Deleted
successfully"));
- load(page, pageSize);
+ refresh();
}
})
.catch(err => Setting.showMessage("error", `${i18next.t("general:Failed
to delete")}: ${err}`));
@@ -209,6 +345,7 @@ export default function LlmRecordsPage({account}: {account:
Account}) {
} else {
Setting.showMessage("success", i18next.t("general:Deleted
successfully"));
load(1, pageSize);
+ loadStats();
loadStatus();
}
})
@@ -241,12 +378,12 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
title: i18next.t("general:Client ip"),
key: "clientIp",
dataIndex: "clientIp",
- width: "130px",
+ width: "120px",
},
{
title: i18next.t("llm:Status"),
key: "status",
- width: "130px",
+ width: "120px",
render: (_value, record) => (
<div className="flex flex-wrap items-center gap-1">
<Badge variant={statusVariant(record.status)}>{record.status ||
i18next.t("llm:No response")}</Badge>
@@ -258,35 +395,61 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
title: i18next.t("agent:Duration"),
key: "durationMs",
dataIndex: "durationMs",
- width: "110px",
+ width: "100px",
render: (value: number) => `${value.toLocaleString()} ms`,
},
{
title: i18next.t("llm:Tokens"),
key: "totalTokens",
- width: "120px",
+ width: "150px",
render: (_value, record) =>
record.totalTokens > 0 ? (
- <div className="flex min-w-0 flex-col gap-0.5">
+ <div className="flex min-w-0 flex-col gap-0.5"
title={tokenBreakdown(record)}>
<span>{record.totalTokens.toLocaleString()}</span>
<span className="text-muted-foreground truncate text-xs">
- {record.promptTokens.toLocaleString()} +
{record.completionTokens.toLocaleString()}
+ {formatTokens(record.promptTokens)} ↑ ·
{formatTokens(record.completionTokens)} ↓
+ {record.cacheReadTokens > 0 ? ` ·
${formatTokens(record.cacheReadTokens)} ⚡` : ""}
</span>
</div>
) : null,
},
+ {
+ title: i18next.t("llm:Cost"),
+ key: "cost",
+ width: "90px",
+ align: "right",
+ render: (_value, record) =>
+ record.priced ? (
+ formatCost(record.cost)
+ ) : (
+ <span className="text-muted-foreground" title={i18next.t("llm:No
price for this model")}>
+ —
+ </span>
+ ),
+ },
{
title: i18next.t("llm:Request"),
key: "summary",
render: (_value, record) => (
- <span className="text-muted-foreground block truncate text-xs"
title={record.summary || record.error}>
- {record.summary || record.error}
- </span>
+ <div className="flex min-w-0 flex-col gap-0.5">
+ <span className="truncate text-xs" title={record.summary ||
record.error}>
+ {record.summary || record.error}
+ </span>
+ {record.messageCount > 0 || record.toolCount > 0 ? (
+ <span className="text-muted-foreground truncate text-xs">
+ {record.messageCount} {i18next.t("llm:messages")} ·
{record.toolCount} {i18next.t("llm:tools")}
+ {record.systemBytes > 0
+ ? ` · ${i18next.t("llm:system")} ${(record.systemBytes /
1000).toFixed(1)}k`
+ : ""}
+ </span>
+ ) : null}
+ </div>
),
},
];
const modeOff = status !== null && status.mode === "off";
+ const modeMetadata = status !== null && status.mode === "metadata";
const description = status
? i18next
.t("llm:Mode {mode}, kept for {days} days, up to {max} records")
@@ -302,11 +465,30 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
description={description}
actions={
<>
+ <SimpleSelect
+ className="w-[150px]"
+ value={windowHours}
+ onChange={setWindowHours}
+ options={[
+ {label: i18next.t("llm:Last hour"), value: "1"},
+ {label: i18next.t("llm:Last 24 hours"), value: "24"},
+ {label: i18next.t("llm:Last 7 days"), value: "168"},
+ {label: i18next.t("llm:All time"), value: "0"},
+ ]}
+ />
<Label className="text-sm font-normal">
- <Switch checked={autoRefresh} onCheckedChange={setAutoRefresh} />
- {i18next.t("agent:Auto refresh")}
+ <Switch checked={live} onCheckedChange={setLive} />
+ <span className="flex items-center gap-1.5">
+ {i18next.t("llm:Live")}
+ {live ? (
+ <span
+ className={liveOpen ? "bg-success size-2 animate-pulse
rounded-full" : "bg-muted-foreground size-2 rounded-full"}
+ title={liveOpen ? i18next.t("llm:Feed connected") :
i18next.t("llm:Feed connecting")}
+ />
+ ) : null}
+ </span>
</Label>
- <Button variant="outline" onClick={() => load(page, pageSize)}
loading={loading}>
+ <Button variant="outline" onClick={refresh} loading={loading}>
<RefreshCw />
{i18next.t("general:Refresh")}
</Button>
@@ -321,6 +503,7 @@ export default function LlmRecordsPage({account}: {account:
Account}) {
/>
{modeOff ? <MessageAlert variant="info" title={i18next.t("llm:Recording
is off")} description={i18next.t("llm:Recording is off detail")} /> : null}
+ {modeMetadata ? <MessageAlert variant="info"
title={i18next.t("llm:Bodies are not stored")}
description={i18next.t("llm:Bodies are not stored detail")} /> : null}
{status && status.dropped > 0 ? (
<MessageAlert
variant="warning"
@@ -329,6 +512,57 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
) : null}
{error ? <MessageAlert title={error} /> : null}
+ <div className="grid gap-3 sm:grid-cols-2 xl:grid-cols-4">
+ <StatCard
+ label={i18next.t("llm:Requests")}
+ value={(stats?.requests ?? 0).toLocaleString()}
+ icon={Activity}
+ hint={
+ stats && stats.failed > 0
+ ? `${stats.failed.toLocaleString()} ${i18next.t("llm:failed")}`
+ : i18next.t("llm:all succeeded")
+ }
+ tone={stats && stats.failed > 0 ? "warning" : "default"}
+ />
+ <StatCard
+ label={i18next.t("llm:Tokens")}
+ value={formatTokens(stats?.totalTokens ?? 0)}
+ icon={Hash}
+ hint={`${formatTokens(stats?.promptTokens ?? 0)}
${i18next.t("llm:in")} · ${formatTokens(stats?.completionTokens ?? 0)}
${i18next.t("llm:out")}`}
+ />
+ <StatCard
+ label={i18next.t("llm:Cache hit rate")}
+ value={`${(stats ? cacheHitRate(stats) : 0).toFixed(0)}%`}
+ icon={Database}
+ percent={stats ? cacheHitRate(stats) : 0}
+ hint={`${formatTokens(stats?.cacheReadTokens ?? 0)}
${i18next.t("llm:read")} · ${formatTokens(stats?.cacheWriteTokens ?? 0)}
${i18next.t("llm:written")}`}
+ />
+ <StatCard
+ label={i18next.t("llm:Cost")}
+ value={formatCost(stats?.cost ?? 0)}
+ icon={CircleDollarSign}
+ hint={
+ stats && stats.unpriced > 0
+ ? i18next.t("llm:{count} records have no
price").replace("{count}", stats.unpriced.toLocaleString())
+ : i18next.t("llm:List prices, see llmPricingFile")
+ }
+ />
+ </div>
+
+ {stats && stats.models.length > 1 ? (
+ <div className="flex flex-wrap items-center gap-2">
+ {stats.models.map(model => (
+ <Badge key={model.model} variant="muted" className="gap-2">
+ <span className="font-mono">{model.model}</span>
+ <span>{model.requests.toLocaleString()}</span>
+ <span className="text-muted-foreground">
+ {formatTokens(model.tokens)} · {formatCost(model.cost)}
+ </span>
+ </Badge>
+ ))}
+ </div>
+ ) : null}
+
<form
className="flex flex-wrap items-center gap-2"
onSubmit={event => {
@@ -361,6 +595,11 @@ export default function LlmRecordsPage({account}:
{account: Account}) {
<Button type="submit" variant="outline">
{i18next.t("agent:Filter")}
</Button>
+ {pending > 0 ? (
+ <Button variant="ghost" onClick={() => load(1, pageSize)}>
+ {i18next.t("llm:{count} new records").replace("{count}",
pending.toLocaleString())}
+ </Button>
+ ) : null}
</form>
<DataTable
diff --git a/web/src/types.ts b/web/src/types.ts
index 9ec4b37..d7edd09 100644
--- a/web/src/types.ts
+++ b/web/src/types.ts
@@ -226,9 +226,19 @@ export interface LlmRecord {
durationMs: number;
attempts: number;
error: string;
+ /** Input billed as fresh: the cached part is counted separately. */
promptTokens: number;
completionTokens: number;
+ cacheReadTokens: number;
+ cacheWriteTokens: number;
+ reasoningTokens: number;
totalTokens: number;
+ /** US dollars, meaningful only where `priced` is true. */
+ cost: number;
+ priced: boolean;
+ systemBytes: number;
+ messageCount: number;
+ toolCount: number;
summary: string;
/** Only returned by getLlmRecord, the list endpoint leaves it out. */
payload: string;
@@ -245,6 +255,35 @@ export interface LlmRecordStatus {
count: number;
}
+/** US dollars per million tokens, as the record was costed. */
+export interface LlmPrice {
+ input: number;
+ output: number;
+ cacheWrite: number;
+ cacheRead: number;
+}
+
+export interface LlmModelStat {
+ model: string;
+ requests: number;
+ tokens: number;
+ cost: number;
+}
+
+export interface LlmRecordStats {
+ requests: number;
+ failed: number;
+ promptTokens: number;
+ completionTokens: number;
+ cacheReadTokens: number;
+ cacheWriteTokens: number;
+ totalTokens: number;
+ cost: number;
+ /** Records whose model has no price entry. */
+ unpriced: number;
+ models: LlmModelStat[];
+}
+
export interface MetricPoint {
data: string;
count: number;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
