This is an automated email from the ASF dual-hosted git repository.
AlexStocks pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/dubbo-go-pixiu.git
The following commit(s) were added to refs/heads/develop by this push:
new 331735d3 perf(llm): drop per-request hex alloc in cooldown credential
hash (#965)
331735d3 is described below
commit 331735d3cfd1dd2170ad53f5c11f845fc34035e6
Author: 承潜 <[email protected]>
AuthorDate: Sun Jun 14 13:53:40 2026 +0800
perf(llm): drop per-request hex alloc in cooldown credential hash (#965)
* perf(llm): drop per-request hex alloc in cooldown credential hash
Every LLM request resolves a cooldownKey at least once (more on the
retry/fallback path) via endpointCredentialHash, which ran
fmt.Sprintf("%x", sum) and allocated a fresh 64-byte hex string that
escaped to the heap. The hex string is pure overhead: credentialHash is
only ever used as a comparable component of the cooldownKey map key.
Use the raw [32]byte sha256 output directly as the key component. A
[32]byte array is comparable and valid in a struct map key, so the hex
encoding and its heap allocation are removed while the cooldown identity
semantics stay byte-for-byte identical. sha256 itself stays per-call
(nanoseconds for short keys); the allocation was the dominant cost.
BenchmarkCooldown_EndpointInCooldown (100 endpoints):
before 300.0 ns/op 168 B/op 6 allocs/op
after 163.2 ns/op 40 B/op 3 allocs/op
Closes #956
* test(llm): pin cooldown benchmark TTL to keep hot path stable
The benchmark relied on the endpoint's 60s HealthCheckInterval as the
cooldown TTL. A long -benchtime (or a slow machine) could let entries
expire mid-run, shifting measurement from the intended in-cooldown path
onto the delete+log path and destabilizing results.
Set HealthCheckInterval to 24h so every iteration stays on the hot path
regardless of -benchtime.
---
pkg/filter/llm/proxy/filter.go | 9 ++++-----
pkg/filter/llm/proxy/filter_test.go | 28 ++++++++++++++++++++++++++++
2 files changed, 32 insertions(+), 5 deletions(-)
diff --git a/pkg/filter/llm/proxy/filter.go b/pkg/filter/llm/proxy/filter.go
index c5495e31..394beac1 100644
--- a/pkg/filter/llm/proxy/filter.go
+++ b/pkg/filter/llm/proxy/filter.go
@@ -118,7 +118,7 @@ type (
clusterName string
endpointID string
endpointAddress string
- credentialHash string
+ credentialHash [32]byte
}
cooldownStore struct {
@@ -582,12 +582,11 @@ func newCooldownKey(clusterName string, endpoint
*model.Endpoint) cooldownKey {
}
}
-func endpointCredentialHash(endpoint *model.Endpoint) string {
+func endpointCredentialHash(endpoint *model.Endpoint) [32]byte {
if endpoint == nil || endpoint.LLMMeta == nil ||
endpoint.LLMMeta.APIKey == "" {
- return ""
+ return [32]byte{}
}
- sum := sha256.Sum256([]byte(endpoint.LLMMeta.APIKey))
- return fmt.Sprintf("%x", sum)
+ return sha256.Sum256([]byte(endpoint.LLMMeta.APIKey))
}
// endpointCooldownInterval returns the per-endpoint cooldown TTL derived from
diff --git a/pkg/filter/llm/proxy/filter_test.go
b/pkg/filter/llm/proxy/filter_test.go
index 65db0bd7..05087914 100644
--- a/pkg/filter/llm/proxy/filter_test.go
+++ b/pkg/filter/llm/proxy/filter_test.go
@@ -412,3 +412,31 @@ func testLLMEndpoint(id string, port int) *model.Endpoint {
},
}
}
+
+func BenchmarkCooldown_EndpointInCooldown(b *testing.B) {
+ const clusterName = "llm-cooldown-bench"
+ const endpointCount = 100
+ // Keep the cooldown TTL far longer than any -benchtime run so every
+ // iteration stays on the intended in-cooldown hot path. Otherwise
entries
+ // could expire mid-benchmark and shift measurement onto the delete+log
path.
+ const cooldownTTLMillis = int64(24 * time.Hour / time.Millisecond)
+ store := newCooldownStore()
+ executor := &RequestExecutor{
+ clusterName: clusterName,
+ cooldowns: store,
+ }
+ endpoints := make([]*model.Endpoint, endpointCount)
+ for i := range endpoints {
+ endpoint := testLLMEndpoint(fmt.Sprintf("ep-%d", i), 19000+i)
+ endpoint.LLMMeta.APIKey = fmt.Sprintf("api-key-%d", i)
+ endpoint.LLMMeta.HealthCheckInterval = cooldownTTLMillis
+ endpoints[i] = endpoint
+ store.markFailure(clusterName, endpoint, time.Now())
+ }
+
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ executor.endpointInCooldown(endpoints[i%endpointCount])
+ }
+}
