Copilot commented on code in PR #1618: URL: https://github.com/apache/dubbo-admin/pull/1618#discussion_r4438210835
########## ai/component/hooks/reasoning.go: ########## @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hooks + +import ( + "encoding/json" + "fmt" + "strings" +) + +// reasoningPartType is the part discriminator the generative AI semantic +// conventions define for reasoning/thinking content received from a model. +const reasoningPartType = "reasoning" + +// reasoningPartsKey is the message field that carries the parts of a chat +// message in the generative AI semantic conventions. +const reasoningPartsKey = "parts" + +// truncatedReasoningLimit is the number of characters kept from a single +// reasoning string when the truncated policy applies. It matches the limit +// used for the other captured JSON strings. +const truncatedReasoningLimit = 512 + +// ApplyReasoningPolicy filters or truncates the reasoning parts of a +// generative AI messages document according to the configured policy. +// +// The document is the serialized JSON of the gen_ai.input.messages or +// gen_ai.output.messages attribute. A reasoning part is an element of a +// "parts" array whose "type" is "reasoning"; every other part is preserved. +// +// - CaptureNone removes the reasoning parts from the document. +// - CaptureTruncated keeps them and shortens their content. +// - CaptureFull returns the document unchanged. +// +// An unset or unknown policy is treated like CaptureNone so reasoning content +// never reaches a trace by accident. A document that is not valid JSON is +// returned unchanged: a parsing failure must never drop captured content. +func ApplyReasoningPolicy(document string, policy string) string { + switch policy { + case CaptureFull: + return document + case CaptureTruncated: + return rewriteReasoningParts(document, truncateReasoningElement) + default: + return rewriteReasoningParts(document, nil) + } +} + +// normalizeCaptureReasoning resolves the reasoning capture policy of a hook +// registration. An unset policy defaults to CaptureNone so reasoning content +// is never captured unless a hook asks for it. +func normalizeCaptureReasoning(policy string) (string, error) { + switch policy { + case "": + return CaptureNone, nil + case CaptureNone, CaptureTruncated, CaptureFull: + return policy, nil + default: + return "", fmt.Errorf("capture_reasoning must be one of %q, %q, or %q", CaptureNone, CaptureTruncated, CaptureFull) + } +} + +// rewriteReasoningParts rewrites the reasoning parts of a messages document +// and returns the document unchanged when it holds no reasoning part. The +// traversal only relies on the "parts" array and the "type" discriminator so +// it does not depend on the remaining message fields. +func rewriteReasoningParts(document string, rewrite func(element map[string]any) bool) string { + if !strings.Contains(document, reasoningPartType) { + return document + } + decoder := json.NewDecoder(strings.NewReader(document)) Review Comment: [P1] This string check bypasses the policy for valid JSON that escapes the type, for example `"type":"\u0072easoning"`. With `CaptureNone`, the unfiltered reasoning content is returned unchanged. Decode the JSON before deciding whether it contains reasoning parts so the policy cannot be bypassed by JSON escapes. ########## ai/component/hooks/reasoning_test.go: ########## @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hooks + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + + "dubbo-admin-ai/config" + "dubbo-admin-ai/runtime" + + "go.opentelemetry.io/otel/attribute" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" +) + +const reasoningDocument = `[{"role":"assistant","parts":[{"type":"reasoning","content":"thinking about the answer"},{"type":"text","content":"the answer"}]}]` + +type reasoningMessage struct { + Role string `json:"role"` + Parts []map[string]any `json:"parts"` +} + +func decodeReasoningDocument(t *testing.T, document string) []reasoningMessage { + t.Helper() + var messages []reasoningMessage + if err := json.Unmarshal([]byte(document), &messages); err != nil { + t.Fatalf("json.Unmarshal(%q) error = %v", document, err) + } + return messages +} + +func TestApplyReasoningPolicyRemovesReasoningParts(t *testing.T) { + filtered := ApplyReasoningPolicy(reasoningDocument, CaptureNone) + messages := decodeReasoningDocument(t, filtered) + if len(messages) != 1 || len(messages[0].Parts) != 1 { + t.Fatalf("filtered document = %s, want a single text part", filtered) + } + part := messages[0].Parts[0] + if part["type"] != "text" || part["content"] != "the answer" { + t.Fatalf("remaining part = %v, want the text part", part) + } + if strings.Contains(filtered, reasoningPartType) { + t.Fatalf("filtered document still holds a reasoning part: %s", filtered) + } +} + +func TestApplyReasoningPolicyTruncatesReasoningContent(t *testing.T) { + long := strings.Repeat("思考", 1000) + document := `[{"role":"assistant","parts":[{"type":"reasoning","content":"` + long + `"},{"type":"text","content":"the answer"}]}]` + filtered := ApplyReasoningPolicy(document, CaptureTruncated) + messages := decodeReasoningDocument(t, filtered) + if len(messages) != 1 || len(messages[0].Parts) != 2 { + t.Fatalf("truncated document = %s, want the reasoning and the text part", filtered) + } + reasoning := messages[0].Parts[0] + if reasoning["type"] != reasoningPartType { + t.Fatalf("first part = %v, want a reasoning part", reasoning) + } + content, ok := reasoning["content"].(string) + if !ok { + t.Fatalf("reasoning content = %v, want a string", reasoning["content"]) + } + if len([]rune(content)) > truncatedReasoningLimit { + t.Fatalf("reasoning content length = %d, want at most %d", len([]rune(content)), truncatedReasoningLimit) + } + if len([]rune(content)) != truncatedReasoningLimit { + t.Fatalf("reasoning content length = %d, want %d", len([]rune(content)), truncatedReasoningLimit) + } + if !strings.HasSuffix(content, "…") { + t.Fatalf("truncated reasoning content = %q, want an ellipsis marker", content) + } + if messages[0].Parts[1]["type"] != "text" { + t.Fatalf("second part = %v, want the text part", messages[0].Parts[1]) + } +} + +func TestApplyReasoningPolicyKeepsFullReasoning(t *testing.T) { + if got := ApplyReasoningPolicy(reasoningDocument, CaptureFull); got != reasoningDocument { + t.Fatalf("document = %s, want it unchanged", got) + } +} + +func TestApplyReasoningPolicyKeepsInvalidDocument(t *testing.T) { + for _, document := range []string{"", "not json", "[{", `{"role":"user"} trailing`} { + if got := ApplyReasoningPolicy(document, CaptureNone); got != document { + t.Fatalf("document = %q, want it unchanged", got) + } + } + // A document without a reasoning part keeps its original representation. + document := `[{"role":"user","parts":[{"type":"text","content":"hi"}]}]` + if got := ApplyReasoningPolicy(document, CaptureNone); got != document { + t.Fatalf("document = %s, want it unchanged", got) + } +} + +func TestApplyReasoningPolicyIgnoresDocumentShape(t *testing.T) { + document := `[{"role":"user","parts":[{"type":"text","content":"hi"}]},{"role":"assistant","parts":[{"type":"reasoning","content":"thinking"}]}]` + filtered := ApplyReasoningPolicy(document, CaptureNone) + messages := decodeReasoningDocument(t, filtered) + if len(messages) != 2 { + t.Fatalf("filtered document = %s, want both messages", filtered) + } + if len(messages[1].Parts) != 0 { + t.Fatalf("second message = %v, want no parts", messages[1]) + } +} + +func TestCaptureReasoningConfiguration(t *testing.T) { + t.Run("default", func(t *testing.T) { + c, err := componentFromHookYAML(t, "hooks: [{name: trace, type: tracing}]") + if err != nil { + t.Fatal(err) + } + entries, err := c.spec.normalizedHooks() + if err != nil { + t.Fatal(err) + } + if entries[0].tracing.CaptureReasoning != CaptureNone { + t.Fatalf("capture_reasoning = %q, want %q", entries[0].tracing.CaptureReasoning, CaptureNone) Review Comment: These tests reference hook APIs that do not exist in this branch: TracingSpec and Registration have no CaptureReasoning field, and Manager has no ReasoningPolicy method. Consequently, the ai hooks test package fails to compile before any assertions run. Include the corresponding configuration/manager/tracing integration, or move these tests and the helper to the companion observability PR. ########## pkg/mcp/tools/trace/jaeger.go: ########## @@ -184,6 +188,218 @@ func (p *jaegerProvider) GetTraceByID(ctx context.Context, traceID string) (*Tra return result, nil } +// SearchTraces discovers candidate traces through the Jaeger v1 query API. +// The summaries are computed from the raw spans so that no information is lost to +// the normalization applied by the single trace lookup. +func (p *jaegerProvider) SearchTraces(ctx context.Context, query TraceSearchQuery) (*TraceSearchResult, error) { + if strings.TrimSpace(query.Service) == "" { + return nil, fmt.Errorf("service is required for trace search") + } + values := url.Values{} + values.Set("service", query.Service) + if query.Operation != "" { + values.Set("operation", query.Operation) + } + if query.TraceID != "" { + values.Set("traceID", query.TraceID) + } + if len(query.Tags) > 0 { + encoded, err := json.Marshal(query.Tags) + if err != nil { + return nil, fmt.Errorf("failed to encode trace search tags: %w", err) + } + values.Set("tags", string(encoded)) + } + if !query.StartTime.IsZero() { + values.Set("start", strconv.FormatInt(query.StartTime.UnixMicro(), 10)) + } + if !query.EndTime.IsZero() { + values.Set("end", strconv.FormatInt(query.EndTime.UnixMicro(), 10)) + } + if query.MinDuration > 0 { + values.Set("minDuration", strconv.FormatInt(query.MinDuration.Microseconds(), 10)) + } Review Comment: Jaeger's `/api/traces` query parser reads `minDuration` with `time.ParseDuration`, so the value needs a unit (for example `250ms` or `1.2s`). A number on its own like `250000` fails to parse (`time: missing unit in duration "250000"`), and Jaeger rejects the request with a 400. Because of this, every `search_traces` call that sets `minDurationMs` fails against a real Jaeger. The httptest stub doesn't validate the parameter, so the tests don't catch it. This fix is required. Send the duration with a unit. The assertion in `search_test.go` (line 92) also needs to change from `"250000"` to `"250ms"`. ########## pkg/mcp/tools/trace/search_test.go: ########## @@ -0,0 +1,440 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trace + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + observabilitycfg "github.com/apache/dubbo-admin/pkg/config/observability" +) + +const testSearchTraceID = "aabbccddeeff00112233445566778899" + +func newSearchProvider(t *testing.T, endpoint string) *jaegerProvider { + t.Helper() + provider, err := newJaegerProvider(observabilitycfg.TraceProviderConfig{ + Name: "jaeger-main", Type: observabilitycfg.TraceProviderJaeger, Endpoint: endpoint, BearerToken: "secret", Tenant: "tenant-a", + }) + if err != nil { + t.Fatal(err) + } + return provider +} + +func TestJaegerProviderSearchTracesBuildsRequest(t *testing.T) { + var captured *http.Request + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + captured = r + _, _ = w.Write([]byte(`{"data":[{"traceID":"` + testSearchTraceID + `","spans":[{"traceID":"` + testSearchTraceID + `","spanID":"01","operationName":"root","references":[],"startTime":2000000,"duration":3000,"tags":[],"logs":[],"processID":"p1"}],"processes":{"p1":{"serviceName":"shop-order","tags":[]}}}]}`)) + })) + defer server.Close() + + provider := newSearchProvider(t, server.URL) + start := time.Unix(1700000000, 0).UTC() + end := time.Unix(1700003600, 0).UTC() + result, err := provider.SearchTraces(context.Background(), TraceSearchQuery{ + Service: "shop-order", + Operation: "checkout", + TraceID: testSearchTraceID, + Tags: map[string]string{"rpc.system": "apache_dubbo"}, + StartTime: start, + EndTime: end, + Limit: 17, + MinDuration: 250 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + if captured == nil { + t.Fatal("expected the provider to issue a request") + } + if captured.URL.Path != "/api/traces" { + t.Fatalf("unexpected path %s", captured.URL.Path) + } + if captured.Header.Get("Authorization") != "Bearer secret" || captured.Header.Get(tenantHeader) != "tenant-a" { + t.Fatalf("missing auth headers: %#v", captured.Header) + } + query := captured.URL.Query() + if query.Get("service") != "shop-order" || query.Get("operation") != "checkout" { + t.Fatalf("unexpected service filter: %v", query) + } + if query.Get("traceID") != testSearchTraceID { + t.Fatalf("unexpected trace id filter: %v", query) + } + if query.Get("tags") != `{"rpc.system":"apache_dubbo"}` { + t.Fatalf("unexpected tags filter: %v", query) + } + if query.Get("start") != "1700000000000000" || query.Get("end") != "1700003600000000" { + t.Fatalf("unexpected time window: %v", query) + } + if query.Get("minDuration") != "250000" { + t.Fatalf("unexpected min duration: %v", query) + } Review Comment: This assertion pins the unitless microsecond form `250000`, which the real Jaeger v1 API rejects (it expects a Go duration string such as `250ms`). The test should assert the duration-string form once the provider is fixed, otherwise it locks in the incorrect wire format. ########## pkg/mcp/tools/trace/analyzer.go: ########## @@ -0,0 +1,821 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trace + +import ( + "context" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + consolectx "github.com/apache/dubbo-admin/pkg/console/context" + "github.com/apache/dubbo-admin/pkg/mcp/common" +) + +const ( + // Bounded output limits. They keep the tool payload small enough to stay inside + // the model context window while still describing the most relevant spans. + maxCriticalPath = 32 + maxServiceBreakdown = 50 + maxSlowSpans = 10 + maxErrorSpans = 20 + maxDependencyEdges = 50 + maxGenAICalls = 20 + + maxAnalysisSummaryChars = 300 + maxErrorTypeChars = 96 + maxErrorMessageChars = 256 +) + +const ( + errorTypeAttribute = "error.type" + errorMessageAttribute = "error.message" + exceptionTypeAttribute = "exception.type" + exceptionMessageAttribute = "exception.message" + spanEventNameAttribute = "event" + standardExceptionEvent = "exception" + genAIExceptionEvent = "gen_ai.client.operation.exception" +) + +// analysisSpan is the analyzer's internal view of a span with a resolved timeline. +type analysisSpan struct { + span *Span + position int + startMicros int64 + endMicros int64 + timeValid bool + selfMicros int64 + waitMicros int64 + parent *analysisSpan + children []*analysisSpan +} + +// timelineInterval is a half-open interval on the absolute microsecond timeline. +type timelineInterval struct { + start int64 + end int64 +} + +// AnalyzeTrace diagnoses a single distributed trace and returns a structured report. +func AnalyzeTrace(ctx consolectx.Context, args map[string]any) (*common.ToolResult, error) { + helper := common.NewArgsHelper(args) + traceID := helper.GetString("traceId", "") + if !traceIDPattern.MatchString(traceID) { + return common.ErrorResult(fmt.Errorf("traceId must be a 16 or 32 character hexadecimal value")), nil + } + providerConfig, err := providerConfigFromContext(ctx, helper.GetString("provider", "")) + if err != nil { + return common.ErrorResult(err), nil + } + provider, err := newProvider(providerConfig) + if err != nil { + return common.ErrorResult(err), nil + } + trace, err := provider.GetTraceByID(traceRequestContext(ctx), traceID) + if err != nil { + return common.ErrorResult(err), nil + } + return common.JsonResult(BuildTraceAnalysis(trace)) +} + +// AnalyzeProperties returns the MCP input schema properties for trace diagnosis. +func AnalyzeProperties() map[string]common.PropertyDef { + return map[string]common.PropertyDef{ + "traceId": {Type: "string", Description: "Jaeger 64-bit or 128-bit hexadecimal TraceID"}, + "provider": {Type: "string", Description: "Optional trace provider name; defaults to defaultProvider"}, + } +} + +// BuildTraceAnalysis turns a normalized trace into a structured diagnosis. +// It is a pure function so the diagnosis rules can be verified without a backend. +func BuildTraceAnalysis(trace *Trace) *TraceAnalysis { + analysis := &TraceAnalysis{} + if trace == nil { + analysis.Summary = "no trace data available" + return analysis + } + analysis.TraceID = trace.TraceID + analysis.Truncated = trace.Truncated + + nodes := buildAnalysisTree(trace.Spans) + computeSelfTimes(nodes, analysis) + criticalPath, criticalPathSet := buildCriticalPath(nodes, analysis) + analysis.CriticalPath = criticalPath + + wallClock, traceStart := analysisEnvelope(nodes, trace) + analysis.DurationMicros = wallClock + analysis.StartTime = traceStart Review Comment: Preserve the provider's full trace envelope when the span list is clipped. normalizeJaegerTrace calculates StartTime and DurationMicros over all raw spans before keeping the first 500, but analysisEnvelope replaces that duration with the envelope of only the retained spans. If later spans extend the trace beyond those retained, analyze_trace reports a shorter duration than search_traces/get_trace_by_id and inflates every latency percentage despite already having the complete duration. Use the provider envelope for the wall-clock denominator in this case. ########## pkg/mcp/tools/trace/analyzer.go: ########## @@ -0,0 +1,821 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trace + +import ( + "context" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + consolectx "github.com/apache/dubbo-admin/pkg/console/context" + "github.com/apache/dubbo-admin/pkg/mcp/common" +) + +const ( + // Bounded output limits. They keep the tool payload small enough to stay inside + // the model context window while still describing the most relevant spans. + maxCriticalPath = 32 + maxServiceBreakdown = 50 + maxSlowSpans = 10 + maxErrorSpans = 20 + maxDependencyEdges = 50 + maxGenAICalls = 20 + + maxAnalysisSummaryChars = 300 + maxErrorTypeChars = 96 + maxErrorMessageChars = 256 +) + +const ( + errorTypeAttribute = "error.type" + errorMessageAttribute = "error.message" + exceptionTypeAttribute = "exception.type" + exceptionMessageAttribute = "exception.message" + spanEventNameAttribute = "event" + standardExceptionEvent = "exception" + genAIExceptionEvent = "gen_ai.client.operation.exception" +) + +// analysisSpan is the analyzer's internal view of a span with a resolved timeline. +type analysisSpan struct { + span *Span + position int + startMicros int64 + endMicros int64 + timeValid bool + selfMicros int64 + waitMicros int64 + parent *analysisSpan + children []*analysisSpan +} + +// timelineInterval is a half-open interval on the absolute microsecond timeline. +type timelineInterval struct { + start int64 + end int64 +} + +// AnalyzeTrace diagnoses a single distributed trace and returns a structured report. +func AnalyzeTrace(ctx consolectx.Context, args map[string]any) (*common.ToolResult, error) { + helper := common.NewArgsHelper(args) + traceID := helper.GetString("traceId", "") + if !traceIDPattern.MatchString(traceID) { + return common.ErrorResult(fmt.Errorf("traceId must be a 16 or 32 character hexadecimal value")), nil + } + providerConfig, err := providerConfigFromContext(ctx, helper.GetString("provider", "")) + if err != nil { + return common.ErrorResult(err), nil + } + provider, err := newProvider(providerConfig) + if err != nil { + return common.ErrorResult(err), nil + } + trace, err := provider.GetTraceByID(traceRequestContext(ctx), traceID) + if err != nil { + return common.ErrorResult(err), nil + } + return common.JsonResult(BuildTraceAnalysis(trace)) +} + +// AnalyzeProperties returns the MCP input schema properties for trace diagnosis. +func AnalyzeProperties() map[string]common.PropertyDef { + return map[string]common.PropertyDef{ + "traceId": {Type: "string", Description: "Jaeger 64-bit or 128-bit hexadecimal TraceID"}, + "provider": {Type: "string", Description: "Optional trace provider name; defaults to defaultProvider"}, + } +} + +// BuildTraceAnalysis turns a normalized trace into a structured diagnosis. +// It is a pure function so the diagnosis rules can be verified without a backend. +func BuildTraceAnalysis(trace *Trace) *TraceAnalysis { + analysis := &TraceAnalysis{} + if trace == nil { + analysis.Summary = "no trace data available" + return analysis + } + analysis.TraceID = trace.TraceID + analysis.Truncated = trace.Truncated + + nodes := buildAnalysisTree(trace.Spans) + computeSelfTimes(nodes, analysis) + criticalPath, criticalPathSet := buildCriticalPath(nodes, analysis) + analysis.CriticalPath = criticalPath + + wallClock, traceStart := analysisEnvelope(nodes, trace) + analysis.DurationMicros = wallClock + analysis.StartTime = traceStart + analysis.SpanCount = maxInt(trace.SpanCount, len(trace.Spans)) + + breakdown := buildServiceBreakdown(nodes, wallClock, analysis) + analysis.ServiceBreakdown = breakdown + analysis.ServiceCount = maxInt(trace.ServiceCount, len(breakdown)) + + slowSpans := buildSlowSpans(nodes, wallClock, criticalPathSet, analysis) + analysis.SlowSpans = make([]SlowSpan, 0, len(slowSpans)) + for _, entry := range slowSpans { + analysis.SlowSpans = append(analysis.SlowSpans, entry.slow) + } + + errorSpans, errorCount := buildErrorSpans(nodes, analysis) + analysis.ErrorSpans = errorSpans + // The provider already counted errors over the complete trace; that count must not + // be hidden by the bounded error span list. + analysis.ErrorSpanCount = maxInt(trace.ErrorSpanCount, errorCount) + + analysis.Dependencies = buildDependencyEdges(nodes, analysis) + + if genai := BuildGenAISummary(trace.Spans); genai.Present { + analysis.GenAI = genai + } + + analysis.Summary = buildAnalysisSummary(analysis, slowSpans) + return analysis +} + +// buildAnalysisTree links the spans into a parent/child forest. A span whose parent +// is missing from the trace degrades into an orphan root: it still contributes to the +// aggregate statistics but it cannot extend a parent chain. +func buildAnalysisTree(spans []Span) []*analysisSpan { + nodes := make([]*analysisSpan, 0, len(spans)) + index := make(map[string]*analysisSpan, len(spans)) + for i := range spans { + span := &spans[i] + node := &analysisSpan{span: span, position: i} + if parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(span.StartTime)); err == nil { + node.timeValid = true + node.startMicros = parsed.UnixMicro() + node.endMicros = node.startMicros + span.DurationMicros + } else { + node.endMicros = span.DurationMicros + } + nodes = append(nodes, node) + if span.SpanID != "" { + index[span.SpanID] = node + } + } + for _, node := range nodes { + parentID := node.span.ParentSpanID + if parentID == "" { + continue + } + parent, ok := index[parentID] + if !ok || parent == node { + continue + } + node.parent = parent + parent.children = append(parent.children, node) + } + return nodes +} + +// computeSelfTimes derives the exclusive time of every span. The covered time is the +// union of the clipped child intervals, so concurrent children are not counted twice. +func computeSelfTimes(nodes []*analysisSpan, analysis *TraceAnalysis) { + for _, node := range nodes { + if !node.timeValid { + // Without a trustworthy start time the span degrades to a leaf. It + // cannot be placed on the timeline, so it must not claim any time + // of its own: counting its full duration here would let the parent + // account for the same time a second time. + node.selfMicros = 0 + node.waitMicros = 0 + analysis.Truncated = true + continue + } + intervals := make([]timelineInterval, 0, len(node.children)) + for _, child := range node.children { + if !child.timeValid { + analysis.Truncated = true + continue + } + start := maxInt64(child.startMicros, node.startMicros) + end := minInt64(child.endMicros, node.endMicros) + if end <= start { + continue + } + intervals = append(intervals, timelineInterval{start: start, end: end}) + } + covered := unionLength(intervals) + node.waitMicros = covered + node.selfMicros = node.span.DurationMicros - covered + if node.selfMicros < 0 { + node.selfMicros = 0 + } + } +} + +// unionLength returns the length of the union of the given intervals. +func unionLength(intervals []timelineInterval) int64 { + if len(intervals) == 0 { + return 0 + } + sort.Slice(intervals, func(i, j int) bool { + if intervals[i].start != intervals[j].start { + return intervals[i].start < intervals[j].start + } + return intervals[i].end < intervals[j].end + }) + total := int64(0) + current := intervals[0] + for _, next := range intervals[1:] { + if next.start > current.end { + total += current.end - current.start + current = next + continue + } + if next.end > current.end { + current.end = next.end + } + } + return total + current.end - current.start +} + +// buildCriticalPath walks the latency determining root-to-leaf chain. +// The second return value holds the analyzer internal positions of the spans on the +// path. The position is used instead of the business span id because a span id can +// be empty or duplicated, which would make two different spans overwrite each other. +func buildCriticalPath(nodes []*analysisSpan, analysis *TraceAnalysis) ([]CriticalPathEntry, map[int]struct{}) { + roots := make([]*analysisSpan, 0, len(nodes)) + for _, node := range nodes { + if node.parent == nil { + roots = append(roots, node) + } + } + entries := make([]CriticalPathEntry, 0, maxCriticalPath) + onPath := map[int]struct{}{} + visited := map[int]struct{}{} + current := selectCriticalRoot(roots) + for current != nil { + if len(entries) >= maxCriticalPath { + // The budget is exhausted while the chain still continues, so the + // remaining hops are dropped and the analysis is incomplete. + analysis.Truncated = true + break + } + if _, seen := visited[current.position]; seen { + break + } + visited[current.position] = struct{}{} + onPath[current.position] = struct{}{} + entries = append(entries, CriticalPathEntry{ + SpanID: current.span.SpanID, + ServiceName: current.span.ServiceName, + OperationName: current.span.OperationName, + StartTime: current.span.StartTime, + DurationMicros: current.span.DurationMicros, + SelfMicros: current.selfMicros, + WaitMicros: current.waitMicros, + }) + current = selectCriticalChild(current) + } + return entries, onPath +} + +// selectCriticalRoot picks the root that finishes last, because that root determines +// the end of the request. +func selectCriticalRoot(roots []*analysisSpan) *analysisSpan { + var best *analysisSpan + for _, root := range roots { + if !root.timeValid { + continue + } + if best == nil || betterCriticalRoot(root, best) { + best = root + } + } + if best != nil { + return best + } + for _, root := range roots { + if best == nil || betterCriticalRoot(root, best) { + best = root + } + } + return best +} + +func betterCriticalRoot(candidate, current *analysisSpan) bool { + if candidate.endMicros != current.endMicros { + return candidate.endMicros > current.endMicros + } + if candidate.span.DurationMicros != current.span.DurationMicros { + return candidate.span.DurationMicros > current.span.DurationMicros + } + // The start time is derived from the end time and the duration, so it is + // already equal here and cannot be used as a tie breaker. + return candidate.span.SpanID < current.span.SpanID +} + +// selectCriticalChild picks the child that finishes last, because it is the activity +// that determined when the parent completed. +func selectCriticalChild(node *analysisSpan) *analysisSpan { + var best *analysisSpan + for _, child := range node.children { + if !child.timeValid { + // A span without a trustworthy timestamp degrades to a leaf and is not + // allowed to extend the chain. + continue + } + if best == nil || betterCriticalChild(child, best) { + best = child + } + } + return best +} + +func betterCriticalChild(candidate, current *analysisSpan) bool { + if candidate.endMicros != current.endMicros { + return candidate.endMicros > current.endMicros + } + if candidate.span.DurationMicros != current.span.DurationMicros { + return candidate.span.DurationMicros > current.span.DurationMicros + } + return candidate.span.SpanID < current.span.SpanID +} + +// analysisEnvelope returns the wall clock duration and the start timestamp of the trace. +func analysisEnvelope(nodes []*analysisSpan, trace *Trace) (int64, string) { + var minStart, maxEnd int64 + hasStart := false + startText := trace.StartTime + for _, node := range nodes { + if !node.timeValid { + continue + } + if !hasStart || node.startMicros < minStart { + minStart = node.startMicros + startText = node.span.StartTime + hasStart = true + } + if node.endMicros > maxEnd { + maxEnd = node.endMicros + } + } + if !hasStart { + return trace.DurationMicros, trace.StartTime + } + return maxEnd - minStart, startText +} + +// buildServiceBreakdown aggregates the spans per service. +func buildServiceBreakdown(nodes []*analysisSpan, wallClock int64, analysis *TraceAnalysis) []ServiceLatency { + aggregates := map[string]*ServiceLatency{} + for _, node := range nodes { + span := node.span + entry, ok := aggregates[span.ServiceName] + if !ok { + entry = &ServiceLatency{ServiceName: span.ServiceName} + aggregates[span.ServiceName] = entry + } + entry.SpanCount++ + if spanFailed(span) { + entry.ErrorSpanCount++ + } + entry.TotalMicros += span.DurationMicros + if span.DurationMicros > entry.MaxMicros { + entry.MaxMicros = span.DurationMicros + } + } + result := make([]ServiceLatency, 0, len(aggregates)) + for _, entry := range aggregates { + if entry.SpanCount > 0 { + entry.AvgMicros = entry.TotalMicros / int64(entry.SpanCount) + } + // The share is the service workload relative to the wall clock of the trace. + // Concurrent spans can push it above 100 percent on purpose. + entry.SharePercent = percentOf(entry.TotalMicros, wallClock) + result = append(result, *entry) + } + sort.SliceStable(result, func(i, j int) bool { + if result[i].TotalMicros != result[j].TotalMicros { + return result[i].TotalMicros > result[j].TotalMicros + } + return result[i].ServiceName < result[j].ServiceName + }) + if len(result) > maxServiceBreakdown { + result = result[:maxServiceBreakdown] + analysis.Truncated = true + } + return result +} + +// slowSpanEntry pairs a slow span with the analyzer internal identity of the span +// it was derived from. The position is used instead of the business span id so +// that spans without an id, or with duplicated ids, cannot overwrite each other. +type slowSpanEntry struct { + slow SlowSpan + position int + selfMicros int64 +} + +// buildSlowSpans ranks the spans by the time they consumed on their own. +// The exclusive time is returned alongside the report so the summary can quote it +// even for spans that are not part of the critical path. +func buildSlowSpans(nodes []*analysisSpan, wallClock int64, criticalPath map[int]struct{}, analysis *TraceAnalysis) []slowSpanEntry { + candidates := make([]slowSpanEntry, 0, len(nodes)) + for _, node := range nodes { + span := node.span + _, onPath := criticalPath[node.position] + candidates = append(candidates, slowSpanEntry{ + slow: SlowSpan{ + SpanID: span.SpanID, + ServiceName: span.ServiceName, + OperationName: span.OperationName, + DurationMicros: span.DurationMicros, + SharePercent: percentOf(node.selfMicros, wallClock), + OnCriticalPath: onPath, + }, + position: node.position, + selfMicros: node.selfMicros, + }) + } + sort.SliceStable(candidates, func(i, j int) bool { + if candidates[i].selfMicros != candidates[j].selfMicros { + return candidates[i].selfMicros > candidates[j].selfMicros + } + return candidates[i].position < candidates[j].position + }) + if len(candidates) > maxSlowSpans { + candidates = candidates[:maxSlowSpans] + analysis.Truncated = true + } Review Comment: `slowSpans` is a top-N ranking by design: it keeps the 10 spans with the most self time. Cutting the candidate list to 10 doesn't lose any analysis data, but this code still sets `analysis.Truncated = true`, so every trace with more than 10 spans is reported as truncated. That makes the flag meaningless. The design doc (§9) and the PR description both say `truncated` should tell the agent the analysis is based on incomplete data and that it must say so in its conclusion. Today the agent would add that caveat to almost every real trace. The critical-path code already avoids this kind of false positive (see `TestBuildCriticalPathStopsAtBudgetWithoutTruncation`). Apply the same rule here: only set the flag when information is actually lost. ########## pkg/mcp/DEVELOPMENT.md: ########## @@ -436,3 +438,5 @@ observability: ``` 诊断工具不会记录完整 PromQL、Bearer Token 或完整 trace attributes。工具只读,不提供流量、配置或部署变更能力。 + +`search_traces` 必须提供 `service` 锚点,默认请求 20 条、最多 100 条,本地最多保留 50 条摘要,超出时 `truncated` 为 `true`。`analyze_trace` 最多输出 32 跳关键路径、10 个慢 span、50 个错误 span、50 个 GenAI 明细,并给出一句话结论。分析器只读取 `gen_ai.*` 元数据属性,不回显 prompt、completion 与工具入参等 Opt-In 内容。 Review Comment: The documented response limits disagree with the implementation: maxSearchTraceSummaries is 100, maxErrorSpans is 20, and maxGenAICalls is 20 per category, not 50 for each of these lists. Please align these limits so consumers can correctly interpret bounded and truncated results. ########## pkg/mcp/tools/trace/analyzer.go: ########## @@ -0,0 +1,821 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trace + +import ( + "context" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + consolectx "github.com/apache/dubbo-admin/pkg/console/context" + "github.com/apache/dubbo-admin/pkg/mcp/common" +) + +const ( + // Bounded output limits. They keep the tool payload small enough to stay inside + // the model context window while still describing the most relevant spans. + maxCriticalPath = 32 + maxServiceBreakdown = 50 + maxSlowSpans = 10 + maxErrorSpans = 20 + maxDependencyEdges = 50 + maxGenAICalls = 20 + + maxAnalysisSummaryChars = 300 + maxErrorTypeChars = 96 + maxErrorMessageChars = 256 +) + +const ( + errorTypeAttribute = "error.type" + errorMessageAttribute = "error.message" + exceptionTypeAttribute = "exception.type" + exceptionMessageAttribute = "exception.message" + spanEventNameAttribute = "event" + standardExceptionEvent = "exception" + genAIExceptionEvent = "gen_ai.client.operation.exception" +) + +// analysisSpan is the analyzer's internal view of a span with a resolved timeline. +type analysisSpan struct { + span *Span + position int + startMicros int64 + endMicros int64 + timeValid bool + selfMicros int64 + waitMicros int64 + parent *analysisSpan + children []*analysisSpan +} + +// timelineInterval is a half-open interval on the absolute microsecond timeline. +type timelineInterval struct { + start int64 + end int64 +} + +// AnalyzeTrace diagnoses a single distributed trace and returns a structured report. +func AnalyzeTrace(ctx consolectx.Context, args map[string]any) (*common.ToolResult, error) { + helper := common.NewArgsHelper(args) + traceID := helper.GetString("traceId", "") + if !traceIDPattern.MatchString(traceID) { + return common.ErrorResult(fmt.Errorf("traceId must be a 16 or 32 character hexadecimal value")), nil + } + providerConfig, err := providerConfigFromContext(ctx, helper.GetString("provider", "")) + if err != nil { + return common.ErrorResult(err), nil + } + provider, err := newProvider(providerConfig) + if err != nil { + return common.ErrorResult(err), nil + } + trace, err := provider.GetTraceByID(traceRequestContext(ctx), traceID) + if err != nil { + return common.ErrorResult(err), nil + } Review Comment: [P1] The underlying lookup returns the original `*url.Error`, including the configured endpoint's query string. Since endpoint validation permits URLs such as `https://jaeger.example?api_key=...`, a transport failure exposes that credential in the `analyze_trace` tool result and model context. Sanitize the transport error before returning it, as is already done for failure logs. ########## pkg/mcp/tools/trace/jaeger.go: ########## @@ -184,6 +188,218 @@ func (p *jaegerProvider) GetTraceByID(ctx context.Context, traceID string) (*Tra return result, nil } +// SearchTraces discovers candidate traces through the Jaeger v1 query API. +// The summaries are computed from the raw spans so that no information is lost to +// the normalization applied by the single trace lookup. +func (p *jaegerProvider) SearchTraces(ctx context.Context, query TraceSearchQuery) (*TraceSearchResult, error) { + if strings.TrimSpace(query.Service) == "" { + return nil, fmt.Errorf("service is required for trace search") + } + values := url.Values{} + values.Set("service", query.Service) + if query.Operation != "" { + values.Set("operation", query.Operation) + } + if query.TraceID != "" { + values.Set("traceID", query.TraceID) + } + if len(query.Tags) > 0 { + encoded, err := json.Marshal(query.Tags) + if err != nil { + return nil, fmt.Errorf("failed to encode trace search tags: %w", err) + } + values.Set("tags", string(encoded)) + } + if !query.StartTime.IsZero() { + values.Set("start", strconv.FormatInt(query.StartTime.UnixMicro(), 10)) + } + if !query.EndTime.IsZero() { + values.Set("end", strconv.FormatInt(query.EndTime.UnixMicro(), 10)) + } + if query.MinDuration > 0 { + values.Set("minDuration", strconv.FormatInt(query.MinDuration.Microseconds(), 10)) + } + if query.Limit > 0 { + values.Set("limit", strconv.Itoa(query.Limit)) + } + + endpoint := *p.baseURL + endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/api/traces" + endpoint.RawQuery = values.Encode() + startedAt := time.Now() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to create jaeger search request: %w", err) + } + req.Header.Set("Accept", "application/json") + if p.config.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+p.config.BearerToken) + } + if p.config.Tenant != "" { + req.Header.Set(tenantHeader, p.config.Tenant) + } + + resp, err := p.client.Do(req) + if err != nil { + p.logSearchFailure(query, startedAt, 0, "transport", err) + // The tool result keeps the transport error as it is: the result enters + // the model context and the model has to be able to answer which lookup + // failed and why. The failure log is sanitized instead. + return nil, fmt.Errorf("jaeger trace search failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + body, _ := io.ReadAll(io.LimitReader(resp.Body, maxTraceErrorBodyBytes)) + p.logSearchFailure(query, startedAt, resp.StatusCode, "upstream_status", nil) + return nil, fmt.Errorf("jaeger trace search failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + body, err := readTraceBody(resp.Body) + if err != nil { + p.logSearchFailure(query, startedAt, resp.StatusCode, "response_limit", err) + return nil, err + } + var upstream jaegerResponse + if err := json.Unmarshal(body, &upstream); err != nil { + p.logSearchFailure(query, startedAt, resp.StatusCode, "decode", err) + return nil, fmt.Errorf("failed to decode jaeger search response: %w", err) + } + if len(upstream.Errors) > 0 { + p.logSearchFailure(query, startedAt, resp.StatusCode, "upstream_error", nil) + return nil, fmt.Errorf("jaeger trace search failed: %s", upstream.Errors[0].Message) + } + result := summarizeJaegerTraces(upstream.Data) Review Comment: [P1] Jaeger has already applied `query.Limit` before this summary is built, so a response containing the default 20 traces is reported as `truncated: false` even when many more matches exist. The agent can then incorrectly conclude that it has examined all failures in the requested window. Mark a result that reaches the backend limit as potentially truncated (or request one extra result and trim it). ########## ai/component/hooks/reasoning.go: ########## @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package hooks + +import ( + "encoding/json" + "fmt" + "strings" +) + +// reasoningPartType is the part discriminator the generative AI semantic +// conventions define for reasoning/thinking content received from a model. +const reasoningPartType = "reasoning" + +// reasoningPartsKey is the message field that carries the parts of a chat +// message in the generative AI semantic conventions. +const reasoningPartsKey = "parts" + +// truncatedReasoningLimit is the number of characters kept from a single +// reasoning string when the truncated policy applies. It matches the limit +// used for the other captured JSON strings. +const truncatedReasoningLimit = 512 + +// ApplyReasoningPolicy filters or truncates the reasoning parts of a +// generative AI messages document according to the configured policy. +// +// The document is the serialized JSON of the gen_ai.input.messages or +// gen_ai.output.messages attribute. A reasoning part is an element of a +// "parts" array whose "type" is "reasoning"; every other part is preserved. +// +// - CaptureNone removes the reasoning parts from the document. +// - CaptureTruncated keeps them and shortens their content. +// - CaptureFull returns the document unchanged. +// +// An unset or unknown policy is treated like CaptureNone so reasoning content +// never reaches a trace by accident. A document that is not valid JSON is +// returned unchanged: a parsing failure must never drop captured content. +func ApplyReasoningPolicy(document string, policy string) string { + switch policy { + case CaptureFull: + return document + case CaptureTruncated: + return rewriteReasoningParts(document, truncateReasoningElement) + default: + return rewriteReasoningParts(document, nil) + } +} + +// normalizeCaptureReasoning resolves the reasoning capture policy of a hook +// registration. An unset policy defaults to CaptureNone so reasoning content +// is never captured unless a hook asks for it. +func normalizeCaptureReasoning(policy string) (string, error) { Review Comment: `ApplyReasoningPolicy` and `normalizeCaptureReasoning` have no callers outside this file and its test. Nothing in the manager, the tracing hook or the config loader uses them, so this is dead code on this branch. The PR description scopes this PR to trace search/diagnosis and says the agent-side reasoning capture (`capture_reasoning`) belongs to #1616. Please move this file to #1616 together with its wiring, so this PR stays self-contained and doesn't ship a policy that is never applied. ########## pkg/mcp/tools/trace/genai.go: ########## @@ -0,0 +1,403 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trace + +import ( + "sort" + "strconv" + "strings" + "time" +) + +// The generative AI semantic conventions are still in development, so the analyzer pins +// the recognized attribute set instead of trusting an open-ended namespace. +// Reference: OpenTelemetry semantic conventions for generative AI systems, commit 6fd0d76. +const genAIPrefix = "gen_ai." + +// Recognized generative AI attributes. +const ( + genAIOperationName = "gen_ai.operation.name" + genAIProviderName = "gen_ai.provider.name" + genAIRequestModel = "gen_ai.request.model" + genAIResponseModel = "gen_ai.response.model" + genAIConversationID = "gen_ai.conversation.id" + genAIAgentName = "gen_ai.agent.name" + genAIAgentID = "gen_ai.agent.id" + genAIAgentDescription = "gen_ai.agent.description" + genAIAgentVersion = "gen_ai.agent.version" + genAIToolName = "gen_ai.tool.name" + genAIToolCallID = "gen_ai.tool.call.id" + genAIToolCallArguments = "gen_ai.tool.call.arguments" + genAIToolCallResult = "gen_ai.tool.call.result" + genAIInputMessages = "gen_ai.input.messages" + genAIOutputMessages = "gen_ai.output.messages" + genAIUsageInputTokens = "gen_ai.usage.input_tokens" + genAIUsageOutputTokens = "gen_ai.usage.output_tokens" + genAIUsageReasoningTokens = "gen_ai.usage.reasoning.output_tokens" + genAIUsageCacheReadTokens = "gen_ai.usage.cache_read.input_tokens" + genAIUsageCacheWriteTokens = "gen_ai.usage.cache_write.input_tokens" + genAIResponseFinishReasons = "gen_ai.response.finish_reasons" + genAIResponseID = "gen_ai.response.id" + genAIDataSourceID = "gen_ai.data_source.id" + genAIOutputType = "gen_ai.output.type" +) + +// genAIUsageTotalTokens is not part of the semantic conventions, the conventions +// only define the input and the output token counters. It is honored when a +// provider emits it anyway, because some vendors already report the aggregate. +// The value stays a reference value in the detail of a single call, the aggregate +// of the trace is always the sum of the input and the output tokens. +const genAIUsageTotalTokens = "gen_ai.usage.total_tokens" + +// genAIContentAttributes hold prompts, completions and tool payloads. They are Opt-In +// in the semantic conventions and may embed PII, so their values are never read. +var genAIContentAttributes = map[string]struct{}{ + genAIInputMessages: {}, + genAIOutputMessages: {}, + genAIToolCallArguments: {}, + genAIToolCallResult: {}, +} + +// genAIRecognizedAttributes is the pinned set of attributes the analyzer is allowed to +// read. An unknown gen_ai.* attribute still marks a span as generative AI, but its +// value is never interpreted. +var genAIRecognizedAttributes = map[string]struct{}{ + genAIOperationName: {}, + genAIProviderName: {}, + genAIRequestModel: {}, + genAIResponseModel: {}, + genAIConversationID: {}, + genAIAgentName: {}, + genAIAgentID: {}, + genAIAgentDescription: {}, + genAIAgentVersion: {}, + genAIToolName: {}, + genAIToolCallID: {}, + genAIToolCallArguments: {}, + genAIToolCallResult: {}, + genAIInputMessages: {}, + genAIOutputMessages: {}, + genAIUsageInputTokens: {}, + genAIUsageOutputTokens: {}, + genAIUsageReasoningTokens: {}, + genAIUsageCacheReadTokens: {}, + genAIUsageCacheWriteTokens: {}, + genAIResponseFinishReasons: {}, + genAIResponseID: {}, + genAIDataSourceID: {}, + genAIOutputType: {}, + genAIUsageTotalTokens: {}, +} + +// genAITokenAttributes lists every token attribute the analyzer recognizes. The cache +// and reasoning counters are breakdowns of the input/output counters, so they are never +// added on top of them. +var genAITokenAttributes = []string{ + genAIUsageInputTokens, + genAIUsageOutputTokens, + genAIUsageReasoningTokens, + genAIUsageCacheReadTokens, + genAIUsageCacheWriteTokens, + genAIUsageTotalTokens, +} + +// genAIModelSignalAttributes mark a span as a model invocation even when the operation +// name is missing or carries a custom value. +var genAIModelSignalAttributes = []string{ + genAIRequestModel, + genAIResponseModel, + genAIResponseID, + genAIResponseFinishReasons, + genAIOutputType, + genAIDataSourceID, +} + +var ( + genAIModelOperations = map[string]struct{}{ + "chat": {}, + "generate_content": {}, + "text_completion": {}, + "embeddings": {}, + "fetch_response": {}, + } + genAIToolOperations = map[string]struct{}{ + "execute_tool": {}, + } + genAIAgentOperations = map[string]struct{}{ + "invoke_agent": {}, + "create_agent": {}, + "invoke_workflow": {}, + "plan": {}, + } +) + +// genAIClass is the mutually exclusive category a generative AI span falls into. +type genAIClass int + +const ( + genAIClassNone genAIClass = iota + genAIClassModel + genAIClassTool + genAIClassAgent +) + +// genAIOrderedSpan pairs a span with its resolved start time so the per-category +// detail lists can be emitted in chronological order. +type genAIOrderedSpan struct { + span *Span + startMicros int64 +} + +// HasGenAISpans reports whether any span carries generative AI attributes. +func HasGenAISpans(spans []Span) bool { + for i := range spans { + if HasGenAIAttributes(spans[i].Attributes) { + return true + } + } + return false +} + +// HasGenAIAttributes reports whether an attribute map contains generative AI attributes. +// The search path needs this check on raw provider tags, so it must not require a +// normalized span. +func HasGenAIAttributes(tags map[string]string) bool { + for key := range tags { + if isGenAIAttributeKey(key) { + return true + } + } + return false +} + +// BuildGenAISummary aggregates the generative AI footprint of the given spans. +// It never fails: spans without generative AI attributes yield Present=false. +func BuildGenAISummary(spans []Span) *GenAISummary { + summary := &GenAISummary{Present: false} + ordered := make([]genAIOrderedSpan, 0, len(spans)) + conversationIDs := map[string]struct{}{} + providers := map[string]struct{}{} + models := map[string]struct{}{} + for i := range spans { + span := &spans[i] + if !HasGenAIAttributes(span.Attributes) { + continue + } + summary.Present = true + ordered = append(ordered, genAIOrderedSpan{span: span, startMicros: spanStartMicros(span)}) + addString(conversationIDs, genAIAttribute(span.Attributes, genAIConversationID)) + addString(providers, genAIAttribute(span.Attributes, genAIProviderName)) + addString(models, genAIAttribute(span.Attributes, genAIRequestModel)) + addString(models, genAIAttribute(span.Attributes, genAIResponseModel)) + summary.InputTokens += genAIIntAttribute(span.Attributes, genAIUsageInputTokens) + summary.OutputTokens += genAIIntAttribute(span.Attributes, genAIUsageOutputTokens) + } + // The semantic conventions define the input and the output token counters + // only, there is no standard total token attribute. The total is therefore a + // derived value with a single definition: it always equals input plus output. + // A provider that reports the non standard gen_ai.usage.total_tokens only + // shows up as a reference value in the detail of a single call, it never + // takes part in the aggregate, so a consumer never has to choose between + // two definitions of the total. + summary.TotalTokens = summary.InputTokens + summary.OutputTokens + if !summary.Present { + return summary + } + sort.SliceStable(ordered, func(i, j int) bool { + if ordered[i].startMicros != ordered[j].startMicros { + return ordered[i].startMicros < ordered[j].startMicros + } + return ordered[i].span.SpanID < ordered[j].span.SpanID + }) + summary.ConversationIDs = sortedKeys(conversationIDs) + summary.Providers = sortedKeys(providers) + summary.Models = sortedKeys(models) + + for _, item := range ordered { + span := item.span + switch classifyGenAISpan(span) { + case genAIClassModel: + inputTokens := genAIIntAttribute(span.Attributes, genAIUsageInputTokens) + outputTokens := genAIIntAttribute(span.Attributes, genAIUsageOutputTokens) + totalTokens := genAISpanTotalTokens(span.Attributes) + summary.ModelCallCount++ + if len(summary.ModelCalls) < maxGenAICalls { + summary.ModelCalls = append(summary.ModelCalls, GenAIModelCall{ + SpanID: span.SpanID, + ServiceName: span.ServiceName, + OperationName: span.OperationName, + Operation: genAIAttribute(span.Attributes, genAIOperationName), + ProviderName: genAIAttribute(span.Attributes, genAIProviderName), + RequestModel: genAIAttribute(span.Attributes, genAIRequestModel), + ResponseModel: genAIAttribute(span.Attributes, genAIResponseModel), + InputTokens: inputTokens, + OutputTokens: outputTokens, + TotalTokens: totalTokens, + DurationMicros: span.DurationMicros, + Error: spanFailed(span), + }) + } else { + summary.Truncated = true + } + case genAIClassTool: + summary.ToolCallCount++ + if len(summary.ToolCalls) < maxGenAICalls { + summary.ToolCalls = append(summary.ToolCalls, GenAIToolCall{ + SpanID: span.SpanID, + ServiceName: span.ServiceName, + ToolName: genAIAttribute(span.Attributes, genAIToolName), + ToolCallID: genAIAttribute(span.Attributes, genAIToolCallID), + DurationMicros: span.DurationMicros, + Error: spanFailed(span), + }) + } else { + summary.Truncated = true + } + case genAIClassAgent: + summary.AgentCount++ + if len(summary.Agents) < maxGenAICalls { + summary.Agents = append(summary.Agents, GenAIAgent{ + SpanID: span.SpanID, + ServiceName: span.ServiceName, + AgentName: genAIAttribute(span.Attributes, genAIAgentName), + AgentID: genAIAttribute(span.Attributes, genAIAgentID), + DurationMicros: span.DurationMicros, + Error: spanFailed(span), + }) + } else { + summary.Truncated = true + } + } + } + return summary +} + +// classifyGenAISpan assigns exactly one category to a generative AI span. +// The precedence is model operation, then tool, then agent, and finally a model call +// inferred from model or token attributes. +func classifyGenAISpan(span *Span) genAIClass { + operation := strings.ToLower(strings.TrimSpace(genAIAttribute(span.Attributes, genAIOperationName))) + if _, ok := genAIModelOperations[operation]; ok { + return genAIClassModel + } + if _, ok := genAIToolOperations[operation]; ok { + return genAIClassTool + } + if _, ok := genAIAgentOperations[operation]; ok { + return genAIClassAgent + } + if _, ok := genAIAttributeOK(span.Attributes, genAIToolName); ok { + return genAIClassTool + } + if _, ok := genAIAttributeOK(span.Attributes, genAIAgentName); ok { + return genAIClassAgent + } + for _, name := range genAIModelSignalAttributes { + if _, ok := genAIAttributeOK(span.Attributes, name); ok { + return genAIClassModel + } + } + for _, name := range genAITokenAttributes { + if _, ok := genAIAttributeOK(span.Attributes, name); ok { + return genAIClassModel + } + } + return genAIClassNone +} + +// genAISpanTotalTokens returns the token total of a single span. A provider that +// reports the aggregate of the span wins, otherwise the total is derived from the +// input and the output counters of that span. The value is only meaningful per +// span: it is a reference value in the detail of a single call and never takes +// part in the aggregate of the trace, whose total is always input plus output. +func genAISpanTotalTokens(attributes map[string]string) int { + if override, ok := genAIAttributeOK(attributes, genAIUsageTotalTokens); ok { + if parsed, err := strconv.Atoi(strings.TrimSpace(override)); err == nil { + return parsed + } + } + return genAIIntAttribute(attributes, genAIUsageInputTokens) + genAIIntAttribute(attributes, genAIUsageOutputTokens) +} + +func isGenAIAttributeKey(key string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(key)), genAIPrefix) +} + +// genAIAttribute reads a recognized, non-content generative AI attribute. +func genAIAttribute(attributes map[string]string, name string) string { + value, _ := genAIAttributeOK(attributes, name) + return value +} + +// genAIAttributeOK reads a recognized, non-content generative AI attribute. +func genAIAttributeOK(attributes map[string]string, name string) (string, bool) { + if _, content := genAIContentAttributes[name]; content { + return "", false + } + if _, recognized := genAIRecognizedAttributes[name]; !recognized { + return "", false + } + return lookupAttributeOK(attributes, name) +} + +func genAIIntAttribute(attributes map[string]string, name string) int { + raw, ok := genAIAttributeOK(attributes, name) + if !ok { + return 0 + } + parsed, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil { + return 0 + } + return parsed Review Comment: Numeric Jaeger tags do not always reach this parser as decimal integer strings. The existing JSON decoder produces `float64`, and `formatAttributeValue` formats 1000000 as `"1e+06"`; `Atoi` rejects that and this silently reports zero input/output tokens. Thus a valid million-token model call loses its usage in both call details and the trace aggregate. Please accept integral scientific notation as well as decimal integers. ########## pkg/mcp/tools/trace/analyzer.go: ########## @@ -0,0 +1,821 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package trace + +import ( + "context" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + consolectx "github.com/apache/dubbo-admin/pkg/console/context" + "github.com/apache/dubbo-admin/pkg/mcp/common" +) + +const ( + // Bounded output limits. They keep the tool payload small enough to stay inside + // the model context window while still describing the most relevant spans. + maxCriticalPath = 32 + maxServiceBreakdown = 50 + maxSlowSpans = 10 + maxErrorSpans = 20 + maxDependencyEdges = 50 + maxGenAICalls = 20 + + maxAnalysisSummaryChars = 300 + maxErrorTypeChars = 96 + maxErrorMessageChars = 256 +) + +const ( + errorTypeAttribute = "error.type" + errorMessageAttribute = "error.message" + exceptionTypeAttribute = "exception.type" + exceptionMessageAttribute = "exception.message" + spanEventNameAttribute = "event" + standardExceptionEvent = "exception" + genAIExceptionEvent = "gen_ai.client.operation.exception" +) + +// analysisSpan is the analyzer's internal view of a span with a resolved timeline. +type analysisSpan struct { + span *Span + position int + startMicros int64 + endMicros int64 + timeValid bool + selfMicros int64 + waitMicros int64 + parent *analysisSpan + children []*analysisSpan +} + +// timelineInterval is a half-open interval on the absolute microsecond timeline. +type timelineInterval struct { + start int64 + end int64 +} + +// AnalyzeTrace diagnoses a single distributed trace and returns a structured report. +func AnalyzeTrace(ctx consolectx.Context, args map[string]any) (*common.ToolResult, error) { + helper := common.NewArgsHelper(args) + traceID := helper.GetString("traceId", "") + if !traceIDPattern.MatchString(traceID) { + return common.ErrorResult(fmt.Errorf("traceId must be a 16 or 32 character hexadecimal value")), nil + } + providerConfig, err := providerConfigFromContext(ctx, helper.GetString("provider", "")) + if err != nil { + return common.ErrorResult(err), nil + } + provider, err := newProvider(providerConfig) + if err != nil { + return common.ErrorResult(err), nil + } + trace, err := provider.GetTraceByID(traceRequestContext(ctx), traceID) + if err != nil { + return common.ErrorResult(err), nil + } + return common.JsonResult(BuildTraceAnalysis(trace)) +} + +// AnalyzeProperties returns the MCP input schema properties for trace diagnosis. +func AnalyzeProperties() map[string]common.PropertyDef { + return map[string]common.PropertyDef{ + "traceId": {Type: "string", Description: "Jaeger 64-bit or 128-bit hexadecimal TraceID"}, + "provider": {Type: "string", Description: "Optional trace provider name; defaults to defaultProvider"}, + } +} + +// BuildTraceAnalysis turns a normalized trace into a structured diagnosis. +// It is a pure function so the diagnosis rules can be verified without a backend. +func BuildTraceAnalysis(trace *Trace) *TraceAnalysis { + analysis := &TraceAnalysis{} + if trace == nil { + analysis.Summary = "no trace data available" + return analysis + } + analysis.TraceID = trace.TraceID + analysis.Truncated = trace.Truncated + + nodes := buildAnalysisTree(trace.Spans) + computeSelfTimes(nodes, analysis) + criticalPath, criticalPathSet := buildCriticalPath(nodes, analysis) + analysis.CriticalPath = criticalPath + + wallClock, traceStart := analysisEnvelope(nodes, trace) + analysis.DurationMicros = wallClock + analysis.StartTime = traceStart + analysis.SpanCount = maxInt(trace.SpanCount, len(trace.Spans)) + + breakdown := buildServiceBreakdown(nodes, wallClock, analysis) + analysis.ServiceBreakdown = breakdown + analysis.ServiceCount = maxInt(trace.ServiceCount, len(breakdown)) + + slowSpans := buildSlowSpans(nodes, wallClock, criticalPathSet, analysis) + analysis.SlowSpans = make([]SlowSpan, 0, len(slowSpans)) + for _, entry := range slowSpans { + analysis.SlowSpans = append(analysis.SlowSpans, entry.slow) + } + + errorSpans, errorCount := buildErrorSpans(nodes, analysis) + analysis.ErrorSpans = errorSpans + // The provider already counted errors over the complete trace; that count must not + // be hidden by the bounded error span list. + analysis.ErrorSpanCount = maxInt(trace.ErrorSpanCount, errorCount) + + analysis.Dependencies = buildDependencyEdges(nodes, analysis) + + if genai := BuildGenAISummary(trace.Spans); genai.Present { + analysis.GenAI = genai + } + + analysis.Summary = buildAnalysisSummary(analysis, slowSpans) + return analysis +} + +// buildAnalysisTree links the spans into a parent/child forest. A span whose parent +// is missing from the trace degrades into an orphan root: it still contributes to the +// aggregate statistics but it cannot extend a parent chain. +func buildAnalysisTree(spans []Span) []*analysisSpan { + nodes := make([]*analysisSpan, 0, len(spans)) + index := make(map[string]*analysisSpan, len(spans)) + for i := range spans { + span := &spans[i] + node := &analysisSpan{span: span, position: i} + if parsed, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(span.StartTime)); err == nil { + node.timeValid = true + node.startMicros = parsed.UnixMicro() + node.endMicros = node.startMicros + span.DurationMicros + } else { + node.endMicros = span.DurationMicros + } + nodes = append(nodes, node) + if span.SpanID != "" { + index[span.SpanID] = node + } + } + for _, node := range nodes { + parentID := node.span.ParentSpanID + if parentID == "" { + continue + } + parent, ok := index[parentID] + if !ok || parent == node { + continue + } + node.parent = parent + parent.children = append(parent.children, node) + } + return nodes +} + +// computeSelfTimes derives the exclusive time of every span. The covered time is the +// union of the clipped child intervals, so concurrent children are not counted twice. +func computeSelfTimes(nodes []*analysisSpan, analysis *TraceAnalysis) { + for _, node := range nodes { + if !node.timeValid { + // Without a trustworthy start time the span degrades to a leaf. It + // cannot be placed on the timeline, so it must not claim any time + // of its own: counting its full duration here would let the parent + // account for the same time a second time. + node.selfMicros = 0 + node.waitMicros = 0 + analysis.Truncated = true + continue + } + intervals := make([]timelineInterval, 0, len(node.children)) + for _, child := range node.children { + if !child.timeValid { + analysis.Truncated = true + continue + } + start := maxInt64(child.startMicros, node.startMicros) + end := minInt64(child.endMicros, node.endMicros) + if end <= start { + continue + } + intervals = append(intervals, timelineInterval{start: start, end: end}) + } + covered := unionLength(intervals) + node.waitMicros = covered + node.selfMicros = node.span.DurationMicros - covered + if node.selfMicros < 0 { + node.selfMicros = 0 + } + } +} + +// unionLength returns the length of the union of the given intervals. +func unionLength(intervals []timelineInterval) int64 { + if len(intervals) == 0 { + return 0 + } + sort.Slice(intervals, func(i, j int) bool { + if intervals[i].start != intervals[j].start { + return intervals[i].start < intervals[j].start + } + return intervals[i].end < intervals[j].end + }) + total := int64(0) + current := intervals[0] + for _, next := range intervals[1:] { + if next.start > current.end { + total += current.end - current.start + current = next + continue + } + if next.end > current.end { + current.end = next.end + } + } + return total + current.end - current.start +} + +// buildCriticalPath walks the latency determining root-to-leaf chain. +// The second return value holds the analyzer internal positions of the spans on the +// path. The position is used instead of the business span id because a span id can +// be empty or duplicated, which would make two different spans overwrite each other. +func buildCriticalPath(nodes []*analysisSpan, analysis *TraceAnalysis) ([]CriticalPathEntry, map[int]struct{}) { + roots := make([]*analysisSpan, 0, len(nodes)) + for _, node := range nodes { + if node.parent == nil { + roots = append(roots, node) + } + } + entries := make([]CriticalPathEntry, 0, maxCriticalPath) + onPath := map[int]struct{}{} + visited := map[int]struct{}{} + current := selectCriticalRoot(roots) + for current != nil { + if len(entries) >= maxCriticalPath { + // The budget is exhausted while the chain still continues, so the + // remaining hops are dropped and the analysis is incomplete. + analysis.Truncated = true + break + } + if _, seen := visited[current.position]; seen { + break + } + visited[current.position] = struct{}{} + onPath[current.position] = struct{}{} + entries = append(entries, CriticalPathEntry{ + SpanID: current.span.SpanID, + ServiceName: current.span.ServiceName, + OperationName: current.span.OperationName, + StartTime: current.span.StartTime, + DurationMicros: current.span.DurationMicros, + SelfMicros: current.selfMicros, + WaitMicros: current.waitMicros, + }) + current = selectCriticalChild(current) + } + return entries, onPath +} + +// selectCriticalRoot picks the root that finishes last, because that root determines +// the end of the request. +func selectCriticalRoot(roots []*analysisSpan) *analysisSpan { + var best *analysisSpan + for _, root := range roots { + if !root.timeValid { + continue + } + if best == nil || betterCriticalRoot(root, best) { + best = root + } + } + if best != nil { + return best + } + for _, root := range roots { + if best == nil || betterCriticalRoot(root, best) { + best = root + } + } + return best +} + +func betterCriticalRoot(candidate, current *analysisSpan) bool { + if candidate.endMicros != current.endMicros { + return candidate.endMicros > current.endMicros + } + if candidate.span.DurationMicros != current.span.DurationMicros { + return candidate.span.DurationMicros > current.span.DurationMicros + } + // The start time is derived from the end time and the duration, so it is + // already equal here and cannot be used as a tie breaker. + return candidate.span.SpanID < current.span.SpanID +} + +// selectCriticalChild picks the child that finishes last, because it is the activity +// that determined when the parent completed. +func selectCriticalChild(node *analysisSpan) *analysisSpan { + var best *analysisSpan + for _, child := range node.children { + if !child.timeValid { + // A span without a trustworthy timestamp degrades to a leaf and is not + // allowed to extend the chain. + continue + } + if best == nil || betterCriticalChild(child, best) { + best = child + } + } + return best +} + +func betterCriticalChild(candidate, current *analysisSpan) bool { + if candidate.endMicros != current.endMicros { + return candidate.endMicros > current.endMicros + } + if candidate.span.DurationMicros != current.span.DurationMicros { + return candidate.span.DurationMicros > current.span.DurationMicros + } + return candidate.span.SpanID < current.span.SpanID +} + +// analysisEnvelope returns the wall clock duration and the start timestamp of the trace. +func analysisEnvelope(nodes []*analysisSpan, trace *Trace) (int64, string) { + var minStart, maxEnd int64 + hasStart := false + startText := trace.StartTime + for _, node := range nodes { + if !node.timeValid { + continue + } + if !hasStart || node.startMicros < minStart { + minStart = node.startMicros + startText = node.span.StartTime + hasStart = true + } + if node.endMicros > maxEnd { + maxEnd = node.endMicros + } + } + if !hasStart { + return trace.DurationMicros, trace.StartTime + } + return maxEnd - minStart, startText +} + +// buildServiceBreakdown aggregates the spans per service. +func buildServiceBreakdown(nodes []*analysisSpan, wallClock int64, analysis *TraceAnalysis) []ServiceLatency { + aggregates := map[string]*ServiceLatency{} + for _, node := range nodes { + span := node.span + entry, ok := aggregates[span.ServiceName] + if !ok { + entry = &ServiceLatency{ServiceName: span.ServiceName} + aggregates[span.ServiceName] = entry + } + entry.SpanCount++ + if spanFailed(span) { + entry.ErrorSpanCount++ + } + entry.TotalMicros += span.DurationMicros + if span.DurationMicros > entry.MaxMicros { + entry.MaxMicros = span.DurationMicros + } + } + result := make([]ServiceLatency, 0, len(aggregates)) + for _, entry := range aggregates { + if entry.SpanCount > 0 { + entry.AvgMicros = entry.TotalMicros / int64(entry.SpanCount) + } + // The share is the service workload relative to the wall clock of the trace. + // Concurrent spans can push it above 100 percent on purpose. + entry.SharePercent = percentOf(entry.TotalMicros, wallClock) + result = append(result, *entry) + } + sort.SliceStable(result, func(i, j int) bool { + if result[i].TotalMicros != result[j].TotalMicros { + return result[i].TotalMicros > result[j].TotalMicros + } + return result[i].ServiceName < result[j].ServiceName + }) + if len(result) > maxServiceBreakdown { + result = result[:maxServiceBreakdown] + analysis.Truncated = true + } + return result +} + +// slowSpanEntry pairs a slow span with the analyzer internal identity of the span +// it was derived from. The position is used instead of the business span id so +// that spans without an id, or with duplicated ids, cannot overwrite each other. +type slowSpanEntry struct { + slow SlowSpan + position int + selfMicros int64 +} + +// buildSlowSpans ranks the spans by the time they consumed on their own. +// The exclusive time is returned alongside the report so the summary can quote it +// even for spans that are not part of the critical path. +func buildSlowSpans(nodes []*analysisSpan, wallClock int64, criticalPath map[int]struct{}, analysis *TraceAnalysis) []slowSpanEntry { + candidates := make([]slowSpanEntry, 0, len(nodes)) + for _, node := range nodes { + span := node.span + _, onPath := criticalPath[node.position] + candidates = append(candidates, slowSpanEntry{ + slow: SlowSpan{ + SpanID: span.SpanID, + ServiceName: span.ServiceName, + OperationName: span.OperationName, + DurationMicros: span.DurationMicros, + SharePercent: percentOf(node.selfMicros, wallClock), + OnCriticalPath: onPath, + }, + position: node.position, + selfMicros: node.selfMicros, + }) + } + sort.SliceStable(candidates, func(i, j int) bool { + if candidates[i].selfMicros != candidates[j].selfMicros { + return candidates[i].selfMicros > candidates[j].selfMicros + } + return candidates[i].position < candidates[j].position + }) + if len(candidates) > maxSlowSpans { + candidates = candidates[:maxSlowSpans] + analysis.Truncated = true + } + return candidates +} + +// errorSpanEntry pairs an error span with the analyzer node it was derived from. +// The node is kept so the ordering can use the start time of the span without +// indexing the table by the business span id. +type errorSpanEntry struct { + span ErrorSpan + node *analysisSpan +} + +// buildErrorSpans collects the failed spans and marks the root causes. +func buildErrorSpans(nodes []*analysisSpan, analysis *TraceAnalysis) ([]ErrorSpan, int) { + failed := make([]*analysisSpan, 0, len(nodes)) + for _, node := range nodes { + if spanFailed(node.span) { + failed = append(failed, node) + } + } + entries := make([]errorSpanEntry, 0, len(failed)) + for _, node := range failed { + span := node.span + errorType, errorMessage := errorDetails(span) + entries = append(entries, errorSpanEntry{ + span: ErrorSpan{ + SpanID: span.SpanID, + ServiceName: span.ServiceName, + OperationName: span.OperationName, + ErrorType: errorType, + ErrorMessage: errorMessage, + Status: errorStatus(span), + RootCause: !hasFailedDescendant(node), + }, + node: node, + }) + } + // Root causes come first so the agent sees the deepest failure before the + // ancestors that only propagate it. + sort.SliceStable(entries, func(i, j int) bool { + left, right := entries[i], entries[j] + if left.span.RootCause != right.span.RootCause { + return left.span.RootCause + } + if left.node.startMicros != right.node.startMicros { + return left.node.startMicros < right.node.startMicros + } + return left.node.position < right.node.position + }) + count := len(entries) + if len(entries) > maxErrorSpans { + entries = entries[:maxErrorSpans] + analysis.Truncated = true + } + result := make([]ErrorSpan, 0, len(entries)) + for _, entry := range entries { + result = append(result, entry.span) + } + return result, count +} + +// errorDetails resolves the error classification and the human readable description. +// +// The sources are ordered "generative AI first, standard second, non standard +// last": +// - The span attribute error.type stays the leading source because the OpenTelemetry +// error registry defines it as the stable error type of a span. +// - The generative AI exception event comes before the standard exception event +// because this project serves the generative AI execution of the agent loop, so +// the typed error of the model call is the more relevant classification. +// - error.message is only the last resort because it is not an OpenTelemetry +// standard attribute: the registry only defines error.type, and the stable +// attribute for the exception text is exception.message. error.message is a +// habit of the Java ecosystem and of some collectors, so it is used only when +// no typed error is available, and it then degrades into the error type. +func errorDetails(span *Span) (string, string) { + errorType := "" + if value, ok := lookupAttributeOK(span.Attributes, errorTypeAttribute); ok { + errorType = strings.TrimSpace(value) + } + exceptionType, exceptionMessage := exceptionDetails(span) + if errorType == "" { + errorType = exceptionType + } + errorMessage := exceptionMessage + if errorMessage == "" { + if value, ok := lookupAttributeOK(span.Attributes, errorMessageAttribute); ok { + errorMessage = strings.TrimSpace(value) + } + } + if errorType == "" && errorMessage != "" { + // Degraded classification: without a typed error the message is the only + // signal left, so it degrades into the error type. + errorType = errorMessage + } + if errorType == "" { + errorType = "unknown" + } + return truncateRunes(errorType, maxErrorTypeChars), truncateRunes(errorMessage, maxErrorMessageChars) +} + +// exceptionDetails reads the exception type and message from the span events. +// The generative AI exception event wins over the standard exception event, +// because this project serves the generative AI execution of the agent loop and +// the generative AI event carries the typed error of the model call. Collectors +// also report the exception as a plain span attribute, so the attributes are read +// as a fallback once the events have been exhausted. +func exceptionDetails(span *Span) (string, string) { + exceptionType := "" + exceptionMessage := "" + collect := func(attributes map[string]string) { + if exceptionType == "" { + if value, ok := lookupAttributeOK(attributes, exceptionTypeAttribute); ok { + exceptionType = strings.TrimSpace(value) + } + } + if exceptionMessage == "" { + if value, ok := lookupAttributeOK(attributes, exceptionMessageAttribute); ok { + exceptionMessage = strings.TrimSpace(value) + } + } + } + for _, event := range span.Events { + if strings.EqualFold(eventName(event), genAIExceptionEvent) { + collect(event.Attributes) + } + } + for _, event := range span.Events { + if strings.EqualFold(eventName(event), standardExceptionEvent) { + collect(event.Attributes) + } + } + // The span attributes are the last resort: the two documented exception + // events stay the primary source, and no other event is consulted. + collect(span.Attributes) + return exceptionType, exceptionMessage +} + +func eventName(event Event) string { + if value, ok := lookupAttributeOK(event.Attributes, spanEventNameAttribute); ok { + return value + } + return "" +} + +// errorStatus returns the normalized status of a failed span. +func errorStatus(span *Span) string { + if spanFailed(span) { + return "ERROR" + } + if span.Status == "" { + return "UNSET" + } + return span.Status +} + +// hasFailedDescendant reports whether a failed span only propagates an error that +// happened deeper in the trace. +func hasFailedDescendant(node *analysisSpan) bool { + visited := map[int]struct{}{} + stack := make([]*analysisSpan, 0, len(node.children)) + stack = append(stack, node.children...) + for len(stack) > 0 { + current := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if _, seen := visited[current.position]; seen { + continue + } + visited[current.position] = struct{}{} + if spanFailed(current.span) { + return true + } + stack = append(stack, current.children...) + } + return false +} + +// buildDependencyEdges derives the cross service call graph of the trace. +func buildDependencyEdges(nodes []*analysisSpan, analysis *TraceAnalysis) []DependencyEdge { + aggregates := map[string]*DependencyEdge{} + for _, node := range nodes { + if node.parent == nil { + continue + } + caller := node.parent.span.ServiceName + callee := node.span.ServiceName + if caller == callee { + continue + } + key := caller + "\x00" + callee + edge, ok := aggregates[key] + if !ok { + edge = &DependencyEdge{CallerService: caller, CalleeService: callee} + aggregates[key] = edge + } + edge.SpanCount++ + edge.TotalMicros += node.span.DurationMicros + if spanFailed(node.span) { + edge.ErrorCount++ + } + } + result := make([]DependencyEdge, 0, len(aggregates)) + for _, edge := range aggregates { + result = append(result, *edge) + } + sort.SliceStable(result, func(i, j int) bool { + if result[i].TotalMicros != result[j].TotalMicros { + return result[i].TotalMicros > result[j].TotalMicros + } + if result[i].CallerService != result[j].CallerService { + return result[i].CallerService < result[j].CallerService + } + return result[i].CalleeService < result[j].CalleeService + }) + if len(result) > maxDependencyEdges { + result = result[:maxDependencyEdges] + analysis.Truncated = true + } + return result +} + +// buildAnalysisSummary renders the one line conclusion of the diagnosis. +func buildAnalysisSummary(analysis *TraceAnalysis, slowSpans []slowSpanEntry) string { + base := fmt.Sprintf("trace %d %s, %d %s, %s, %d %s", + analysis.ServiceCount, pluralize(analysis.ServiceCount, "service", "services"), + analysis.SpanCount, pluralize(analysis.SpanCount, "span", "spans"), + formatMicros(analysis.DurationMicros), + analysis.ErrorSpanCount, pluralize(analysis.ErrorSpanCount, "error", "errors")) + + var slowPart string + if len(slowSpans) > 0 { + slowest := slowSpans[0] + slowPart = fmt.Sprintf("; slowest: %s/%s self %s of %s (%.0f%% of trace)", + sanitizeSummaryText(slowest.slow.ServiceName), sanitizeSummaryText(slowest.slow.OperationName), + formatMicros(slowest.selfMicros), formatMicros(slowest.slow.DurationMicros), slowest.slow.SharePercent) + } + var rootPart string + for _, span := range analysis.ErrorSpans { + if !span.RootCause { + continue + } + rootPart = fmt.Sprintf("; root cause: %s/%s %s %s", + sanitizeSummaryText(span.ServiceName), sanitizeSummaryText(span.OperationName), + span.Status, sanitizeSummaryText(span.ErrorType)) + break + } + for _, candidate := range []string{base + slowPart + rootPart, base + rootPart, base} { + // The budget counts runes, not bytes: a Chinese service name uses three + // bytes per character and would otherwise push the diagnosis out of the + // summary even though it still fits. + if utf8.RuneCountInString(candidate) <= maxAnalysisSummaryChars { + return candidate + } + } + return truncateRunes(base+slowPart+rootPart, maxAnalysisSummaryChars) +} + +// pluralize returns the singular or the plural form depending on the count. +func pluralize(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + +// lookupAttributeOK resolves an attribute value, tolerating casing and padding +// differences introduced by different collectors. +func lookupAttributeOK(attributes map[string]string, name string) (string, bool) { + if value, ok := attributes[name]; ok { + return value, true + } + bestKey := "" + bestValue := "" + found := false + for key, value := range attributes { + if !strings.EqualFold(strings.TrimSpace(key), name) { + continue + } + if !found || key < bestKey { + bestKey, bestValue, found = key, value, true + } + } + return bestValue, found +} + +// percentOf computes a share with two decimals. A zero denominator yields zero +// instead of a division by zero. +func percentOf(value, total int64) float64 { + if total <= 0 { + return 0 + } + return math.Round(float64(value)/float64(total)*100*100) / 100 +} + +// formatMicros renders a microsecond duration in a compact human readable form. +func formatMicros(micros int64) string { + switch { + case micros < 1000: + return strconv.FormatInt(micros, 10) + "us" + case micros < 1000*1000: + return trimDecimal(float64(micros)/1000, 1) + "ms" + default: + return trimDecimal(float64(micros)/(1000*1000), 2) + "s" + } +} + +func trimDecimal(value float64, decimals int) string { + formatted := strconv.FormatFloat(value, 'f', decimals, 64) + if strings.Contains(formatted, ".") { + formatted = strings.TrimRight(formatted, "0") + formatted = strings.TrimSuffix(formatted, ".") + } + return formatted +} + +// sanitizeSummaryText keeps the one line summary free of line breaks. +func sanitizeSummaryText(value string) string { + replaced := strings.NewReplacer("\r", " ", "\n", " ", "\t", " ").Replace(value) + return strings.TrimSpace(replaced) +} + +// truncateRunes limits a string by rune count so multi byte characters stay intact. +func truncateRunes(value string, limit int) string { + if limit <= 0 { + return "" + } + runes := []rune(value) + if len(runes) <= limit { + return value + } + return string(runes[:limit]) +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func maxInt64(a, b int64) int64 { + if a > b { + return a + } + return b +} + +func minInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +// requestContext derives the outbound request context from the console context. Review Comment: The doc comment names `requestContext`, but the function is `traceRequestContext`. Also, `GetTraceByID` in `tools.go` (lines 47–50) repeats the same fallback inline. Optionally, have it call this helper so all three trace tools derive the request context the same way. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
