hanahmily commented on code in PR #918: URL: https://github.com/apache/skywalking-banyandb/pull/918#discussion_r2652766463
########## fodc/proxy/internal/metrics/aggregator.go: ########## @@ -0,0 +1,303 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package metrics provides functionality for aggregating and enriching metrics from all agents. +package metrics + +import ( + "context" + "fmt" + "sync" + "time" + + fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1" + "github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry" + "github.com/apache/skywalking-banyandb/pkg/logger" +) + +const ( + // defaultCollectionTimeout is the default timeout for collecting metrics from agents. + defaultCollectionTimeout = 10 * time.Second + // maxCollectionTimeout is the maximum timeout allowed for collecting metrics, + // preventing excessively long waits for wide time windows. + maxCollectionTimeout = 5 * time.Minute +) + +// AggregatedMetric represents an aggregated metric with node metadata. +type AggregatedMetric struct { + Labels map[string]string + Timestamp time.Time + Name string + AgentID string + NodeRole string + Description string + Value float64 +} + +// Filter defines filters for metrics collection. +type Filter struct { + StartTime *time.Time + EndTime *time.Time + Role string + Address string + AgentIDs []string +} + +// Aggregator aggregates and enriches metrics from all agents. +type Aggregator struct { + registry *registry.AgentRegistry + logger *logger.Logger + grpcService RequestSender + metricsCh chan *AggregatedMetric + collecting map[string]chan []*AggregatedMetric + mu sync.RWMutex + collectingMu sync.RWMutex +} + +// RequestSender is an interface for sending metrics requests to agents. +type RequestSender interface { + RequestMetrics(ctx context.Context, agentID string, startTime, endTime *time.Time) error +} + +// NewAggregator creates a new MetricsAggregator instance. +func NewAggregator(registry *registry.AgentRegistry, grpcService RequestSender, logger *logger.Logger) *Aggregator { + return &Aggregator{ + registry: registry, + grpcService: grpcService, + logger: logger, + metricsCh: make(chan *AggregatedMetric, 1000), + collecting: make(map[string]chan []*AggregatedMetric), + } +} + +// SetGRPCService sets the gRPC service for sending metrics requests. +func (ma *Aggregator) SetGRPCService(grpcService RequestSender) { + ma.mu.Lock() + defer ma.mu.Unlock() + ma.grpcService = grpcService +} + +// ProcessMetricsFromAgent processes metrics received from an agent. +func (ma *Aggregator) ProcessMetricsFromAgent(ctx context.Context, agentID string, agentInfo *registry.AgentInfo, req *fodcv1.StreamMetricsRequest) error { + aggregatedMetrics := make([]*AggregatedMetric, 0, len(req.Metrics)) + + for _, metric := range req.Metrics { + labels := make(map[string]string) + for key, value := range metric.Labels { + labels[key] = value + } + + labels["agent_id"] = agentID + labels["node_role"] = agentInfo.NodeRole + labels["ip"] = agentInfo.PrimaryAddress.IP + labels["port"] = fmt.Sprintf("%d", agentInfo.PrimaryAddress.Port) + + for key, value := range agentInfo.Labels { + labels[key] = value + } + + var timestamp time.Time + switch { + case metric.Timestamp != nil: + timestamp = metric.Timestamp.AsTime() + case req.Timestamp != nil: + timestamp = req.Timestamp.AsTime() + default: + timestamp = time.Now() + } + + aggregatedMetric := &AggregatedMetric{ + Name: metric.Name, + Labels: labels, + Value: metric.Value, + Timestamp: timestamp, + AgentID: agentID, + NodeRole: agentInfo.NodeRole, + Description: metric.Description, + } + + aggregatedMetrics = append(aggregatedMetrics, aggregatedMetric) + } + + ma.collectingMu.RLock() + collectCh, exists := ma.collecting[agentID] + ma.collectingMu.RUnlock() + + if exists { Review Comment: If the agentID does not exist, raise an error or log it. ########## fodc/proxy/internal/grpc/service.go: ########## @@ -0,0 +1,385 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package grpc + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1" + "github.com/apache/skywalking-banyandb/fodc/proxy/internal/metrics" + "github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry" + "github.com/apache/skywalking-banyandb/pkg/logger" +) + +// AgentConnection represents a connection to an agent. +type AgentConnection struct { + MetricsStream fodcv1.FODCService_StreamMetricsServer + Context context.Context + Stream grpc.ServerStream + Cancel context.CancelFunc + LastActivity time.Time + AgentID string + Identity registry.AgentIdentity + mu sync.RWMutex +} + +// UpdateActivity updates the last activity time. +func (ac *AgentConnection) UpdateActivity() { + ac.mu.Lock() + defer ac.mu.Unlock() + ac.LastActivity = time.Now() +} + +// GetLastActivity returns the last activity time. +func (ac *AgentConnection) GetLastActivity() time.Time { + ac.mu.RLock() + defer ac.mu.RUnlock() + return ac.LastActivity +} + +// FODCService implements the FODC gRPC service. +type FODCService struct { + fodcv1.UnimplementedFODCServiceServer + registry *registry.AgentRegistry + metricsAggregator *metrics.Aggregator + logger *logger.Logger + connections map[string]*AgentConnection + connectionsMu sync.RWMutex + heartbeatInterval time.Duration +} + +// NewFODCService creates a new FODCService instance. +func NewFODCService(registry *registry.AgentRegistry, metricsAggregator *metrics.Aggregator, logger *logger.Logger, heartbeatInterval time.Duration) *FODCService { + return &FODCService{ + registry: registry, + metricsAggregator: metricsAggregator, + logger: logger, + connections: make(map[string]*AgentConnection), + heartbeatInterval: heartbeatInterval, + } +} + +// RegisterAgent handles bi-directional agent registration stream. +func (s *FODCService) RegisterAgent(stream fodcv1.FODCService_RegisterAgentServer) error { + ctx, cancel := context.WithCancel(stream.Context()) + defer cancel() + + var agentID string + var agentConn *AgentConnection + initialRegistration := true + + for { + req, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + s.logger.Debug().Str("agent_id", agentID).Msg("Registration stream closed by agent") + break + } + if recvErr != nil { + s.logger.Error().Err(recvErr).Str("agent_id", agentID).Msg("Error receiving registration request") + return recvErr + } + + if initialRegistration { + identity := registry.AgentIdentity{ + IP: req.PrimaryAddress.Ip, + Port: int(req.PrimaryAddress.Port), + Role: req.NodeRole, + Labels: req.Labels, + } + + primaryAddr := registry.Address{ + IP: req.PrimaryAddress.Ip, + Port: int(req.PrimaryAddress.Port), + } + + registeredAgentID, registerErr := s.registry.RegisterAgent(ctx, identity, primaryAddr) + if registerErr != nil { + resp := &fodcv1.RegisterAgentResponse{ + Success: false, + Message: registerErr.Error(), + } + if sendErr := stream.Send(resp); sendErr != nil { + s.logger.Error().Err(sendErr).Msg("Failed to send registration error response") + } + return registerErr + } + + agentID = registeredAgentID + agentConn = &AgentConnection{ + AgentID: agentID, + Identity: identity, + Stream: stream, + Context: ctx, + Cancel: cancel, + LastActivity: time.Now(), + } + + s.connectionsMu.Lock() + s.connections[agentID] = agentConn + s.connectionsMu.Unlock() + + resp := &fodcv1.RegisterAgentResponse{ + Success: true, + Message: "Agent registered successfully", + HeartbeatIntervalSeconds: int64(s.heartbeatInterval.Seconds()), + AgentId: agentID, + } + + if sendErr := stream.Send(resp); sendErr != nil { + s.logger.Error().Err(sendErr).Str("agent_id", agentID).Msg("Failed to send registration response") + s.cleanupConnection(agentID) + // Unregister agent since we couldn't send confirmation + if unregisterErr := s.registry.UnregisterAgent(agentID); unregisterErr != nil { + s.logger.Error().Err(unregisterErr).Str("agent_id", agentID).Msg("Failed to unregister agent after send error") + } + return sendErr + } + + initialRegistration = false + s.logger.Info(). + Str("agent_id", agentID). + Str("ip", identity.IP). + Int("port", identity.Port). + Str("role", identity.Role). + Msg("Agent registration stream established") + } else { + if updateErr := s.registry.UpdateHeartbeat(agentID); updateErr != nil { + s.logger.Error().Err(updateErr).Str("agent_id", agentID).Msg("Failed to update heartbeat") + s.cleanupConnection(agentID) + return updateErr + } + + if agentConn != nil { + agentConn.UpdateActivity() + } + } + } + + s.cleanupConnection(agentID) + return nil +} + +// StreamMetrics handles bi-directional metrics streaming. +func (s *FODCService) StreamMetrics(stream fodcv1.FODCService_StreamMetricsServer) error { + ctx := stream.Context() + + agentID := s.getAgentIDFromContext(ctx) + if agentID == "" { + agentID = s.getAgentIDFromPeer(ctx) + if agentID != "" { + s.logger.Warn(). + Str("agent_id", agentID). + Msg("Agent ID not found in metadata, using peer address fallback (this may be unreliable)") + } + } + + if agentID == "" { + s.logger.Error().Msg("Agent ID not found in context metadata or peer address") + return status.Errorf(codes.Unauthenticated, "agent ID not found in context or peer address") + } + + s.connectionsMu.Lock() + existingConn, exists := s.connections[agentID] + if exists { + existingConn.MetricsStream = stream + existingConn.UpdateActivity() + } else { + agentConn := &AgentConnection{ + AgentID: agentID, + Stream: stream, + MetricsStream: stream, + Context: ctx, + LastActivity: time.Now(), + } + s.connections[agentID] = agentConn + } + s.connectionsMu.Unlock() + + defer s.cleanupConnection(agentID) + + recvCh := make(chan *fodcv1.StreamMetricsRequest, 1) + recvErrCh := make(chan error, 1) + + go func() { + for { + req, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + recvErrCh <- nil + return + } + if recvErr != nil { + recvErrCh <- recvErr + return + } + select { + case recvCh <- req: + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case recvErr := <-recvErrCh: + if recvErr != nil { + if errors.Is(recvErr, context.Canceled) || errors.Is(recvErr, context.DeadlineExceeded) { + s.logger.Debug().Err(recvErr).Str("agent_id", agentID).Msg("Metrics stream closed") + } else if st, ok := status.FromError(recvErr); ok { + code := st.Code() + if code == codes.Canceled || code == codes.DeadlineExceeded { + s.logger.Debug().Err(recvErr).Str("agent_id", agentID).Msg("Metrics stream closed") + } else { + s.logger.Error().Err(recvErr).Str("agent_id", agentID).Msg("Error receiving metrics") + } + } else { + s.logger.Error().Err(recvErr).Str("agent_id", agentID).Msg("Error receiving metrics") + } + return recvErr + } + return nil + case req := <-recvCh: + if req != nil { + s.connectionsMu.Lock() + conn, connExists := s.connections[agentID] + s.connectionsMu.Unlock() + if connExists { + conn.UpdateActivity() + } + + agentInfo, getErr := s.registry.GetAgentByID(agentID) + if getErr != nil { + s.logger.Error().Err(getErr).Str("agent_id", agentID).Msg("Failed to get agent info") + continue + } + + if processErr := s.metricsAggregator.ProcessMetricsFromAgent(ctx, agentID, agentInfo, req); processErr != nil { + s.logger.Error().Err(processErr).Str("agent_id", agentID).Msg("Failed to process metrics") + } + } + } + } +} + +// RequestMetrics requests metrics from an agent via the metrics stream. +func (s *FODCService) RequestMetrics(_ context.Context, agentID string, startTime, endTime *time.Time) error { + s.connectionsMu.RLock() + agentConn, exists := s.connections[agentID] + s.connectionsMu.RUnlock() Review Comment: ```suggestion defer s.connectionsMu.RUnlock() agentConn, exists := s.connections[agentID] ``` ########## fodc/proxy/internal/metrics/aggregator.go: ########## @@ -0,0 +1,303 @@ +// Licensed to Apache Software Foundation (ASF) under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Apache Software Foundation (ASF) licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package metrics provides functionality for aggregating and enriching metrics from all agents. +package metrics + +import ( + "context" + "fmt" + "sync" + "time" + + fodcv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/fodc/v1" + "github.com/apache/skywalking-banyandb/fodc/proxy/internal/registry" + "github.com/apache/skywalking-banyandb/pkg/logger" +) + +const ( + // defaultCollectionTimeout is the default timeout for collecting metrics from agents. + defaultCollectionTimeout = 10 * time.Second + // maxCollectionTimeout is the maximum timeout allowed for collecting metrics, + // preventing excessively long waits for wide time windows. + maxCollectionTimeout = 5 * time.Minute +) + +// AggregatedMetric represents an aggregated metric with node metadata. +type AggregatedMetric struct { + Labels map[string]string + Timestamp time.Time + Name string + AgentID string + NodeRole string + Description string + Value float64 +} + +// Filter defines filters for metrics collection. +type Filter struct { + StartTime *time.Time + EndTime *time.Time + Role string + Address string + AgentIDs []string +} + +// Aggregator aggregates and enriches metrics from all agents. +type Aggregator struct { + registry *registry.AgentRegistry + logger *logger.Logger + grpcService RequestSender + metricsCh chan *AggregatedMetric + collecting map[string]chan []*AggregatedMetric + mu sync.RWMutex + collectingMu sync.RWMutex +} + +// RequestSender is an interface for sending metrics requests to agents. +type RequestSender interface { + RequestMetrics(ctx context.Context, agentID string, startTime, endTime *time.Time) error +} + +// NewAggregator creates a new MetricsAggregator instance. +func NewAggregator(registry *registry.AgentRegistry, grpcService RequestSender, logger *logger.Logger) *Aggregator { + return &Aggregator{ + registry: registry, + grpcService: grpcService, + logger: logger, + metricsCh: make(chan *AggregatedMetric, 1000), + collecting: make(map[string]chan []*AggregatedMetric), + } +} + +// SetGRPCService sets the gRPC service for sending metrics requests. +func (ma *Aggregator) SetGRPCService(grpcService RequestSender) { + ma.mu.Lock() + defer ma.mu.Unlock() + ma.grpcService = grpcService +} + +// ProcessMetricsFromAgent processes metrics received from an agent. +func (ma *Aggregator) ProcessMetricsFromAgent(ctx context.Context, agentID string, agentInfo *registry.AgentInfo, req *fodcv1.StreamMetricsRequest) error { + aggregatedMetrics := make([]*AggregatedMetric, 0, len(req.Metrics)) + + for _, metric := range req.Metrics { + labels := make(map[string]string) + for key, value := range metric.Labels { + labels[key] = value + } + + labels["agent_id"] = agentID + labels["node_role"] = agentInfo.NodeRole + labels["ip"] = agentInfo.PrimaryAddress.IP + labels["port"] = fmt.Sprintf("%d", agentInfo.PrimaryAddress.Port) + + for key, value := range agentInfo.Labels { + labels[key] = value + } + + var timestamp time.Time + switch { + case metric.Timestamp != nil: + timestamp = metric.Timestamp.AsTime() + case req.Timestamp != nil: + timestamp = req.Timestamp.AsTime() + default: + timestamp = time.Now() + } + + aggregatedMetric := &AggregatedMetric{ + Name: metric.Name, + Labels: labels, + Value: metric.Value, + Timestamp: timestamp, + AgentID: agentID, + NodeRole: agentInfo.NodeRole, + Description: metric.Description, + } + + aggregatedMetrics = append(aggregatedMetrics, aggregatedMetric) + } + + ma.collectingMu.RLock() + collectCh, exists := ma.collecting[agentID] + ma.collectingMu.RUnlock() Review Comment: ```suggestion defer ma.collectingMu.RUnlock() collectCh, exists := ma.collecting[agentID] ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
