This is an automated email from the ASF dual-hosted git repository.
Alanxtl pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/dubbo-go.git
The following commit(s) were added to refs/heads/develop by this push:
new d9557d572 feat(metadata): expose observability for metadata source,
storage typ… (#3685)
d9557d572 is described below
commit d9557d57219a3fcdcbabb6191c16ce97ac2410f3
Author: Modo <[email protected]>
AuthorDate: Tue Aug 25 10:25:20 2026 +0800
feat(metadata): expose observability for metadata source, storage typ…
(#3685)
* feat(metadata): expose observability for metadata source, storage type,
revision, and cache hit/miss along the metadata path
* modified HELP information
* fix ci
* feat: restore metadata fetch metrics after listener refactor
* feat: add logs for cache hit/miss
---
metrics/metadata/collector.go | 24 +++++++++
metrics/metadata/metric_set.go | 50 +++++++++++++++++
.../customizer/service_revision_customizer.go | 6 +++
.../servicediscovery/service_discovery_registry.go | 2 +
.../service_instances_changed_listener_impl.go | 62 ++++++++++++++++++----
5 files changed, 134 insertions(+), 10 deletions(-)
diff --git a/metrics/metadata/collector.go b/metrics/metadata/collector.go
index d0d0a9aa8..81dea8d64 100644
--- a/metrics/metadata/collector.go
+++ b/metrics/metadata/collector.go
@@ -66,6 +66,10 @@ func (c *MetadataMetricCollector) start() {
c.handleMetadataMappingListen(event)
case MetadataMappingRemove:
c.handleMetadataMappingRemove(event)
+ case MetadataCache:
+ c.handleMetadataCache(event)
+ case MetadataFetch:
+ c.handleMetadataFetch(event)
default:
}
}
@@ -120,6 +124,26 @@ func (c *MetadataMetricCollector)
handleMetadataMappingRemove(event *MetadataMet
c.R.Rt(metrics.NewMetricId(metadataMappingRemoveRt, level),
&metrics.RtOpts{}).Observe(event.CostMs())
}
+func (c *MetadataMetricCollector) handleMetadataCache(event
*MetadataMetricEvent) {
+ labels := metrics.GetApplicationLevel().Tags()
+ labels[TagProviderApp] = event.Attachment[TagProviderApp]
+ c.R.Counter(metrics.NewMetricIdByLabels(metadataCacheNum, labels)).Inc()
+ if event.Succ {
+ c.R.Counter(metrics.NewMetricIdByLabels(metadataCacheHit,
labels)).Inc()
+ } else {
+ c.R.Counter(metrics.NewMetricIdByLabels(metadataCacheMiss,
labels)).Inc()
+ }
+}
+
+func (c *MetadataMetricCollector) handleMetadataFetch(event
*MetadataMetricEvent) {
+ labels := metrics.GetApplicationLevel().Tags()
+ labels[TagProviderApp] = event.Attachment[TagProviderApp]
+ labels[TagSource] = event.Attachment[TagSource]
+ labels[TagStorageType] = event.Attachment[TagStorageType]
+ labels[TagResult] = event.Attachment[TagResult]
+ c.R.Counter(metrics.NewMetricIdByLabels(metadataFetchNum, labels)).Inc()
+}
+
type metadataMappingMetricLevel struct {
*metrics.ApplicationMetricLevel
attachment map[string]string
diff --git a/metrics/metadata/metric_set.go b/metrics/metadata/metric_set.go
index c760e201c..4f937fcfe 100644
--- a/metrics/metadata/metric_set.go
+++ b/metrics/metadata/metric_set.go
@@ -35,6 +35,8 @@ const (
MetadataMappingGet
MetadataMappingListen
MetadataMappingRemove
+ MetadataCache
+ MetadataFetch
)
const (
@@ -53,6 +55,8 @@ const (
dubboMetadataMappingListenRt =
"dubbo_metadata_mapping_listen_rt_milliseconds"
dubboMetadataMappingRemove = "dubbo_metadata_mapping_remove_num"
dubboMetadataMappingRemoveRt =
"dubbo_metadata_mapping_remove_rt_milliseconds"
+ dubboMetadataCache = "dubbo_metadata_cache"
+ dubboMetadataFetch = "dubbo_metadata_fetch"
)
const (
@@ -116,4 +120,50 @@ var (
metadataMappingRemoveSucceed =
metrics.NewMetricKey(dubboMetadataMappingRemove+succSuffix, "Succeed Metadata
Mapping Remove Num")
metadataMappingRemoveFailed =
metrics.NewMetricKey(dubboMetadataMappingRemove+failedSuffix, "Failed Metadata
Mapping Remove Num")
metadataMappingRemoveRt =
metrics.NewMetricKey(dubboMetadataMappingRemoveRt, "Metadata Mapping Remove
Time")
+
+ /*
+ # HELP dubbo_metadata_cache_total Total Metadata Cache Lookup Num
+ # TYPE dubbo_metadata_cache_total counter
+
dubbo_metadata_cache_total{application_name="metrics-consumer",hostname="localhost",ip="10.252.156.213",provider_app="metrics-provider",}
5.0
+
dubbo_metadata_cache_hit_total{application_name="metrics-consumer",hostname="localhost",ip="10.252.156.213",provider_app="metrics-provider",}
4.0
+
dubbo_metadata_cache_miss_total{application_name="metrics-consumer",hostname="localhost",ip="10.252.156.213",provider_app="metrics-provider",}
1.0
+ */
+ // app level, tagged by provider app
+ metadataCacheNum =
metrics.NewMetricKey(dubboMetadataCache+totalSuffix, "Total Lookups Against the
Metadata LRU Cache")
+ metadataCacheHit =
metrics.NewMetricKey(dubboMetadataCache+"_hit"+totalSuffix, "Successful Lookups
Against the Metadata LRU Cache")
+ metadataCacheMiss =
metrics.NewMetricKey(dubboMetadataCache+"_miss"+totalSuffix, "Failed Lookups
Against the Metadata LRU Cache")
+
+ /*
+ # HELP dubbo_metadata_fetch_total Total Metadata Fetch Num
+ # TYPE dubbo_metadata_fetch_total counter
+
dubbo_metadata_fetch_total{application_name="metrics-consumer",hostname="localhost",ip="10.252.156.213",provider_app="metrics-provider",result="success",source="report",storage_type="remote",}
1.0
+ */
+ // app level, tagged by provider app, fetch source, storage type and
result
+ metadataFetchNum = metrics.NewMetricKey(dubboMetadataFetch+totalSuffix,
"Total Metadata Fetch Num")
+)
+
+const (
+ TagProviderApp = "provider_app"
+ TagSource = "source"
+ TagStorageType = "storage_type"
+ TagResult = "result"
+)
+
+// Metadata fetch source values
+const (
+ SourceCache = "cache"
+ SourceReport = "report"
+ SourceRpc = "rpc"
+)
+
+// Metadata storage type values
+const (
+ StorageTypeLocal = "local"
+ StorageTypeRemote = "remote"
+)
+
+// Metadata fetch result values
+const (
+ ResultSuccess = "success"
+ ResultFailure = "failure"
)
diff --git
a/registry/servicediscovery/customizer/service_revision_customizer.go
b/registry/servicediscovery/customizer/service_revision_customizer.go
index 158df9c4a..8d2af2178 100644
--- a/registry/servicediscovery/customizer/service_revision_customizer.go
+++ b/registry/servicediscovery/customizer/service_revision_customizer.go
@@ -56,7 +56,9 @@ func (e *exportedServicesRevisionMetadataCustomizer)
Customize(instance registry
}
metaInfo := metadata.GetMetadataInfo(registryId)
var urls []*common.URL
+ app := ""
if metaInfo != nil {
+ app = metaInfo.App
urls = metaInfo.GetExportedServiceURLs()
}
revision := resolveRevision(urls)
@@ -64,6 +66,7 @@ func (e *exportedServicesRevisionMetadataCustomizer)
Customize(instance registry
revision = defaultRevision
}
instance.GetMetadata()[constant.ExportedServicesRevisionPropertyName] =
revision
+ logger.Infof("[Metadata][Revision] calculated %s revision, app=%s
registryId=%s urls=%d revision=%s", "exported", app, registryId, len(urls),
revision)
}
// subscribedServicesRevisionMetadataCustomizer writes a revision derived from
@@ -84,7 +87,9 @@ func (e *subscribedServicesRevisionMetadataCustomizer)
Customize(instance regist
}
metaInfo := metadata.GetMetadataInfo(registryId)
var urls []*common.URL
+ app := ""
if metaInfo != nil {
+ app = metaInfo.App
urls = metaInfo.GetSubscribedURLs()
}
revision := resolveRevision(urls)
@@ -92,6 +97,7 @@ func (e *subscribedServicesRevisionMetadataCustomizer)
Customize(instance regist
revision = defaultRevision
}
instance.GetMetadata()[constant.SubscribedServicesRevisionPropertyName]
= revision
+ logger.Infof("[Metadata][Revision] calculated %s revision, app=%s
registryId=%s urls=%d revision=%s", "subscribed", app, registryId, len(urls),
revision)
}
// resolveRevision calculates a deterministic revision from the given URLs.
diff --git a/registry/servicediscovery/service_discovery_registry.go
b/registry/servicediscovery/service_discovery_registry.go
index d19e5ef89..ebba18e4a 100644
--- a/registry/servicediscovery/service_discovery_registry.go
+++ b/registry/servicediscovery/service_discovery_registry.go
@@ -151,6 +151,8 @@ func (s *serviceDiscoveryRegistry) RegisterService() error {
if err := s.metadataReport.PublishAppMetadata(metaInfo.App,
metaInfo.Revision, metaInfo); err != nil {
return err
}
+ logger.Infof("[Metadata][Publish] published app metadata,
app=%s revision=%s urls=%d",
+ metaInfo.App, metaInfo.Revision, len(urls))
}
for _, instance := range instances {
diff --git
a/registry/servicediscovery/service_instances_changed_listener_impl.go
b/registry/servicediscovery/service_instances_changed_listener_impl.go
index ae5916f16..e6275102f 100644
--- a/registry/servicediscovery/service_instances_changed_listener_impl.go
+++ b/registry/servicediscovery/service_instances_changed_listener_impl.go
@@ -40,6 +40,8 @@ import (
"dubbo.apache.org/dubbo-go/v3/common/constant"
"dubbo.apache.org/dubbo-go/v3/metadata"
"dubbo.apache.org/dubbo-go/v3/metadata/info"
+ "dubbo.apache.org/dubbo-go/v3/metrics"
+ metricsMetadata "dubbo.apache.org/dubbo-go/v3/metrics/metadata"
"dubbo.apache.org/dubbo-go/v3/registry"
"dubbo.apache.org/dubbo-go/v3/registry/servicediscovery/store"
"dubbo.apache.org/dubbo-go/v3/remoting"
@@ -384,23 +386,61 @@ func GetMetadataInfoWithContext(ctx context.Context, app
string, instance regist
})
cacheKey := metadataCacheKey(app, registryId, revision)
if metadataInfo, ok := metaCache.Get(cacheKey); ok {
+ logger.Debugf("[Metadata][Cache] app=%s registry=%s revision=%s
host=%s result=hit",
+ app, registryId, revision, instance.GetHost())
+ publishMetadataCacheEvent(app, true)
+ publishMetadataFetchEvent(app, metricsMetadata.SourceCache, "",
nil)
return metadataInfo.(*info.MetadataInfo), nil
}
+ logger.Debugf("[Metadata][Cache] app=%s registry=%s revision=%s host=%s
result=miss",
+ app, registryId, revision, instance.GetHost())
+ publishMetadataCacheEvent(app, false)
var metadataInfo *info.MetadataInfo
+ var metricStorageType string
+ var storageType string
+ var source string
var err error
- if getMetadataStorageType(instance) ==
constant.RemoteMetadataStorageType {
- metadataInfo, err = getRemoteMetadataInfo(ctx, app, instance,
revision, registryId)
+
+ storageType = getMetadataStorageType(instance)
+ metricStorageType = metricsMetadata.StorageTypeLocal
+
+ if storageType == constant.RemoteMetadataStorageType {
+ metricStorageType = metricsMetadata.StorageTypeRemote
+ metadataInfo, source, err = getRemoteMetadataInfo(ctx, app,
instance, revision, registryId)
} else {
- metadataInfo, err = getMetadataInfoFromRPC(ctx, app, instance,
revision, registryId)
+ metadataInfo, source, err = getMetadataInfoFromRPC(ctx, app,
instance, revision, registryId)
}
if err != nil {
+ publishMetadataFetchEvent(app, source, metricStorageType, err)
return nil, err
}
metaCache.Set(cacheKey, metadataInfo)
+ publishMetadataFetchEvent(app, source, metricStorageType, nil)
return metadataInfo, nil
}
+func publishMetadataCacheEvent(app string, hit bool) {
+ event :=
metricsMetadata.NewMetadataMetricTimeEvent(metricsMetadata.MetadataCache)
+ event.Succ = hit
+ event.Attachment[metricsMetadata.TagProviderApp] = app
+ metrics.Publish(event)
+}
+
+func publishMetadataFetchEvent(app, source, storageType string, err error) {
+ event :=
metricsMetadata.NewMetadataMetricTimeEvent(metricsMetadata.MetadataFetch)
+ event.Succ = err == nil
+ event.Attachment[metricsMetadata.TagProviderApp] = app
+ event.Attachment[metricsMetadata.TagSource] = source
+ event.Attachment[metricsMetadata.TagStorageType] = storageType
+ if err == nil {
+ event.Attachment[metricsMetadata.TagResult] =
metricsMetadata.ResultSuccess
+ } else {
+ event.Attachment[metricsMetadata.TagResult] =
metricsMetadata.ResultFailure
+ }
+ metrics.Publish(event)
+}
+
var (
// metadataRetryInitialDelay is the first backoff delay before retrying
a failed
// metadata fetch. Package-level so tests can shrink it.
@@ -509,27 +549,29 @@ func getMetadataStorageType(instance
registry.ServiceInstance) string {
return storageType
}
-func getMetadataInfoFromRPC(ctx context.Context, app string, instance
registry.ServiceInstance, revision string, registryId string)
(*info.MetadataInfo, error) {
+func getMetadataInfoFromRPC(ctx context.Context, app string, instance
registry.ServiceInstance, revision string, registryId string)
(*info.MetadataInfo, string, error) {
metadataInfo, err := metadata.GetMetadataFromRpcWithContext(ctx,
revision, instance)
if err != nil {
- return nil, perrors.Wrapf(err,
+ return nil, metricsMetadata.SourceRpc, perrors.Wrapf(err,
"failed app=%s registry=%s revision=%s", app,
registryId, revision)
}
- return requireMetadataInfo(metadataInfo, app, registryId, revision)
+ metadataInfo, err = requireMetadataInfo(metadataInfo, app, registryId,
revision)
+ return metadataInfo, metricsMetadata.SourceRpc, err
}
-func getRemoteMetadataInfo(ctx context.Context, app string, instance
registry.ServiceInstance, revision string, registryId string)
(*info.MetadataInfo, error) {
+func getRemoteMetadataInfo(ctx context.Context, app string, instance
registry.ServiceInstance, revision string, registryId string)
(*info.MetadataInfo, string, error) {
metadataInfo, reportErr :=
metadata.GetMetadataFromMetadataReport(revision, instance, registryId)
if reportErr == nil && metadataInfo != nil {
- return metadataInfo, nil
+ return metadataInfo, metricsMetadata.SourceReport, nil
}
logMetadataReportFallback(app, registryId, revision, reportErr)
metadataInfo, rpcErr := metadata.GetMetadataFromRpcWithContext(ctx,
revision, instance)
if rpcErr != nil {
- return nil, wrapMetadataRPCFallbackError(rpcErr, reportErr)
+ return nil, metricsMetadata.SourceRpc,
wrapMetadataRPCFallbackError(rpcErr, reportErr)
}
- return requireMetadataInfo(metadataInfo, app, registryId, revision)
+ metadataInfo, rpcErr = requireMetadataInfo(metadataInfo, app,
registryId, revision)
+ return metadataInfo, metricsMetadata.SourceRpc, rpcErr
}
func logMetadataReportFallback(app, registryId, revision string, reportErr
error) {
