This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new cb4008f6407 branch-4.1:[fix](metric) Preserve labels for histogram
metrics to fix wrong metric name for prometheus (#63485) (#63716)
cb4008f6407 is described below
commit cb4008f6407aebbab99b5dce49da897cc23902b1
Author: seawinde <[email protected]>
AuthorDate: Wed Jun 3 14:36:36 2026 +0800
branch-4.1:[fix](metric) Preserve labels for histogram metrics to fix wrong
metric name for prometheus (#63485) (#63716)
pr: #63485
commitId: 01241a91577
Co-authored-by: Copilot <[email protected]>
---
.../java/org/apache/doris/metric/CloudMetrics.java | 26 +++--
.../apache/doris/metric/DorisMetricRegistry.java | 36 +++++++
.../{MetricVisitor.java => HistogramMetric.java} | 47 +++++----
.../org/apache/doris/metric/JsonMetricVisitor.java | 18 ++++
.../java/org/apache/doris/metric/MetricRepo.java | 27 ++++--
.../org/apache/doris/metric/MetricVisitor.java | 4 +
.../doris/metric/PrometheusMetricVisitor.java | 24 +++--
.../doris/metric/SimpleCoreMetricVisitor.java | 6 ++
.../java/org/apache/doris/metric/MetricsTest.java | 105 ++++++++++++++++-----
9 files changed, 220 insertions(+), 73 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java
index 816bf6964e7..a14f988cf63 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java
@@ -20,9 +20,12 @@ package org.apache.doris.metric;
import org.apache.doris.common.Config;
import org.apache.doris.metric.Metric.MetricUnit;
-import com.codahale.metrics.Histogram;
import com.codahale.metrics.MetricRegistry;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
public class CloudMetrics {
protected static String CLOUD_CLUSTER_DELIMITER = "@delimiter#";
protected static AutoMappedMetric<LongCounterMetric>
CLUSTER_REQUEST_ALL_COUNTER;
@@ -33,7 +36,7 @@ public class CloudMetrics {
protected static AutoMappedMetric<GaugeMetricImpl<Double>>
CLUSTER_QUERY_PER_SECOND_GAUGE;
protected static AutoMappedMetric<GaugeMetricImpl<Double>>
CLUSTER_QUERY_ERR_RATE_GAUGE;
- protected static AutoMappedMetric<Histogram> CLUSTER_QUERY_LATENCY_HISTO;
+ protected static AutoMappedMetric<HistogramMetric>
CLUSTER_QUERY_LATENCY_HISTO;
protected static AutoMappedMetric<GaugeMetricImpl<Integer>>
CLUSTER_BACKEND_ALIVE;
protected static AutoMappedMetric<GaugeMetricImpl<Integer>>
CLUSTER_BACKEND_ALIVE_TOTAL;
@@ -50,7 +53,7 @@ public class CloudMetrics {
public static AutoMappedMetric<LongCounterMetric> META_SERVICE_RPC_FAILED;
public static AutoMappedMetric<LongCounterMetric> META_SERVICE_RPC_RETRY;
public static AutoMappedMetric<GaugeMetricImpl<Double>>
META_SERVICE_RPC_PER_SECOND;
- public static AutoMappedMetric<Histogram> META_SERVICE_RPC_LATENCY;
+ public static AutoMappedMetric<HistogramMetric> META_SERVICE_RPC_LATENCY;
// Aggregate meta-service metrics
public static LongCounterMetric META_SERVICE_RPC_ALL_TOTAL;
@@ -93,13 +96,15 @@ public class CloudMetrics {
MetricUnit.NOUNIT, "backend alive num in cluster", 0));
CLUSTER_QUERY_LATENCY_HISTO = new AutoMappedMetric<>(key -> {
- String[] values = key.split(CLOUD_CLUSTER_DELIMITER);
+ String[] values = key.split(CLOUD_CLUSTER_DELIMITER, 2);
String clusterId = values[0];
String clusterName = values[1];
- String metricName = MetricRegistry.name("query", "latency", "ms",
"cluster_id="
- + clusterId, "cluster_name=" + clusterName);
- return MetricRepo.METRIC_REGISTER.histogram(metricName);
+ List<MetricLabel> labels = Arrays.asList(new
MetricLabel("cluster_id", clusterId),
+ new MetricLabel("cluster_name", clusterName));
+ return new HistogramMetric(MetricRegistry.name("query", "latency",
"ms"), labels);
});
+ MetricRepo.DORIS_METRIC_REGISTER.addHistogramMetrics(
+ "cloud_cluster_query_latency", CLUSTER_QUERY_LATENCY_HISTO,
Config::isCloudMode);
CLUSTER_WARM_UP_JOB_EXEC_COUNT = new AutoMappedMetric<>(name -> new
LongCounterMetric(
"file_cache_warm_up_job_exec_count", MetricUnit.NOUNIT, "warm
up job execution count"));
@@ -135,10 +140,11 @@ public class CloudMetrics {
return gauge;
});
META_SERVICE_RPC_LATENCY = new AutoMappedMetric<>(methodName -> {
- String metricName = MetricRegistry.name("meta_service", "rpc",
"latency", "ms",
- "method=" + methodName);
- return MetricRepo.METRIC_REGISTER.histogram(metricName);
+ List<MetricLabel> labels = Collections.singletonList(new
MetricLabel("method", methodName));
+ return new HistogramMetric(MetricRegistry.name("meta_service",
"rpc", "latency", "ms"), labels);
});
+ MetricRepo.DORIS_METRIC_REGISTER.addHistogramMetrics(
+ "meta_service_rpc_latency", META_SERVICE_RPC_LATENCY,
Config::isCloudMode);
// Aggregate meta-service metrics
META_SERVICE_RPC_ALL_TOTAL = new
LongCounterMetric("meta_service_rpc_all_total",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
index 21e5e366ebc..5627767f277 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
@@ -27,11 +27,13 @@ import java.util.List;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
import java.util.stream.Collectors;
public class DorisMetricRegistry {
ConcurrentHashMap<String, MetricList> metrics = new ConcurrentHashMap<>();
ConcurrentHashMap<String, MetricList> systemMetrics = new
ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, HistogramMetricGroup>
histogramMetricGroups = new ConcurrentHashMap<>();
public DorisMetricRegistry() {
@@ -56,6 +58,18 @@ public class DorisMetricRegistry {
}
}
+ public void addHistogramMetrics(String name,
AutoMappedMetric<HistogramMetric> histogramMetrics) {
+ addHistogramMetrics(name, histogramMetrics, () -> true);
+ }
+
+ public void addHistogramMetrics(String name,
AutoMappedMetric<HistogramMetric> histogramMetrics,
+ Supplier<Boolean> enabled) {
+ // Same reason as comment in addMetrics()
+ if (!Env.isCheckpointThread()) {
+ histogramMetricGroups.put(name, new
HistogramMetricGroup(histogramMetrics, enabled));
+ }
+ }
+
public void accept(MetricVisitor visitor) {
final List<MetricList> metricsList = Lists.newArrayList();
metrics.forEach((name, list) -> metricsList.add(list));
@@ -73,6 +87,18 @@ public class DorisMetricRegistry {
}
}
+ public void acceptHistograms(MetricVisitor visitor) {
+ for (HistogramMetricGroup group : histogramMetricGroups.values()) {
+ if (!group.enabled.get()) {
+ continue;
+ }
+ for (HistogramMetric metric :
group.histogramMetrics.getMetrics().values()) {
+ visitor.visitHistogram(MetricVisitor.FE_PREFIX,
+ metric.getName(), metric.getHistogram(),
metric.getLabels());
+ }
+ }
+ }
+
// the metrics by metric name
public List<Metric> getMetricsByName(String name) {
MetricList list = metrics.get(name);
@@ -132,4 +158,14 @@ public class DorisMetricRegistry {
metrics.remove(labelId);
}
}
+
+ private static class HistogramMetricGroup {
+ private final AutoMappedMetric<HistogramMetric> histogramMetrics;
+ private final Supplier<Boolean> enabled;
+
+ private HistogramMetricGroup(AutoMappedMetric<HistogramMetric>
histogramMetrics, Supplier<Boolean> enabled) {
+ this.histogramMetrics = histogramMetrics;
+ this.enabled = enabled;
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/HistogramMetric.java
similarity index 51%
copy from fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
copy to fe/fe-core/src/main/java/org/apache/doris/metric/HistogramMetric.java
index 32f368a2270..ed8349f2eb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/HistogramMetric.java
@@ -17,38 +17,37 @@
package org.apache.doris.metric;
-import org.apache.doris.monitor.jvm.JvmStats;
-
+import com.codahale.metrics.ExponentiallyDecayingReservoir;
import com.codahale.metrics.Histogram;
-/*
- * MetricVisitor will visit the metrics in metric repo and print them in
StringBuilder
- */
-public abstract class MetricVisitor {
-
- // for FE metrics
- public static final String FE_PREFIX = "doris_fe_";
- // for system metrics
- public static final String SYS_PREFIX = "system_";
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
- protected StringBuilder sb = new StringBuilder();
+public class HistogramMetric {
+ private final String name;
+ private final Histogram histogram;
+ private final List<MetricLabel> labels;
- public MetricVisitor() {
+ public HistogramMetric(String name, List<MetricLabel> labels) {
+ this.name = name;
+ this.histogram = new Histogram(new ExponentiallyDecayingReservoir());
+ this.labels = Collections.unmodifiableList(new ArrayList<>(labels));
}
- public abstract void visitJvm(JvmStats jvmStats);
-
- public abstract void visit(String prefix, Metric metric);
-
- public abstract void visitHistogram(String prefix, String name, Histogram
histogram);
-
- public abstract void visitNodeInfo();
+ public void update(long value) {
+ histogram.update(value);
+ }
- public abstract void visitCloudTableStats();
+ public String getName() {
+ return name;
+ }
- public abstract void visitWorkloadGroup();
+ public Histogram getHistogram() {
+ return histogram;
+ }
- public String finish() {
- return sb.toString();
+ public List<MetricLabel> getLabels() {
+ return labels;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/JsonMetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/JsonMetricVisitor.java
index 0364781cd3b..01652bc5cc8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/JsonMetricVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/JsonMetricVisitor.java
@@ -160,6 +160,24 @@ public class JsonMetricVisitor extends MetricVisitor {
}
}
final String fullName = prefix + String.join("_", names);
+ appendHistogram(fullName, histogram, tags);
+ }
+
+ @Override
+ public void visitHistogram(String prefix, String name, Histogram
histogram, List<MetricLabel> labels) {
+ if (histogramOrdinal++ == 0) {
+ sb.append(",\n");
+ }
+
+ final String fullName = prefix + name.replace('.', '_');
+ List<String> tags = new ArrayList<>();
+ for (MetricLabel label : labels) {
+ tags.add(String.format("\"%s\":\"%s\"", label.getKey(),
label.getValue()));
+ }
+ appendHistogram(fullName, histogram, tags);
+ }
+
+ private void appendHistogram(String fullName, Histogram histogram,
List<String> tags) {
Snapshot snapshot = histogram.getSnapshot();
setHistogramJsonMetric(sb, fullName, "\"quantile\":\"0.75\"", tags,
snapshot.get75thPercentile());
setHistogramJsonMetric(sb, fullName, "\"quantile\":\"0.95\"", tags,
snapshot.get95thPercentile());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index a55214fb061..cacb8e5adb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -121,7 +121,7 @@ public final class MetricRepo {
public static AutoMappedMetric<LongCounterMetric> USER_COUNTER_QUERY_ALL;
public static AutoMappedMetric<LongCounterMetric> USER_COUNTER_QUERY_ERR;
public static Histogram HISTO_QUERY_LATENCY;
- public static AutoMappedMetric<Histogram> USER_HISTO_QUERY_LATENCY;
+ public static AutoMappedMetric<HistogramMetric> USER_HISTO_QUERY_LATENCY;
public static AutoMappedMetric<GaugeMetricImpl<Long>>
USER_GAUGE_QUERY_INSTANCE_NUM;
public static AutoMappedMetric<GaugeMetricImpl<Integer>>
USER_GAUGE_CONNECTIONS;
public static AutoMappedMetric<LongCounterMetric>
USER_COUNTER_QUERY_INSTANCE_BEGIN;
@@ -511,9 +511,10 @@ public final class MetricRepo {
HISTO_QUERY_LATENCY = METRIC_REGISTER.histogram(
MetricRegistry.name("query", "latency", "ms"));
USER_HISTO_QUERY_LATENCY = new AutoMappedMetric<>(name -> {
- String metricName = MetricRegistry.name("query", "latency", "ms",
"user=" + name);
- return METRIC_REGISTER.histogram(metricName);
+ List<MetricLabel> labels = Collections.singletonList(new
MetricLabel("user", name));
+ return new HistogramMetric(MetricRegistry.name("query", "latency",
"ms"), labels);
});
+ DORIS_METRIC_REGISTER.addHistogramMetrics("user_query_latency",
USER_HISTO_QUERY_LATENCY);
USER_COUNTER_QUERY_INSTANCE_BEGIN = addLabeledMetrics("user", () ->
new LongCounterMetric("query_instance_begin",
MetricUnit.NOUNIT,
"number of query instance begin"));
@@ -1566,10 +1567,7 @@ public final class MetricRepo {
DORIS_METRIC_REGISTER.accept(visitor);
// histogram
- SortedMap<String, Histogram> histograms =
METRIC_REGISTER.getHistograms();
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
- }
+ visitHistograms(visitor);
visitor.visitNodeInfo();
@@ -1589,6 +1587,15 @@ public final class MetricRepo {
});
}
+ public static void visitHistograms(MetricVisitor visitor) {
+ SortedMap<String, Histogram> histograms =
METRIC_REGISTER.getHistograms();
+ for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
+ visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
+ }
+
+ DORIS_METRIC_REGISTER.acceptHistograms(visitor);
+ }
+
// update some metrics to make a ready to be visited
private static void updateMetrics() {
SYSTEM_METRICS.update();
@@ -1957,9 +1964,9 @@ public final class MetricRepo {
CloudMetrics.CLUSTER_QUERY_ERR_RATE_GAUGE.remove(clusterId);
DORIS_METRIC_REGISTER.removeMetricsByNameAndLabels(queryErrRateGauge.getName(),
labels);
- METRIC_REGISTER.getHistograms().keySet().stream()
- .filter(k -> k.contains(clusterId))
- .forEach(METRIC_REGISTER::remove);
+ CloudMetrics.CLUSTER_QUERY_LATENCY_HISTO.remove(clusterId +
CloudMetrics.CLOUD_CLUSTER_DELIMITER
+ + clusterName);
+ // Meta-service RPC latency is keyed by method name only, so it is
not removed by cluster.
for (Backend backend : backends) {
List<MetricLabel> backendLabels = new ArrayList<>();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
index 32f368a2270..eb8c2741e13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricVisitor.java
@@ -21,6 +21,8 @@ import org.apache.doris.monitor.jvm.JvmStats;
import com.codahale.metrics.Histogram;
+import java.util.List;
+
/*
* MetricVisitor will visit the metrics in metric repo and print them in
StringBuilder
*/
@@ -42,6 +44,8 @@ public abstract class MetricVisitor {
public abstract void visitHistogram(String prefix, String name, Histogram
histogram);
+ public abstract void visitHistogram(String prefix, String name, Histogram
histogram, List<MetricLabel> labels);
+
public abstract void visitNodeInfo();
public abstract void visitCloudTableStats();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
index 2103dcb6c21..e32097f54df 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/metric/PrometheusMetricVisitor.java
@@ -188,8 +188,20 @@ public class PrometheusMetricVisitor extends MetricVisitor
{
}
}
final String fullName = prefix + String.join("_", names);
+ appendHistogram(fullName, histogram, tags);
+ }
+
+ @Override
+ public void visitHistogram(String prefix, String name, Histogram
histogram, List<MetricLabel> labels) {
+ final String fullName = prefix + name.replace('.', '_');
+ List<String> tags = labels.stream()
+ .map(l -> l.getKey() + "=\"" + l.getValue() + "\"")
+ .collect(Collectors.toList());
+ appendHistogram(fullName, histogram, tags);
+ }
+
+ private void appendHistogram(String fullName, Histogram histogram,
List<String> tags) {
final String fullTag = String.join(",", tags);
- // we should define metric name only once
if (!metricNames.contains(fullName)) {
sb.append(HELP).append(fullName).append(" ").append("\n");
sb.append(TYPE).append(fullName).append(" ").append("summary\n");
@@ -198,15 +210,15 @@ public class PrometheusMetricVisitor extends
MetricVisitor {
String delimiter = tags.isEmpty() ? "" : ",";
Snapshot snapshot = histogram.getSnapshot();
sb.append(fullName).append("{quantile=\"0.75\"").append(delimiter).append(fullTag).append("}
")
- .append(snapshot.get75thPercentile()).append("\n");
+ .append(snapshot.get75thPercentile()).append("\n");
sb.append(fullName).append("{quantile=\"0.95\"").append(delimiter).append(fullTag).append("}
")
- .append(snapshot.get95thPercentile()).append("\n");
+ .append(snapshot.get95thPercentile()).append("\n");
sb.append(fullName).append("{quantile=\"0.98\"").append(delimiter).append(fullTag).append("}
")
- .append(snapshot.get98thPercentile()).append("\n");
+ .append(snapshot.get98thPercentile()).append("\n");
sb.append(fullName).append("{quantile=\"0.99\"").append(delimiter).append(fullTag).append("}
")
- .append(snapshot.get99thPercentile()).append("\n");
+ .append(snapshot.get99thPercentile()).append("\n");
sb.append(fullName).append("{quantile=\"0.999\"").append(delimiter).append(fullTag).append("}
")
- .append(snapshot.get999thPercentile()).append("\n");
+ .append(snapshot.get999thPercentile()).append("\n");
sb.append(fullName).append("_sum{").append(fullTag).append("} ")
.append(histogram.getCount() *
snapshot.getMean()).append("\n");
sb.append(fullName).append("_count{").append(fullTag).append("} ")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/metric/SimpleCoreMetricVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/SimpleCoreMetricVisitor.java
index 06a2b8cc1a7..ba729fefdcb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/metric/SimpleCoreMetricVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/metric/SimpleCoreMetricVisitor.java
@@ -31,6 +31,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
/*
@@ -128,6 +129,11 @@ public class SimpleCoreMetricVisitor extends MetricVisitor
{
String.format("%.0f",
snapshot.get99thPercentile()))).append("\n");
}
+ @Override
+ public void visitHistogram(String prefix, String name, Histogram
histogram, List<MetricLabel> labels) {
+ visitHistogram(prefix, name, histogram);
+ }
+
@Override
public void visitNodeInfo() {
long feDeadNum =
Env.getCurrentEnv().getFrontends(null).stream().filter(f ->
!f.isAlive()).count();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
index 48857bd6d4c..b9994bb5706 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
@@ -23,8 +23,8 @@ import org.apache.doris.metric.Metric.MetricUnit;
import org.apache.doris.monitor.jvm.JvmService;
import org.apache.doris.monitor.jvm.JvmStats;
-import com.codahale.metrics.Histogram;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -33,8 +33,6 @@ import org.junit.Test;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
@@ -72,12 +70,10 @@ public class MetricsTest {
MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd("test_user").increase(1L);
MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd("test_user").increase(1L);
MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd("test_user").update(10L);
+
MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd("[email protected]").update(20L);
MetricVisitor visitor = new PrometheusMetricVisitor();
MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
- SortedMap<String, Histogram> histograms =
MetricRepo.METRIC_REGISTER.getHistograms();
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
- }
+ MetricRepo.visitHistograms(visitor);
String metricResult = visitor.finish();
Assert.assertTrue(metricResult.contains("# TYPE doris_fe_query_total
counter"));
Assert.assertTrue(metricResult.contains("doris_fe_query_total{user=\"test_user\"}
1"));
@@ -86,7 +82,82 @@ public class MetricsTest {
Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_query_latency_ms summary"));
Assert.assertTrue(metricResult.contains("doris_fe_query_latency_ms{quantile=\"0.999\"}
0.0"));
Assert.assertTrue(metricResult.contains("doris_fe_query_latency_ms{quantile=\"0.999\",user=\"test_user\"}
10.0"));
+ Assert.assertTrue(metricResult.contains(
+
"doris_fe_query_latency_ms{quantile=\"0.999\",user=\"[email protected]\"} 20.0"));
+
Assert.assertFalse(metricResult.contains("doris_fe_query_latency_ms_lu@lbk_one"));
+
+ }
+
+ @Test
+ public void
testPrometheusVisitorKeepsLabeledHistogramValuesOutOfMetricName() {
+ HistogramMetric histogramMetric = new
HistogramMetric("query.latency.ms",
+ Lists.newArrayList(new MetricLabel("user",
"[email protected]")));
+ histogramMetric.update(30L);
+ MetricVisitor prometheusVisitor = new PrometheusMetricVisitor();
+ prometheusVisitor.visitHistogram(MetricVisitor.FE_PREFIX,
histogramMetric.getName(),
+ histogramMetric.getHistogram(), histogramMetric.getLabels());
+ String prometheusResult = prometheusVisitor.finish();
+ Assert.assertTrue(prometheusResult.contains(
+
"doris_fe_query_latency_ms{quantile=\"0.999\",user=\"[email protected]\"}
30.0"));
+
Assert.assertFalse(prometheusResult.contains("doris_fe_query_latency_ms_liu@developertools_com"));
+ Assert.assertFalse(prometheusResult.contains("user=\"thomas\""));
+ }
+ @Test
+ public void testJsonVisitorKeepsLabeledHistogramValuesOutOfMetricName() {
+ HistogramMetric histogramMetric = new
HistogramMetric("query.latency.ms",
+ Lists.newArrayList(new MetricLabel("user",
"[email protected]")));
+ histogramMetric.update(20L);
+ MetricVisitor jsonVisitor = new JsonMetricVisitor();
+ jsonVisitor.visitHistogram(MetricVisitor.FE_PREFIX,
histogramMetric.getName(),
+ histogramMetric.getHistogram(), histogramMetric.getLabels());
+ String jsonResult = jsonVisitor.finish();
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_query_latency_ms\""));
+ Assert.assertTrue(jsonResult.contains("\"user\":\"[email protected]\""));
+
Assert.assertFalse(jsonResult.contains("\"metric\":\"doris_fe_query_latency_ms_lu@lbk_one\""));
+ }
+
+ @Test
+ public void testHistogramMetricRegistryWithSpecialCharacters() {
+ DorisMetricRegistry registry = new DorisMetricRegistry();
+ AutoMappedMetric<HistogramMetric> clusterHisto = new
AutoMappedMetric<>(key -> {
+ String[] values = key.split(CloudMetrics.CLOUD_CLUSTER_DELIMITER,
2);
+ return new HistogramMetric("query.latency.ms", Lists.newArrayList(
+ new MetricLabel("cluster_id", values[0]), new
MetricLabel("cluster_name", values[1])));
+ });
+ AutoMappedMetric<HistogramMetric> metaHisto = new
AutoMappedMetric<>(methodName ->
+ new HistogramMetric("meta_service.rpc.latency.ms",
+ Lists.newArrayList(new MetricLabel("method",
methodName))));
+ AutoMappedMetric<HistogramMetric> disabledHisto = new
AutoMappedMetric<>(name ->
+ new HistogramMetric("disabled.latency.ms",
+ Lists.newArrayList(new MetricLabel("name", name))));
+ AutoMappedMetric<HistogramMetric> staleHisto = new
AutoMappedMetric<>(name ->
+ new HistogramMetric("stale.latency.ms",
+ Lists.newArrayList(new MetricLabel("name", name))));
+
+ registry.addHistogramMetrics("cluster_query_latency", staleHisto);
+ registry.addHistogramMetrics("cluster_query_latency", clusterHisto);
+ registry.addHistogramMetrics("meta_service_rpc_latency", metaHisto);
+ registry.addHistogramMetrics("disabled_latency", disabledHisto, () ->
false);
+
+ String clusterKey = "cluster.id-1" +
CloudMetrics.CLOUD_CLUSTER_DELIMITER + "cluster.name@prod";
+ staleHisto.getOrAdd("stale.name").update(30L);
+ clusterHisto.getOrAdd(clusterKey).update(40L);
+ metaHisto.getOrAdd("get.Instance").update(50L);
+ disabledHisto.getOrAdd("disabled.name").update(60L);
+
+ MetricVisitor prometheusVisitor = new PrometheusMetricVisitor();
+ registry.acceptHistograms(prometheusVisitor);
+ String prometheusResult = prometheusVisitor.finish();
+ Assert.assertTrue(prometheusResult.contains(
+
"doris_fe_query_latency_ms{quantile=\"0.999\",cluster_id=\"cluster.id-1\","
+ + "cluster_name=\"cluster.name@prod\"} 40.0"));
+ Assert.assertTrue(prometheusResult.contains(
+
"doris_fe_meta_service_rpc_latency_ms{quantile=\"0.999\",method=\"get.Instance\"}
50.0"));
+
Assert.assertFalse(prometheusResult.contains("doris_fe_query_latency_ms_id-1_cluster"));
+
Assert.assertFalse(prometheusResult.contains("doris_fe_meta_service_rpc_latency_ms_Instance"));
+
Assert.assertFalse(prometheusResult.contains("doris_fe_stale_latency_ms"));
+
Assert.assertFalse(prometheusResult.contains("doris_fe_disabled_latency_ms"));
}
@Test
@@ -173,10 +244,7 @@ public class MetricsTest {
// doris metrics and system metrics.
MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
// histogram
- SortedMap<String, Histogram> histograms =
MetricRepo.METRIC_REGISTER.getHistograms();
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
- }
+ MetricRepo.visitHistograms(visitor);
String metricResult = visitor.finish();
Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_async_materialized_view_task_failed_num counter"));
@@ -196,10 +264,7 @@ public class MetricsTest {
// doris metrics and system metrics.
MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
// histogram
- SortedMap<String, Histogram> histograms =
MetricRepo.METRIC_REGISTER.getHistograms();
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
- }
+ MetricRepo.visitHistograms(visitor);
String metricResult = visitor.finish();
Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_statistics_succeed_analyze_job counter"));
@@ -226,10 +291,7 @@ public class MetricsTest {
// doris metrics and system metrics.
MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
// histogram
- SortedMap<String, Histogram> histograms =
MetricRepo.METRIC_REGISTER.getHistograms();
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
- }
+ MetricRepo.visitHistograms(visitor);
String metricResult = visitor.finish();
Assert.assertTrue(metricResult.contains("# TYPE doris_fe_sql_cache_num
gauge"));
@@ -245,10 +307,7 @@ public class MetricsTest {
// doris metrics and system metrics.
MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
// histogram
- SortedMap<String, Histogram> histograms =
MetricRepo.METRIC_REGISTER.getHistograms();
- for (Map.Entry<String, Histogram> entry : histograms.entrySet()) {
- visitor.visitHistogram(MetricVisitor.FE_PREFIX, entry.getKey(),
entry.getValue());
- }
+ MetricRepo.visitHistograms(visitor);
String metricResult = visitor.finish();
Assert.assertTrue(metricResult.contains("# TYPE doris_fe_plan_num
gauge"));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
