This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new c41d316114f [feature](metrics) add catalog/db/table num metrics
(#47891) (#47998)
c41d316114f is described below
commit c41d316114f25e610d27499d9b2984069e03055a
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Feb 24 20:35:39 2025 +0800
[feature](metrics) add catalog/db/table num metrics (#47891) (#47998)
bp #47891
---
.../java/org/apache/doris/catalog/Database.java | 4 ++
.../org/apache/doris/datasource/CatalogMgr.java | 4 ++
.../apache/doris/datasource/InternalCatalog.java | 4 ++
.../java/org/apache/doris/metric/MetricRepo.java | 35 ++++++++++++++++++
.../java/org/apache/doris/metric/MetricsTest.java | 43 ++++++++++++++++++++++
5 files changed, 90 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 50a0367dfec..016a70132a4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -918,4 +918,8 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
public String toString() {
return toJson();
}
+
+ public int getTableNum() {
+ return idToTable.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 53a3dd47b8a..cd11d9424e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -839,4 +839,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
public Set<CatalogIf> getCopyOfCatalog() {
return new HashSet<>(idToCatalog.values());
}
+
+ public int getCatalogNum() {
+ return idToCatalog.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index fe4957a30bd..a58c0e4a149 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -258,6 +258,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
return Lists.newArrayList(idToDb.keySet());
}
+ public int getDbNum() {
+ return idToDb.size();
+ }
+
@Nullable
@Override
public Database getDbNullable(String dbName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index eed713cc708..abe555ac7f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -20,6 +20,7 @@ package org.apache.doris.metric;
import org.apache.doris.alter.Alter;
import org.apache.doris.alter.AlterJobV2.JobType;
+import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.common.Config;
@@ -134,6 +135,12 @@ public final class MetricRepo {
public static GaugeMetricImpl<Double> GAUGE_REQUEST_PER_SECOND;
public static GaugeMetricImpl<Double> GAUGE_QUERY_ERR_RATE;
public static GaugeMetricImpl<Long> GAUGE_MAX_TABLET_COMPACTION_SCORE;
+
+ // Catlaog/Database/Table num
+ public static GaugeMetric<Integer> GAUGE_CATALOG_NUM;
+ public static GaugeMetric<Integer> GAUGE_INTERNAL_DATABASE_NUM;
+ public static GaugeMetric<Integer> GAUGE_INTERNAL_TABLE_NUM;
+
private static Map<Pair<EtlJobType, JobState>, Long> loadJobNum =
Maps.newHashMap();
private static ScheduledThreadPoolExecutor metricTimer =
ThreadPoolManager.newDaemonScheduledThreadPool(1,
@@ -503,6 +510,34 @@ public final class MetricRepo {
THRIFT_COUNTER_RPC_LATENCY = addLabeledMetrics("method", () ->
new LongCounterMetric("thrift_rpc_latency_ms",
MetricUnit.MILLISECONDS, ""));
+ GAUGE_CATALOG_NUM = new GaugeMetric<Integer>("catalog_num",
+ MetricUnit.NOUNIT, "total catalog num") {
+ @Override
+ public Integer getValue() {
+ return Env.getCurrentEnv().getCatalogMgr().getCatalogNum();
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_CATALOG_NUM);
+
+ GAUGE_INTERNAL_DATABASE_NUM = new
GaugeMetric<Integer>("internal_database_num",
+ MetricUnit.NOUNIT, "total internal database num") {
+ @Override
+ public Integer getValue() {
+ return
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog().getDbNum();
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_INTERNAL_DATABASE_NUM);
+
+ GAUGE_INTERNAL_TABLE_NUM = new
GaugeMetric<Integer>("internal_table_num",
+ MetricUnit.NOUNIT, "total internal table num") {
+ @Override
+ public Integer getValue() {
+ return
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog().getAllDbs().stream()
+ .map(d -> (Database)
d).map(Database::getTableNum).reduce(0, Integer::sum);
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_INTERNAL_TABLE_NUM);
+
// init system metrics
initSystemMetrics();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
index 5b8c4bdd728..3ce3b475153 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.metric;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.JsonUtil;
+import org.apache.doris.metric.Metric.MetricUnit;
import org.apache.doris.monitor.jvm.JvmService;
import org.apache.doris.monitor.jvm.JvmStats;
@@ -122,4 +123,46 @@ public class MetricsTest {
Assert.assertTrue(size.get() <
JsonUtil.parseArray(finalMetricJson).size());
}
+
+ @Test
+ public void testCatalogAndDatabaseMetrics() {
+ List<Metric> catalogMetrics =
MetricRepo.getMetricsByName("catalog_num");
+ Assert.assertEquals(1, catalogMetrics.size());
+ GaugeMetric<Integer> catalogMetric = (GaugeMetric<Integer>)
catalogMetrics.get(0);
+ Assert.assertEquals("catalog_num", catalogMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, catalogMetric.getUnit());
+ Assert.assertEquals("total catalog num",
catalogMetric.getDescription());
+
+ List<Metric> dbMetrics =
MetricRepo.getMetricsByName("internal_database_num");
+ Assert.assertEquals(1, dbMetrics.size());
+ GaugeMetric<Integer> dbMetric = (GaugeMetric<Integer>)
dbMetrics.get(0);
+ Assert.assertEquals("internal_database_num", dbMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, dbMetric.getUnit());
+ Assert.assertEquals("total internal database num",
dbMetric.getDescription());
+
+ List<Metric> tableMetrics =
MetricRepo.getMetricsByName("internal_table_num");
+ Assert.assertEquals(1, tableMetrics.size());
+ GaugeMetric<Integer> tableMetric = (GaugeMetric<Integer>)
tableMetrics.get(0);
+ Assert.assertEquals("internal_table_num", tableMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, tableMetric.getUnit());
+ Assert.assertEquals("total internal table num",
tableMetric.getDescription());
+
+ // Test metrics in Prometheus format
+ MetricVisitor visitor = new PrometheusMetricVisitor();
+ MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
+ String metricResult = visitor.finish();
+
+ Assert.assertTrue(metricResult.contains("# TYPE doris_fe_catalog_num
gauge"));
+ Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_internal_database_num gauge"));
+ Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_internal_table_num gauge"));
+
+ // Test metrics in JSON format
+ JsonMetricVisitor jsonVisitor = new JsonMetricVisitor();
+ MetricRepo.DORIS_METRIC_REGISTER.accept(jsonVisitor);
+ String jsonResult = jsonVisitor.finish();
+
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_catalog_num\""));
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_internal_database_num\""));
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_internal_table_num\""));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
