This is an automated email from the ASF dual-hosted git repository.
neuyilan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ef0e514ca6 [IOTDB-3955] Improved node information metrics to push more
detailed node information, online and offline total and survival information of
each node (#6801)
ef0e514ca6 is described below
commit ef0e514ca60181e4f0836dcee273453940c75266
Author: 23931017wu <[email protected]>
AuthorDate: Tue Aug 2 13:14:26 2022 +0800
[IOTDB-3955] Improved node information metrics to push more detailed node
information, online and offline total and survival information of each node
(#6801)
---
.../iotdb/confignode/manager/load/LoadManager.java | 215 +++++++++++++++++++++
.../iotdb/confignode/persistence/NodeInfo.java | 9 +-
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 8 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 20 +-
.../micrometer/MicrometerMetricManagerTest.java | 11 ++
.../apache/iotdb/commons/cluster/NodeStatus.java | 4 +
6 files changed, 251 insertions(+), 16 deletions(-)
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
index 9423419df4..6c0a21cfd9 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
@@ -54,6 +54,11 @@ import
org.apache.iotdb.confignode.manager.load.heartbeat.ConfigNodeHeartbeatCac
import
org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.IRegionGroupCache;
+import org.apache.iotdb.db.service.metrics.MetricsService;
+import org.apache.iotdb.db.service.metrics.enums.Metric;
+import org.apache.iotdb.db.service.metrics.enums.Tag;
+import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
+import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.mpp.rpc.thrift.THeartbeatReq;
import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq;
@@ -244,6 +249,7 @@ public class LoadManager {
/** Stop the heartbeat service and the load balancing service */
public void stop() {
+ removeMetrics();
LOGGER.debug("Stop Heartbeat Service and LoadBalancing Service of
LoadManager");
synchronized (scheduleMonitor) {
if (currentHeartbeatFuture != null) {
@@ -286,6 +292,7 @@ public class LoadManager {
if (isNeedBroadcast.get()) {
broadcastLatestRegionRouteMap();
}
+ addMetrics();
}
private void broadcastLatestRegionRouteMap() {
@@ -420,6 +427,214 @@ public class LoadManager {
.collect(Collectors.toList());
}
+ public List<TConfigNodeLocation> getUnknownConfigNodes() {
+ return getNodeManager().getRegisteredConfigNodes().stream()
+ .filter(
+ registeredConfigNode ->
+ nodeCacheMap
+ .get(registeredConfigNode.getConfigNodeId())
+ .getNodeStatus()
+ .equals(NodeStatus.Unknown))
+ .collect(Collectors.toList());
+ }
+
+ public List<TDataNodeConfiguration> getUnknownDataNodes(int dataNodeId) {
+ return getNodeManager().getRegisteredDataNodes(dataNodeId).stream()
+ .filter(
+ registeredDataNode ->
+ nodeCacheMap
+ .get(registeredDataNode.getLocation().getDataNodeId())
+ .getNodeStatus()
+ .equals(NodeStatus.Unknown))
+ .collect(Collectors.toList());
+ }
+
+ public int getRunningConfigNodesNum() {
+ List<TConfigNodeLocation> allConfigNodes = getOnlineConfigNodes();
+ if (allConfigNodes == null) {
+ return 0;
+ }
+ for (TConfigNodeLocation configNodeLocation : allConfigNodes) {
+ String name =
+ "EndPoint("
+ + configNodeLocation.getInternalEndPoint().ip
+ + ":"
+ + configNodeLocation.getInternalEndPoint().port
+ + ")";
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.CLUSTER_NODE_STATUS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ name,
+ Tag.TYPE.toString(),
+ "ConfigNode")
+ .set(1);
+ }
+ return allConfigNodes.size();
+ }
+
+ public int getRunningDataNodesNum() {
+ List<TDataNodeConfiguration> allDataNodes = getOnlineDataNodes(-1);
+ if (allDataNodes == null) {
+ return 0;
+ }
+ for (TDataNodeConfiguration dataNodeInfo : allDataNodes) {
+ TDataNodeLocation dataNodeLocation = dataNodeInfo.getLocation();
+ String name =
+ "EndPoint("
+ + dataNodeLocation.getClientRpcEndPoint().ip
+ + ":"
+ + dataNodeLocation.getClientRpcEndPoint().port
+ + ")";
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.CLUSTER_NODE_STATUS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ name,
+ Tag.TYPE.toString(),
+ "DataNode")
+ .set(1);
+ }
+ return allDataNodes.size();
+ }
+
+ public int getUnknownConfigNodesNum() {
+ List<TConfigNodeLocation> allConfigNodes = getUnknownConfigNodes();
+ if (allConfigNodes == null) {
+ return 0;
+ }
+ for (TConfigNodeLocation configNodeLocation : allConfigNodes) {
+ String name =
+ "EndPoint("
+ + configNodeLocation.getInternalEndPoint().ip
+ + ":"
+ + configNodeLocation.getInternalEndPoint().port
+ + ")";
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.CLUSTER_NODE_STATUS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ name,
+ Tag.TYPE.toString(),
+ "ConfigNode")
+ .set(0);
+ }
+ return allConfigNodes.size();
+ }
+
+ public int getUnknownDataNodesNum() {
+ List<TDataNodeConfiguration> allDataNodes = getUnknownDataNodes(-1);
+ if (allDataNodes == null) {
+ return 0;
+ }
+ for (TDataNodeConfiguration dataNodeInfo : allDataNodes) {
+ TDataNodeLocation dataNodeLocation = dataNodeInfo.getLocation();
+ String name =
+ "EndPoint("
+ + dataNodeLocation.getClientRpcEndPoint().ip
+ + ":"
+ + dataNodeLocation.getClientRpcEndPoint().port
+ + ")";
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.CLUSTER_NODE_STATUS.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.NAME.toString(),
+ name,
+ Tag.TYPE.toString(),
+ "DataNode")
+ .set(0);
+ }
+ return allDataNodes.size();
+ }
+
+ public void addMetrics() {
+ if
(MetricConfigDescriptor.getInstance().getMetricConfig().getEnableMetric()) {
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.CONFIG_NODE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Online.toString())
+ .set(getRunningConfigNodesNum());
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.DATA_NODE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Online.toString())
+ .set(getRunningDataNodesNum());
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.CONFIG_NODE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Unknown.toString())
+ .set(getUnknownConfigNodesNum());
+ MetricsService.getInstance()
+ .getMetricManager()
+ .getOrCreateGauge(
+ Metric.DATA_NODE.toString(),
+ MetricLevel.CORE,
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Unknown.toString())
+ .set(getUnknownDataNodesNum());
+ }
+ }
+
+ public void removeMetrics() {
+ MetricsService.getInstance()
+ .getMetricManager()
+ .removeGauge(
+ Metric.CONFIG_NODE.toString(),
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Online.toString());
+ MetricsService.getInstance()
+ .getMetricManager()
+ .removeGauge(
+ Metric.DATA_NODE.toString(),
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Online.toString());
+ MetricsService.getInstance()
+ .getMetricManager()
+ .removeGauge(
+ Metric.CONFIG_NODE.toString(),
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Unknown.toString());
+ MetricsService.getInstance()
+ .getMetricManager()
+ .removeGauge(
+ Metric.DATA_NODE.toString(),
+ Tag.NAME.toString(),
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Unknown.toString());
+ }
+
public static void printRegionRouteMap(
long timestamp, Map<TConsensusGroupId, TRegionReplicaSet>
regionRouteMap) {
LOGGER.info("[latestRegionRouteMap] timestamp:{}", timestamp);
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
index 5830c6127a..7b4e89ea8d 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
@@ -112,7 +113,9 @@ public class NodeInfo implements SnapshotProcessor {
registeredConfigNodes,
o -> getRegisteredConfigNodeCount(),
Tag.NAME.toString(),
- "online");
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Registered.toString());
MetricsService.getInstance()
.getMetricManager()
.getOrCreateAutoGauge(
@@ -121,7 +124,9 @@ public class NodeInfo implements SnapshotProcessor {
registeredDataNodes,
Map::size,
Tag.NAME.toString(),
- "online");
+ "total",
+ Tag.STATUS.toString(),
+ NodeStatus.Registered.toString());
}
}
diff --git a/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
b/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
index a1cb9ef531..9a2035f6de 100644
--- a/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
+++ b/docs/UserGuide/Maintenance-Tools/Metric-Tool.md
@@ -124,10 +124,10 @@ Next, we will choose Prometheus format data as samples to
describe each kind of
| ------------------------- |
------------------------------------------------------------------ | ---------
|
--------------------------------------------------------------------------------------------
| ----------------------------------------------------------------------------
|
| cluster_node_leader_count | name="{{ip}}"
| important | The count of ```dataGroupLeader``` on each
node, which reflects the distribution of leaders |
cluster_node_leader_count{name="127.0.0.1",} 2.0 |
| cluster_uncommitted_log | name="{{ip_datagroupHeader}}"
| important | The count of ```uncommitted_log``` on each node
in data groups it belongs to |
cluster_uncommitted_log{name="127.0.0.1_Data-127.0.0.1-40010-raftId-0",} 0.0 |
-| cluster_node_status | name="{{ip}}"
| important | The current node status, 1=online 2=offline
|
cluster_node_status{name="127.0.0.1",} 1.0 |
+| cluster_node_status |
name="{{ip}}:{{port}}",type="ConfigNode/DataNode" | important
| The current node status, 0=Unkonwn 1=online
|
cluster_node_status{name="EndPoint(0.0.0.0:22277)",type="ConfigNode",} 1.0 |
| cluster_elect_total | name="{{ip}}",status="fail/win"
| important | The count and result (won or failed) of
elections the node participated in. |
cluster_elect_total{name="127.0.0.1",status="win",} 1.0 |
-| config_node | name="online"
| core | The number of online confignodes
| config_node{name="online",} 3.0
|
-| data_node | name="online"
| core | The number of online datanodes
| data_node{name="online",} 3.0
|
+| config_node | name="total",status="Registered/Online/Unknown"
| core | The number of registered/online/offline
confignodes |
config_node{name="total",status="Online",} 3.0 |
+| data_node | name="total",status="Registered/Online/Unknown"
| core | The number of registered/online/offline
datanodes |
data_node{name="total",status="Registered",} 3.0 |
| partition_table | name="number"
| core | The number of partition table
| partition_table{name="number",}
2.0 |
| region |
name="total/{{ip}}:{{port}}",type="SchemaRegion/DataRegion" | important
| The number of schemaRegion/dataRegion of cluster or specific node
| region{name="127.0.0.1:6671",type="DataRegion",} 10.0
|
| region |
name="{{storageGroupName}}",type="SchemaRegion/DataRegion" | normal
| The number of DataRegion/SchemaRegion in storage group
| region{name="root.schema.sg1",type="DataRegion",} 14.0
|
@@ -438,4 +438,4 @@ When creating Grafana, you can select the json file you
just downloaded to `Impo
4. `The time consumed of GC (per minute)`: IoTDB's average GC time per
minute, including Young GC and Full GC.
5. `Heap Memory`: The heap memory of IoTDB.
6. `Off-heap Memory`: The off-heap memory of IoTDB.
- 7. `The number of Java Thread`: The number of threads in different states
of IoTDB.
\ No newline at end of file
+ 7. `The number of Java Thread`: The number of threads in different states
of IoTDB.
diff --git a/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
b/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
index b18dbb6e56..91929f5bab 100644
--- a/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
+++ b/docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md
@@ -120,16 +120,16 @@ IoTDB对外提供JMX和Prometheus格式的监控指标，对于JMX，可以通
| Metric | Tag
| level | 说明
| 示例
|
| ------------------------- |
------------------------------------------------------------------ | ---------
| ------------------------------------------------------------- |
---------------------------------------------------------------------------- |
-| cluster_node_leader_count | name="{{ip}}"
| important | 节点上```dataGroupLeader```的数量，用来观察leader是否分布均匀 |
cluster_node_leader_count{name="127.0.0.1",} 2.0 |
-| cluster_uncommitted_log | name="{{ip_datagroupHeader}}"
| important | 节点```uncommitted_log```的数量
|
cluster_uncommitted_log{name="127.0.0.1_Data-127.0.0.1-40010-raftId-0",} 0.0 |
-| cluster_node_status | name="{{ip}}"
| important | 节点状态，1=online 2=offline
| cluster_node_status{name="127.0.0.1",} 1.0
|
-| cluster_elect_total | name="{{ip}}",status="fail/win"
| important | 节点参与选举的次数及结果
| cluster_elect_total{name="127.0.0.1",status="win",} 1.0
|
-| config_node | name="online"
| core | 上线confignode的节点数量
| config_node{name="online",} 3.0
|
-| data_node | name="online"
| core | 上线datanode的节点数量
| data_node{name="online",} 3.0
|
-| partition_table | name="number"
| core | partition table表的个数
| partition_table{name="number",} 2.0
|
-| region |
name="total/{{ip}}:{{port}}",type="SchemaRegion/DataRegion" | important
| 全部或某个节点的schemaRegion/dataRegion个数 |
region{name="127.0.0.1:6671",type="DataRegion",} 10.0 |
-| region |
name="{{storageGroupName}}",type="SchemaRegion/DataRegion" | normal
| 存储组的DataRegion/Schema个数 |
region{name="root.schema.sg1",type="DataRegion",} 14.0 |
-| slot |
name="{{storageGroupName}}",type="schemaSlotNumber/dataSlotNumber" | normal
| 存储组的schemaSlot/dataSlot个数 |
slot{name="root.schema.sg1",type="schemaSlotNumber",} 2.0 |
+| cluster_node_leader_count | name="{{ip}}"
| important | 节点上```dataGroupLeader```的数量，用来观察leader是否分布均匀
| cluster_node_leader_count{name="127.0.0.1",} 2.0 |
+| cluster_uncommitted_log | name="{{ip_datagroupHeader}}"
| important | 节点```uncommitted_log```的数量
|
cluster_uncommitted_log{name="127.0.0.1_Data-127.0.0.1-40010-raftId-0",} 0.0 |
+| cluster_node_status |
name="{{ip}}:{{port}}",type="ConfigNode/DataNode" | important
| 节点状态，0=Unkonwn 1=online |
cluster_node_status{name="EndPoint(0.0.0.0:22277)",type="ConfigNode",} 1.0 |
+| cluster_elect_total | name="{{ip}}",status="fail/win"
| important | 节点参与选举的次数及结果
| cluster_elect_total{name="127.0.0.1",status="win",} 1.0
|
+| config_node | name="total",status="Registered/Online/Unknown"
| core | 已注册/在线/离线 confignode 的节点数量
| config_node{name="total",status="Online",} 2.0
|
+| data_node | name="total",status="Registered/Online/Unknown"
| core | 已注册/在线/离线 datanode 的节点数量
| data_node{name="total",status="Registered",} 3.0
|
+| partition_table | name="number"
| core | partition table表的个数
| partition_table{name="number",} 2.0
|
+| region |
name="total/{{ip}}:{{port}}",type="SchemaRegion/DataRegion" | important
| 全部或某个节点的schemaRegion/dataRegion个数 |
region{name="127.0.0.1:6671",type="DataRegion",} 10.0 |
+| region |
name="{{storageGroupName}}",type="SchemaRegion/DataRegion" | normal
| 存储组的DataRegion/Schema个数 |
region{name="root.schema.sg1",type="DataRegion",} 14.0 |
+| slot |
name="{{storageGroupName}}",type="schemaSlotNumber/dataSlotNumber" | normal
| 存储组的schemaSlot/dataSlot个数 |
slot{name="root.schema.sg1",type="schemaSlotNumber",} 2.0 |
### 4.4. IoTDB 预定义指标集
diff --git
a/metrics/micrometer-metrics/src/test/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManagerTest.java
b/metrics/micrometer-metrics/src/test/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManagerTest.java
index c0ae5258e3..89a58bf38f 100644
---
a/metrics/micrometer-metrics/src/test/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManagerTest.java
+++
b/metrics/micrometer-metrics/src/test/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManagerTest.java
@@ -36,6 +36,7 @@ import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThrows;
@@ -92,4 +93,14 @@ public class MicrometerMetricManagerTest {
System.gc();
assertEquals(0L, autoGauge.value());
}
+
+ @Test
+ public void removeGauge() {
+ Gauge gauge1 =
+ metricManager.getOrCreateGauge("gauge_remove", MetricLevel.IMPORTANT,
"tag1", "tag2");
+ metricManager.removeGauge("gauge_remove", "tag1", "tag2");
+ Gauge gauge2 =
+ metricManager.getOrCreateGauge("gauge_remove", MetricLevel.IMPORTANT,
"tag1", "tag2");
+ assertNotEquals(gauge1, gauge2);
+ }
}
diff --git
a/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
b/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
index 08c8d9963d..8d2a3577a2 100644
---
a/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
+++
b/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
@@ -20,6 +20,10 @@ package org.apache.iotdb.commons.cluster;
/** Node status for showing cluster */
public enum NodeStatus {
+ // Node registered
+ Registered("Registered"),
+ // Node online ,right now Online is Running
+ Online("Online"),
// Node running properly
Running("Running"),
// Node connection failure
