This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b95b7040c4 [IOTDB-3355] Adapt metric framework for config node and
check metrics (#6096)
b95b7040c4 is described below
commit b95b7040c494fc47a19c5ed69ae02815133a5248
Author: ZhangHongYin <[email protected]>
AuthorDate: Wed Jun 1 13:22:32 2022 +0800
[IOTDB-3355] Adapt metric framework for config node and check metrics
(#6096)
* fix metric init.
* add metric service into confignode.
---
.../iotdb/confignode/service/ConfigNode.java | 5 +++
.../resources/confignode1conf}/iotdb-metric.yml | 34 +++++++++++++++++++-
.../resources/confignode2conf}/iotdb-metric.yml | 34 +++++++++++++++++++-
.../resources/confignode3conf}/iotdb-metric.yml | 34 +++++++++++++++++++-
distribution/src/assembly/confignode.xml | 4 +++
.../metrics/config/MetricConfigDescriptor.java | 5 +++
.../iotdb/metrics/config/MetricConstant.java | 2 ++
.../java/org/apache/iotdb/db/service/DataNode.java | 1 -
.../java/org/apache/iotdb/db/service/IoTDB.java | 1 -
.../db/service/metrics/predefined/FileMetrics.java | 36 ++++++++++++++--------
.../test/resources/datanode1conf/iotdb-metric.yml | 34 +++++++++++++++++++-
.../test/resources/datanode2conf/iotdb-metric.yml | 34 +++++++++++++++++++-
.../test/resources/datanode3conf/iotdb-metric.yml | 34 +++++++++++++++++++-
13 files changed, 238 insertions(+), 20 deletions(-)
diff --git
a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
index c4641bc4d9..753d45fac7 100644
---
a/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
+++
b/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNode.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.confignode.service.thrift.ConfigNodeRPCService;
import
org.apache.iotdb.confignode.service.thrift.ConfigNodeRPCServiceProcessor;
+import org.apache.iotdb.db.service.metrics.MetricsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,11 +81,15 @@ public class ConfigNode implements ConfigNodeMBean {
registerManager.register(new JMXService());
JMXService.registerMBean(this, mbeanName);
+ registerManager.register(MetricsService.getInstance());
registerUdfServices();
configNodeRPCService.initSyncedServiceImpl(configNodeRPCServiceProcessor);
registerManager.register(configNodeRPCService);
LOGGER.info("Init rpc server success");
+
+ // start reporter
+ MetricsService.getInstance().startAllReporter();
}
private void registerUdfServices() throws StartupException {
diff --git a/server/src/test/resources/datanode1conf/iotdb-metric.yml
b/confignode/src/test/resources/confignode1conf/iotdb-metric.yml
similarity index 51%
copy from server/src/test/resources/datanode1conf/iotdb-metric.yml
copy to confignode/src/test/resources/confignode1conf/iotdb-metric.yml
index 14ab7bc2d6..cf64f6ad88 100644
--- a/server/src/test/resources/datanode1conf/iotdb-metric.yml
+++ b/confignode/src/test/resources/confignode1conf/iotdb-metric.yml
@@ -18,4 +18,36 @@
#
# whether enable the module
-enableMetric: false
\ No newline at end of file
+enableMetric: false
+
+# Is stat performance of operation latency
+enablePerformanceStat: false
+
+# Multiple reporter, options: [JMX, PROMETHEUS, IOTDB], IOTDB is off by default
+metricReporterList:
+ - JMX
+ - PROMETHEUS
+
+# Type of monitor frame, options: [MICROMETER, DROPWIZARD]
+monitorType: MICROMETER
+
+# Level of metric level, options: [CORE, IMPORTANT, NORMAL, ALL]
+metricLevel: IMPORTANT
+
+# Predefined metric, options: [JVM, LOGBACK, FILE, PROCESS, SYSTEM]
+predefinedMetrics:
+ - JVM
+ - FILE
+
+# The http server's port for prometheus exporter to get metric data.
+prometheusExporterPort: 9091
+
+# The config of iotdb reporter
+ioTDBReporterConfig:
+ host: 127.0.0.1
+ port: 6667
+ username: root
+ password: root
+ maxConnectionNumber: 3
+ database: _metric
+ pushPeriodInSecond: 15
\ No newline at end of file
diff --git a/server/src/test/resources/datanode1conf/iotdb-metric.yml
b/confignode/src/test/resources/confignode2conf/iotdb-metric.yml
similarity index 51%
copy from server/src/test/resources/datanode1conf/iotdb-metric.yml
copy to confignode/src/test/resources/confignode2conf/iotdb-metric.yml
index 14ab7bc2d6..26fdd95834 100644
--- a/server/src/test/resources/datanode1conf/iotdb-metric.yml
+++ b/confignode/src/test/resources/confignode2conf/iotdb-metric.yml
@@ -18,4 +18,36 @@
#
# whether enable the module
-enableMetric: false
\ No newline at end of file
+enableMetric: false
+
+# Is stat performance of operation latency
+enablePerformanceStat: false
+
+# Multiple reporter, options: [JMX, PROMETHEUS, IOTDB], IOTDB is off by default
+metricReporterList:
+ - JMX
+ - PROMETHEUS
+
+# Type of monitor frame, options: [MICROMETER, DROPWIZARD]
+monitorType: MICROMETER
+
+# Level of metric level, options: [CORE, IMPORTANT, NORMAL, ALL]
+metricLevel: IMPORTANT
+
+# Predefined metric, options: [JVM, LOGBACK, FILE, PROCESS, SYSTEM]
+predefinedMetrics:
+ - JVM
+ - FILE
+
+# The http server's port for prometheus exporter to get metric data.
+prometheusExporterPort: 9093
+
+# The config of iotdb reporter
+ioTDBReporterConfig:
+ host: 127.0.0.1
+ port: 6667
+ username: root
+ password: root
+ maxConnectionNumber: 3
+ database: _metric
+ pushPeriodInSecond: 15
\ No newline at end of file
diff --git a/server/src/test/resources/datanode1conf/iotdb-metric.yml
b/confignode/src/test/resources/confignode3conf/iotdb-metric.yml
similarity index 51%
copy from server/src/test/resources/datanode1conf/iotdb-metric.yml
copy to confignode/src/test/resources/confignode3conf/iotdb-metric.yml
index 14ab7bc2d6..85f12c4f23 100644
--- a/server/src/test/resources/datanode1conf/iotdb-metric.yml
+++ b/confignode/src/test/resources/confignode3conf/iotdb-metric.yml
@@ -18,4 +18,36 @@
#
# whether enable the module
-enableMetric: false
\ No newline at end of file
+enableMetric: false
+
+# Is stat performance of operation latency
+enablePerformanceStat: false
+
+# Multiple reporter, options: [JMX, PROMETHEUS, IOTDB], IOTDB is off by default
+metricReporterList:
+ - JMX
+ - PROMETHEUS
+
+# Type of monitor frame, options: [MICROMETER, DROPWIZARD]
+monitorType: MICROMETER
+
+# Level of metric level, options: [CORE, IMPORTANT, NORMAL, ALL]
+metricLevel: IMPORTANT
+
+# Predefined metric, options: [JVM, LOGBACK, FILE, PROCESS, SYSTEM]
+predefinedMetrics:
+ - JVM
+ - FILE
+
+# The http server's port for prometheus exporter to get metric data.
+prometheusExporterPort: 9095
+
+# The config of iotdb reporter
+ioTDBReporterConfig:
+ host: 127.0.0.1
+ port: 6667
+ username: root
+ password: root
+ maxConnectionNumber: 3
+ database: _metric
+ pushPeriodInSecond: 15
\ No newline at end of file
diff --git a/distribution/src/assembly/confignode.xml
b/distribution/src/assembly/confignode.xml
index 3c830e28c7..a3be401300 100644
--- a/distribution/src/assembly/confignode.xml
+++ b/distribution/src/assembly/confignode.xml
@@ -37,6 +37,10 @@
</dependencySet>
</dependencySets>
<fileSets>
+ <fileSet>
+ <outputDirectory>conf</outputDirectory>
+
<directory>${maven.multiModuleProjectDirectory}/metrics/interface/src/main/assembly/resources/conf</directory>
+ </fileSet>
<fileSet>
<directory>${maven.multiModuleProjectDirectory}/confignode/src/assembly/resources/sbin</directory>
<outputDirectory>sbin</outputDirectory>
diff --git
a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
index b5d91534bd..069b5b097c 100644
---
a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
+++
b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfigDescriptor.java
@@ -53,6 +53,10 @@ public class MetricConfigDescriptor {
*/
private String getPropsUrl() {
String url = System.getProperty(MetricConstant.IOTDB_CONF, null);
+ // try to get config node conf
+ if (url == null) {
+ url = System.getProperty(MetricConstant.CONFIGNODE_CONF, null);
+ }
if (url == null) {
logger.warn(
"Cannot find IOTDB_CONF environment variable when loading "
@@ -62,6 +66,7 @@ public class MetricConfigDescriptor {
} else {
url += (File.separatorChar + MetricConstant.CONFIG_NAME);
}
+
return url;
}
diff --git
a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java
b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java
index d0444af4ce..540aa424cd 100644
---
a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java
+++
b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConstant.java
@@ -25,5 +25,7 @@ public class MetricConstant {
static final String IOTDB_CONF = "IOTDB_CONF";
+ static final String CONFIGNODE_CONF = "CONFIGNODE_CONF";
+
static final String CONFIG_NAME = "iotdb-metric.yml";
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
index 71528067fb..72b6a8df00 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -294,7 +294,6 @@ public class DataNode implements DataNodeMBean {
registerUdfServices();
registerManager.register(ReceiverService.getInstance());
- registerManager.register(MetricsService.getInstance());
initProtocols();
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 18911877ae..941ea27ebf 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -184,7 +184,6 @@ public class IoTDB implements IoTDBMBean {
+ "udf"
+ File.separator));
registerManager.register(ReceiverService.getInstance());
- registerManager.register(MetricsService.getInstance());
initProtocols();
// in cluster mode, InfluxDBMManager has been initialized, so there is no
need to init again to
diff --git
a/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
b/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
index 3e846c132e..bc7250362f 100644
---
a/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
+++
b/server/src/main/java/org/apache/iotdb/db/service/metrics/predefined/FileMetrics.java
@@ -40,7 +40,7 @@ public class FileMetrics implements IMetricSet {
Metric.FILE_SIZE.toString(),
MetricLevel.IMPORTANT,
walDirs,
- value -> Stream.of(value).mapToLong(dir ->
FileUtils.getDirSize(dir)).sum(),
+ value -> Stream.of(value).mapToLong(FileUtils::getDirSize).sum(),
Tag.NAME.toString(),
"wal");
@@ -83,11 +83,13 @@ public class FileMetrics implements IMetricSet {
dir -> {
File walFolder = new File(dir);
File[] walNodeFolders =
walFolder.listFiles(File::isDirectory);
- for (File walNodeFolder : walNodeFolders) {
- if (walNodeFolder.exists() &&
walNodeFolder.isDirectory()) {
- return org.apache.commons.io.FileUtils.listFiles(
- walNodeFolder, null, true)
- .size();
+ if (null != walNodeFolders) {
+ for (File walNodeFolder : walNodeFolders) {
+ if (walNodeFolder.exists() &&
walNodeFolder.isDirectory()) {
+ return org.apache.commons.io.FileUtils.listFiles(
+ walNodeFolder, null, true)
+ .size();
+ }
}
}
return 0L;
@@ -104,9 +106,14 @@ public class FileMetrics implements IMetricSet {
.mapToLong(
dir -> {
dir += File.separator +
IoTDBConstant.SEQUENCE_FLODER_NAME;
- return org.apache.commons.io.FileUtils.listFiles(
- new File(dir), new String[] {"tsfile"}, true)
- .size();
+ File folder = new File(dir);
+ if (folder.exists()) {
+ return org.apache.commons.io.FileUtils.listFiles(
+ new File(dir), new String[] {"tsfile"}, true)
+ .size();
+ } else {
+ return 0L;
+ }
})
.sum(),
Tag.NAME.toString(),
@@ -120,9 +127,14 @@ public class FileMetrics implements IMetricSet {
.mapToLong(
dir -> {
dir += File.separator +
IoTDBConstant.UNSEQUENCE_FLODER_NAME;
- return org.apache.commons.io.FileUtils.listFiles(
- new File(dir), new String[] {"tsfile"}, true)
- .size();
+ File folder = new File(dir);
+ if (folder.exists()) {
+ return org.apache.commons.io.FileUtils.listFiles(
+ new File(dir), new String[] {"tsfile"}, true)
+ .size();
+ } else {
+ return 0L;
+ }
})
.sum(),
Tag.NAME.toString(),
diff --git a/server/src/test/resources/datanode1conf/iotdb-metric.yml
b/server/src/test/resources/datanode1conf/iotdb-metric.yml
index 14ab7bc2d6..033d7f74f0 100644
--- a/server/src/test/resources/datanode1conf/iotdb-metric.yml
+++ b/server/src/test/resources/datanode1conf/iotdb-metric.yml
@@ -18,4 +18,36 @@
#
# whether enable the module
-enableMetric: false
\ No newline at end of file
+enableMetric: false
+
+# Is stat performance of operation latency
+enablePerformanceStat: false
+
+# Multiple reporter, options: [JMX, PROMETHEUS, IOTDB], IOTDB is off by default
+metricReporterList:
+ - JMX
+ - PROMETHEUS
+
+# Type of monitor frame, options: [MICROMETER, DROPWIZARD]
+monitorType: MICROMETER
+
+# Level of metric level, options: [CORE, IMPORTANT, NORMAL, ALL]
+metricLevel: IMPORTANT
+
+# Predefined metric, options: [JVM, LOGBACK, FILE, PROCESS, SYSTEM]
+predefinedMetrics:
+ - JVM
+ - FILE
+
+# The http server's port for prometheus exporter to get metric data.
+prometheusExporterPort: 9097
+
+# The config of iotdb reporter
+ioTDBReporterConfig:
+ host: 127.0.0.1
+ port: 6667
+ username: root
+ password: root
+ maxConnectionNumber: 3
+ database: _metric
+ pushPeriodInSecond: 15
\ No newline at end of file
diff --git a/server/src/test/resources/datanode2conf/iotdb-metric.yml
b/server/src/test/resources/datanode2conf/iotdb-metric.yml
index 14ab7bc2d6..9225877b9a 100644
--- a/server/src/test/resources/datanode2conf/iotdb-metric.yml
+++ b/server/src/test/resources/datanode2conf/iotdb-metric.yml
@@ -18,4 +18,36 @@
#
# whether enable the module
-enableMetric: false
\ No newline at end of file
+enableMetric: false
+
+# Is stat performance of operation latency
+enablePerformanceStat: false
+
+# Multiple reporter, options: [JMX, PROMETHEUS, IOTDB], IOTDB is off by default
+metricReporterList:
+ - JMX
+ - PROMETHEUS
+
+# Type of monitor frame, options: [MICROMETER, DROPWIZARD]
+monitorType: MICROMETER
+
+# Level of metric level, options: [CORE, IMPORTANT, NORMAL, ALL]
+metricLevel: IMPORTANT
+
+# Predefined metric, options: [JVM, LOGBACK, FILE, PROCESS, SYSTEM]
+predefinedMetrics:
+ - JVM
+ - FILE
+
+# The http server's port for prometheus exporter to get metric data.
+prometheusExporterPort: 9099
+
+# The config of iotdb reporter
+ioTDBReporterConfig:
+ host: 127.0.0.1
+ port: 6667
+ username: root
+ password: root
+ maxConnectionNumber: 3
+ database: _metric
+ pushPeriodInSecond: 15
\ No newline at end of file
diff --git a/server/src/test/resources/datanode3conf/iotdb-metric.yml
b/server/src/test/resources/datanode3conf/iotdb-metric.yml
index 14ab7bc2d6..adc2c4ca53 100644
--- a/server/src/test/resources/datanode3conf/iotdb-metric.yml
+++ b/server/src/test/resources/datanode3conf/iotdb-metric.yml
@@ -18,4 +18,36 @@
#
# whether enable the module
-enableMetric: false
\ No newline at end of file
+enableMetric: false
+
+# Is stat performance of operation latency
+enablePerformanceStat: false
+
+# Multiple reporter, options: [JMX, PROMETHEUS, IOTDB], IOTDB is off by default
+metricReporterList:
+ - JMX
+ - PROMETHEUS
+
+# Type of monitor frame, options: [MICROMETER, DROPWIZARD]
+monitorType: MICROMETER
+
+# Level of metric level, options: [CORE, IMPORTANT, NORMAL, ALL]
+metricLevel: IMPORTANT
+
+# Predefined metric, options: [JVM, LOGBACK, FILE, PROCESS, SYSTEM]
+predefinedMetrics:
+ - JVM
+ - FILE
+
+# The http server's port for prometheus exporter to get metric data.
+prometheusExporterPort: 9101
+
+# The config of iotdb reporter
+ioTDBReporterConfig:
+ host: 127.0.0.1
+ port: 6667
+ username: root
+ password: root
+ maxConnectionNumber: 3
+ database: _metric
+ pushPeriodInSecond: 15
\ No newline at end of file
