This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch feature/optimize_memory
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/feature/optimize_memory by
this push:
new b5d88a6590b add memory threshold code
b5d88a6590b is described below
commit b5d88a6590bf8c3955cadc189f74146a2df1b789
Author: spricoder <[email protected]>
AuthorDate: Thu Dec 26 15:07:23 2024 +0800
add memory threshold code
---
.../db/service/metrics/DataNodeMetricsHelper.java | 82 +++++-----
.../metrics/memory/ThresholdMemoryMetrics.java | 174 +++++++++++++++++++++
.../iotdb/commons/service/metric/enums/Metric.java | 2 +-
.../iotdb/commons/service/metric/enums/Tag.java | 4 +-
4 files changed, 220 insertions(+), 42 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index 17463aa357f..dda0a403bc5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -39,6 +39,7 @@ import
org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet;
import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet;
import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
+import org.apache.iotdb.db.service.metrics.memory.ThresholdMemoryMetrics;
import
org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesNumberMetricsSet;
import
org.apache.iotdb.db.storageengine.load.metrics.ActiveLoadingFilesSizeMetricsSet;
import org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet;
@@ -58,52 +59,56 @@ import java.util.List;
public class DataNodeMetricsHelper {
/** Bind predefined metric sets into DataNode. */
public static void bind() {
- MetricService.getInstance().addMetricSet(new UpTimeMetrics());
- MetricService.getInstance().addMetricSet(new JvmMetrics());
- MetricService.getInstance().addMetricSet(ThreadPoolMetrics.getInstance());
- MetricService.getInstance().addMetricSet(new LogbackMetrics());
- MetricService.getInstance().addMetricSet(FileMetrics.getInstance());
- MetricService.getInstance().addMetricSet(CompactionMetrics.getInstance());
- MetricService.getInstance().addMetricSet(new ProcessMetrics());
- MetricService.getInstance().addMetricSet(new
DiskMetrics(IoTDBConstant.DN_ROLE));
- MetricService.getInstance().addMetricSet(new
NetMetrics(IoTDBConstant.DN_ROLE));
-
MetricService.getInstance().addMetricSet(ClientManagerMetrics.getInstance());
- initCpuMetrics();
- initSystemMetrics();
- MetricService.getInstance().addMetricSet(WritingMetrics.getInstance());
+ MetricService metricService = MetricService.getInstance();
+ metricService.addMetricSet(new UpTimeMetrics());
+ metricService.addMetricSet(new JvmMetrics());
+ metricService.addMetricSet(ThreadPoolMetrics.getInstance());
+ metricService.addMetricSet(new LogbackMetrics());
+ metricService.addMetricSet(FileMetrics.getInstance());
+ metricService.addMetricSet(CompactionMetrics.getInstance());
+ metricService.addMetricSet(new ProcessMetrics());
+ metricService.addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE));
+ metricService.addMetricSet(new NetMetrics(IoTDBConstant.DN_ROLE));
+ metricService.addMetricSet(ClientManagerMetrics.getInstance());
+ initCpuMetrics(metricService);
+ initSystemMetrics(metricService);
+ metricService.addMetricSet(WritingMetrics.getInstance());
// bind query related metrics
-
MetricService.getInstance().addMetricSet(QueryPlanCostMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(SeriesScanCostMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(QueryExecutionMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(QueryResourceMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(DataExchangeCostMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(DataExchangeCountMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(DriverSchedulerMetricSet.getInstance());
-
MetricService.getInstance().addMetricSet(QueryRelatedResourceMetricSet.getInstance());
+ metricService.addMetricSet(QueryPlanCostMetricSet.getInstance());
+ metricService.addMetricSet(SeriesScanCostMetricSet.getInstance());
+ metricService.addMetricSet(QueryExecutionMetricSet.getInstance());
+ metricService.addMetricSet(QueryResourceMetricSet.getInstance());
+ metricService.addMetricSet(DataExchangeCostMetricSet.getInstance());
+ metricService.addMetricSet(DataExchangeCountMetricSet.getInstance());
+ metricService.addMetricSet(DriverSchedulerMetricSet.getInstance());
+ metricService.addMetricSet(QueryRelatedResourceMetricSet.getInstance());
// bind performance overview related metrics
-
MetricService.getInstance().addMetricSet(PerformanceOverviewMetrics.getInstance());
+ metricService.addMetricSet(PerformanceOverviewMetrics.getInstance());
// bind gc metrics
-
MetricService.getInstance().addMetricSet(JvmGcMonitorMetrics.getInstance());
+ metricService.addMetricSet(JvmGcMonitorMetrics.getInstance());
// bind pipe related metrics
-
MetricService.getInstance().addMetricSet(PipeDataNodeMetrics.getInstance());
+ metricService.addMetricSet(PipeDataNodeMetrics.getInstance());
// bind load tsfile memory related metrics
-
MetricService.getInstance().addMetricSet(LoadTsFileMemMetricSet.getInstance());
+ metricService.addMetricSet(LoadTsFileMemMetricSet.getInstance());
// bind subscription related metrics
-
MetricService.getInstance().addMetricSet(SubscriptionMetrics.getInstance());
+ metricService.addMetricSet(SubscriptionMetrics.getInstance());
// bind load related metrics
-
MetricService.getInstance().addMetricSet(LoadTsFileCostMetricsSet.getInstance());
-
MetricService.getInstance().addMetricSet(ActiveLoadingFilesNumberMetricsSet.getInstance());
-
MetricService.getInstance().addMetricSet(ActiveLoadingFilesSizeMetricsSet.getInstance());
+ metricService.addMetricSet(LoadTsFileCostMetricsSet.getInstance());
+
metricService.addMetricSet(ActiveLoadingFilesNumberMetricsSet.getInstance());
+ metricService.addMetricSet(ActiveLoadingFilesSizeMetricsSet.getInstance());
+
+ // bind memory related metrics
+ metricService.addMetricSet(ThresholdMemoryMetrics.getInstance());
}
- private static void initSystemMetrics() {
+ private static void initSystemMetrics(MetricService metricService) {
ArrayList<String> diskDirs = new ArrayList<>();
diskDirs.add(IoTDBDescriptor.getInstance().getConfig().getSystemDir());
diskDirs.add(IoTDBDescriptor.getInstance().getConfig().getConsensusDir());
@@ -112,20 +117,19 @@ public class DataNodeMetricsHelper {
diskDirs.add(CommonDescriptor.getInstance().getConfig().getSyncDir());
diskDirs.add(IoTDBDescriptor.getInstance().getConfig().getSortTmpDir());
SystemMetrics.getInstance().setDiskDirs(diskDirs);
- MetricService.getInstance().addMetricSet(SystemMetrics.getInstance());
+ metricService.addMetricSet(SystemMetrics.getInstance());
}
- private static void initCpuMetrics() {
+ private static void initCpuMetrics(MetricService metricService) {
List<String> threadModules = new ArrayList<>();
Arrays.stream(ThreadModule.values()).forEach(x ->
threadModules.add(x.toString()));
List<String> pools = new ArrayList<>();
Arrays.stream(ThreadName.values()).forEach(x -> pools.add(x.name()));
- MetricService.getInstance()
- .addMetricSet(
- new CpuUsageMetrics(
- threadModules,
- pools,
- x -> ThreadName.getModuleTheThreadBelongs(x).toString(),
- x -> ThreadName.getThreadPoolTheThreadBelongs(x).name()));
+ metricService.addMetricSet(
+ new CpuUsageMetrics(
+ threadModules,
+ pools,
+ x -> ThreadName.getModuleTheThreadBelongs(x).toString(),
+ x -> ThreadName.getThreadPoolTheThreadBelongs(x).name()));
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java
new file mode 100644
index 00000000000..bdb333a3795
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/memory/ThresholdMemoryMetrics.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.service.metrics.memory;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+public class ThresholdMemoryMetrics implements IMetricSet {
+ private static final IoTDBConfig config =
IoTDBDescriptor.getInstance().getConfig();
+ private static final SystemInfo systemInfo = SystemInfo.getInstance();
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ metricService
+ .getOrCreateGauge(
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ MetricLevel.NORMAL,
+ Tag.NAME.toString(),
+ "StorageEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0")
+ .set(config.getAllocateMemoryForStorageEngine());
+ metricService
+ .getOrCreateGauge(
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ MetricLevel.NORMAL,
+ Tag.NAME.toString(),
+ "QueryEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0")
+ .set(config.getAllocateMemoryForRead());
+ metricService
+ .getOrCreateGauge(
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ MetricLevel.NORMAL,
+ Tag.NAME.toString(),
+ "SchemaEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0")
+ .set(config.getAllocateMemoryForSchema());
+ metricService
+ .getOrCreateGauge(
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ MetricLevel.NORMAL,
+ Tag.NAME.toString(),
+ "Consensus",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0")
+ .set(config.getAllocateMemoryForConsensus());
+ metricService
+ .getOrCreateGauge(
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ MetricLevel.NORMAL,
+ Tag.NAME.toString(),
+ "StreamEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0")
+ .set(config.getAllocateMemoryForPipe());
+ metricService
+ .getOrCreateGauge(
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ MetricLevel.NORMAL,
+ Tag.NAME.toString(),
+ "DirectBuffer",
+ Tag.TYPE.toString(),
+ "OffHeap",
+ Tag.LEVEL.toString(),
+ "0")
+ .set(systemInfo.getTotalDirectBufferMemorySizeLimit());
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ metricService.remove(
+ MetricType.GAUGE,
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ Tag.NAME.toString(),
+ "StorageEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0");
+ metricService.remove(
+ MetricType.GAUGE,
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ Tag.NAME.toString(),
+ "QueryEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0");
+ metricService.remove(
+ MetricType.GAUGE,
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ Tag.NAME.toString(),
+ "SchemaEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0");
+ metricService.remove(
+ MetricType.GAUGE,
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ Tag.NAME.toString(),
+ "Consensus",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0");
+ metricService.remove(
+ MetricType.GAUGE,
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ Tag.NAME.toString(),
+ "StreamEngine",
+ Tag.TYPE.toString(),
+ "OnHeap",
+ Tag.LEVEL.toString(),
+ "0");
+ metricService.remove(
+ MetricType.GAUGE,
+ Metric.THRESHOLD_MEMORY_SIZE.toString(),
+ Tag.NAME.toString(),
+ "DirectBuffer",
+ Tag.TYPE.toString(),
+ "OffHeap",
+ Tag.LEVEL.toString(),
+ "0");
+ }
+
+ public static ThresholdMemoryMetrics getInstance() {
+ return ThresholdMemoryMetrics.ThresholdMemoryMetricsHolder.INSTANCE;
+ }
+
+ private static class ThresholdMemoryMetricsHolder {
+
+ private static final ThresholdMemoryMetrics INSTANCE = new
ThresholdMemoryMetrics();
+
+ private ThresholdMemoryMetricsHolder() {}
+ }
+}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index 6d104a64376..b70aae9057f 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -187,7 +187,7 @@ public enum Metric {
MEMTABLE_POINT_COUNT("memtable_point_count"),
BINARY_ALLOCATOR("binary_allocator"),
// memory related
- MEMORY_SIZE("memory_size"),
+ THRESHOLD_MEMORY_SIZE("threshold_memory_size"),
;
final String value;
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java
index 30ee06e8484..5fecc6b3122 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java
@@ -31,8 +31,8 @@ public enum Tag {
INTERFACE("interface"),
CREATION_TIME("creation_time"),
INDEX("index"),
- MODULE("module")
- ;
+ MODULE("module"),
+ LEVEL("level");
final String value;
