This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch metric in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e611d8c2f945b3687f5e55e03b88202c595a125e Author: HTHou <[email protected]> AuthorDate: Mon Nov 27 18:04:47 2023 +0800 deving --- .../iotdb/db/service/metrics/WritingMetrics.java | 139 +++++++++++++++++++++ .../iotdb/db/storageengine/StorageEngine.java | 5 + .../dataregion/memtable/TsFileProcessor.java | 2 + 3 files changed, 146 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java index 0b170bc54a1..725070112f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java @@ -346,11 +346,34 @@ public class WritingMetrics implements IMetricSet { private Histogram flushThreholdHistogram = DoNothingMetricManager.DO_NOTHING_HISTOGRAM; private Histogram rejectThreholdHistogram = DoNothingMetricManager.DO_NOTHING_HISTOGRAM; + private Timer memtableLiveTimer = DoNothingMetricManager.DO_NOTHING_TIMER; + public void bindDataRegionMetrics() { List<DataRegion> allDataRegions = StorageEngine.getInstance().getAllDataRegions(); List<DataRegionId> allDataRegionIds = StorageEngine.getInstance().getAllDataRegionIds(); allDataRegions.forEach(this::createDataRegionMemoryCostMetrics); allDataRegionIds.forEach(this::createFlushingMemTableStatusMetrics); + allDataRegionIds.forEach(this::createSeriesFullFlushMemTableCounterMetrics); + allDataRegionIds.forEach(this::createTimedFlushMemTableCounterMetrics); + allDataRegionIds.forEach(this::createWalFlushMemTableCounterMetrics); + allDataRegionIds.forEach(this::createActiveMemtableCounterMetrics); + allDataRegionIds.forEach(this::createActiveTimePartitionCounterMetrics); + + flushThreholdHistogram = + MetricService.getInstance() + .getOrCreateHistogram( + Metric.FLUSH_THRESHOLD.toString(), MetricLevel.IMPORTANT, "flush_threshold"); + rejectThreholdHistogram = + MetricService.getInstance() + .getOrCreateHistogram( + Metric.REJECT_THRESHOLD.toString(), MetricLevel.IMPORTANT, "reject_threshold"); + + memtableLiveTimer = + MetricService.getInstance() + .getOrCreateTimer( + Metric.MEMTABLE_LIVE_DURATION.toString(), + MetricLevel.IMPORTANT, + Tag.STAGE.toString()); } public void unbindDataRegionMetrics() { @@ -359,7 +382,16 @@ public class WritingMetrics implements IMetricSet { dataRegionId -> { removeDataRegionMemoryCostMetrics(dataRegionId); removeFlushingMemTableStatusMetrics(dataRegionId); + removeSeriesFullFlushMemTableCounterMetrics(dataRegionId); + removeTimedFlushMemTableCounterMetrics(dataRegionId); + removeWalFlushMemTableCounterMetrics(dataRegionId); + removeActiveMemtableCounterMetrics(dataRegionId); + removeActiveTimePartitionCounterMetrics(dataRegionId); }); + MetricService.getInstance().remove(MetricType.HISTOGRAM, Metric.FLUSH_THRESHOLD.toString()); + MetricService.getInstance().remove(MetricType.HISTOGRAM, Metric.REJECT_THRESHOLD.toString()); + MetricService.getInstance() + .remove(MetricType.TIMER, Metric.MEMTABLE_LIVE_DURATION.toString(), Tag.STAGE.toString()); } public void createDataRegionMemoryCostMetrics(DataRegion dataRegion) { @@ -437,6 +469,106 @@ public class WritingMetrics implements IMetricSet { dataRegionId.toString())); } + public void createWalFlushMemTableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .getOrCreateCounter( + Metric.WAL_FLUSH_MEMTABLE_COUNT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void createTimedFlushMemTableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .getOrCreateCounter( + Metric.TIMED_FLUSH_MEMTABLE_COUNT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void createSeriesFullFlushMemTableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .getOrCreateCounter( + Metric.SERIES_FULL_FLUSH_MEMTABLE.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void createActiveMemtableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .getOrCreateCounter( + Metric.ACTIVE_MEMTABLE_COUNT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void createActiveTimePartitionCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .getOrCreateCounter( + Metric.ACTIVE_MEMTABLE_COUNT.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void removeSeriesFullFlushMemTableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .remove( + MetricType.COUNTER, + Metric.SERIES_FULL_FLUSH_MEMTABLE.toString(), + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void removeTimedFlushMemTableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .remove( + MetricType.COUNTER, + Metric.TIMED_FLUSH_MEMTABLE_COUNT.toString(), + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void removeWalFlushMemTableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .remove( + MetricType.COUNTER, + Metric.WAL_FLUSH_MEMTABLE_COUNT.toString(), + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void removeActiveMemtableCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .remove( + MetricType.COUNTER, + Metric.ACTIVE_MEMTABLE_COUNT.toString(), + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + + public void removeActiveTimePartitionCounterMetrics(DataRegionId dataRegionId) { + MetricService.getInstance() + .remove( + MetricType.COUNTER, + Metric.ACTIVE_TIME_PARTITION_COUNT.toString(), + Tag.NAME.toString(), + Tag.REGION.toString(), + dataRegionId.toString()); + } + public void removeFlushingMemTableStatusMetrics(DataRegionId dataRegionId) { Arrays.asList( MEM_TABLE_SIZE, @@ -663,6 +795,13 @@ public class WritingMetrics implements IMetricSet { public void recordRejectThreshold(double rejectThreshold) { rejectThreholdHistogram.update((long) rejectThreshold); } + + public void recordMemTableLiveDuration(long durationMillis) { + memtableLiveTimer.updateMillis(durationMillis); + } + + public void recordTimedFlushMemTableCount(DataRegionId dataRegionId, int number) {} + // endregion @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 90ff0740b14..bab14db6a66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -644,6 +644,11 @@ public class StorageEngine implements IService { region.markDeleted(); WRITING_METRICS.removeDataRegionMemoryCostMetrics(regionId); WRITING_METRICS.removeFlushingMemTableStatusMetrics(regionId); + WRITING_METRICS.removeActiveMemtableCounterMetrics(regionId); + WRITING_METRICS.removeWalFlushMemTableCounterMetrics(regionId); + WRITING_METRICS.removeTimedFlushMemTableCounterMetrics(regionId); + WRITING_METRICS.removeSeriesFullFlushMemTableCounterMetrics(regionId); + WRITING_METRICS.removeActiveTimePartitionCounterMetrics(regionId); try { region.abortCompaction(); region.syncDeleteDataFiles(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 7fadcda4267..21e8bb3e445 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1057,6 +1057,8 @@ public class TsFileProcessor { if (!(tobeFlushed.isSignalMemTable() || tobeFlushed.isEmpty())) { totalMemTableSize += tobeFlushed.memSize(); } + WritingMetrics.getInstance() + .recordMemTableLiveDuration(System.currentTimeMillis() - getWorkMemTableCreatedTime()); workMemTable = null; return FlushManager.getInstance().registerTsFileProcessor(this); }
