This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch cp_fix_wal_stuck in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2be735155372a5b84900f182da3b791b51604c0e Author: Haonan <[email protected]> AuthorDate: Thu Dec 12 14:25:04 2024 +0800 fix wal queue stuck and fix metric (#14387) --- .../org/apache/iotdb/db/service/metrics/WritingMetrics.java | 12 ++++++------ .../dataregion/wal/utils/MemoryControlledWALEntryQueue.java | 2 +- .../iotdb/db/storageengine/rescon/memory/SystemInfo.java | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java index 3d7e218b71d..8820be02fa8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/WritingMetrics.java @@ -257,6 +257,12 @@ public class WritingMetrics implements IMetricSet { MetricLevel.IMPORTANT, Tag.NAME.toString(), WRITE_WAL_BUFFER_COST_NS); + walQueueMaxMemSizeGauge = + metricService.getOrCreateGauge( + Metric.WAL_QUEUE_MEM_COST.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + WAL_QUEUE_MAX_MEM_COST); SystemInfo systemInfo = SystemInfo.getInstance(); metricService.createAutoGauge( Metric.WAL_QUEUE_MEM_COST.toString(), @@ -265,12 +271,6 @@ public class WritingMetrics implements IMetricSet { SystemInfo::getCurrentWalQueueMemoryCost, Tag.NAME.toString(), WAL_QUEUE_CURRENT_MEM_COST); - walQueueMaxMemSizeGauge = - metricService.getOrCreateGauge( - Metric.WAL_QUEUE_MEM_COST.toString(), - MetricLevel.IMPORTANT, - Tag.NAME.toString(), - WAL_QUEUE_MAX_MEM_COST); } private void unbindWALMetrics(AbstractMetricService metricService) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/MemoryControlledWALEntryQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/MemoryControlledWALEntryQueue.java index 3fabd49f3f7..ddb1310564b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/MemoryControlledWALEntryQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/MemoryControlledWALEntryQueue.java @@ -29,7 +29,7 @@ import java.util.concurrent.TimeUnit; public class MemoryControlledWALEntryQueue { private final BlockingQueue<WALEntry> queue; - private final Object nonFullCondition = new Object(); + private static final Object nonFullCondition = new Object(); public MemoryControlledWALEntryQueue() { queue = new LinkedBlockingQueue<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java index e0b9d50cb81..56e2ef4dae6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java @@ -532,6 +532,7 @@ public class SystemInfo { REJECT_THRESHOLD = memorySizeForMemtable * config.getRejectProportion(); WritingMetrics.getInstance().recordFlushThreshold(FLUSH_THRESHOLD); WritingMetrics.getInstance().recordRejectThreshold(REJECT_THRESHOLD); + WritingMetrics.getInstance().recordWALQueueMaxMemorySize(memorySizeForWalBufferQueue); } public synchronized void releaseTemporaryMemoryForFlushing(long estimatedTemporaryMemSize) { @@ -540,6 +541,7 @@ public class SystemInfo { REJECT_THRESHOLD = memorySizeForMemtable * config.getRejectProportion(); WritingMetrics.getInstance().recordFlushThreshold(FLUSH_THRESHOLD); WritingMetrics.getInstance().recordRejectThreshold(REJECT_THRESHOLD); + WritingMetrics.getInstance().recordWALQueueMaxMemorySize(memorySizeForWalBufferQueue); } public long getTotalMemTableSize() {
