This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_last_flush_time_memory_count in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 398f26568a169319d2b2fe0bbe19575bf02ff18f Author: HTHou <[email protected]> AuthorDate: Mon Jan 29 15:45:20 2024 +0800 Fix more issue of last flush time memory calculate --- .../db/storageengine/dataregion/DataRegion.java | 33 ++++++++++++------ .../rescon/memory/TimePartitionInfo.java | 12 +------ .../rescon/memory/TimePartitionManager.java | 6 ---- .../rescon/memory/TimePartitionManagerTest.java | 39 +++++----------------- 4 files changed, 31 insertions(+), 59 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 73a24f63c13..97355ac2a57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -539,8 +539,7 @@ public class DataRegion implements IDataRegionForQuery { latestPartitionId, false, Long.MAX_VALUE, - lastFlushTimeMap.getMemSize(latestPartitionId), - true)); + lastFlushTimeMap.getMemSize(latestPartitionId))); } } // wait until all unsealed TsFiles have been recovered @@ -822,10 +821,26 @@ public class DataRegion implements IDataRegionForQuery { recoverSealedTsFiles(tsFileResource, context, isSeq); } if (config.isEnableSeparateData()) { - lastFlushTimeMap.checkAndCreateFlushedTimePartition(partitionId); + if (!lastFlushTimeMap.checkAndCreateFlushedTimePartition(partitionId)) { + TimePartitionManager.getInstance() + .registerTimePartitionInfo( + new TimePartitionInfo( + new DataRegionId(Integer.parseInt(dataRegionId)), + partitionId, + false, + Long.MAX_VALUE, + lastFlushTimeMap.getMemSize(partitionId))); + } for (TsFileResource tsFileResource : resourceList) { updateLastFlushTime(tsFileResource, isSeq); } + TimePartitionManager.getInstance() + .updateAfterFlushing( + new DataRegionId(Integer.parseInt(dataRegionId)), + partitionId, + System.currentTimeMillis(), + lastFlushTimeMap.getMemSize(partitionId), + false); } } @@ -876,8 +891,7 @@ public class DataRegion implements IDataRegionForQuery { timePartitionId, true, Long.MAX_VALUE, - 0, - tsFileManager.isLatestTimePartition(timePartitionId))); + 0)); } boolean isSequence = @@ -969,8 +983,7 @@ public class DataRegion implements IDataRegionForQuery { beforeTimePartition, true, Long.MAX_VALUE, - 0, - tsFileManager.isLatestTimePartition(beforeTimePartition))); + 0)); } long lastFlushTime = @@ -3080,8 +3093,7 @@ public class DataRegion implements IDataRegionForQuery { timePartitionId, true, Long.MAX_VALUE, - 0, - tsFileManager.isLatestTimePartition(timePartitionId))); + 0)); } boolean isSequence = @@ -3156,8 +3168,7 @@ public class DataRegion implements IDataRegionForQuery { timePartitionIds[i], true, Long.MAX_VALUE, - 0, - tsFileManager.isLatestTimePartition(timePartitionIds[i]))); + 0)); } areSequence[i] = config.isEnableSeparateData() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionInfo.java index aa602fad9d5..9a25b6501a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionInfo.java @@ -31,8 +31,6 @@ public class TimePartitionInfo { long lastSystemFlushTime; - boolean isLatestPartition; - long memSize; public TimePartitionInfo( @@ -40,14 +38,12 @@ public class TimePartitionInfo { long partitionId, boolean isActive, long lastSystemFlushTime, - long memsize, - boolean isLatestPartition) { + long memsize) { this.dataRegionId = dataRegionId; this.partitionId = partitionId; this.isActive = isActive; this.lastSystemFlushTime = lastSystemFlushTime; this.memSize = memsize; - this.isLatestPartition = isLatestPartition; } public int comparePriority(TimePartitionInfo timePartitionInfo) { @@ -59,12 +55,6 @@ public class TimePartitionInfo { if (cmp != 0) { return cmp; } - - cmp = Boolean.compare(isLatestPartition, timePartitionInfo.isLatestPartition); - if (cmp != 0) { - return cmp; - } - return Long.compare(lastSystemFlushTime, timePartitionInfo.lastSystemFlushTime); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java index 55aeec94113..622afa05367 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java @@ -56,12 +56,6 @@ public class TimePartitionManager { timePartitionInfoMap.computeIfAbsent( timePartitionInfo.dataRegionId, k -> new TreeMap<>()); - Map.Entry<Long, TimePartitionInfo> entry = - timePartitionInfoMapForRegion.floorEntry(timePartitionInfo.partitionId); - if (entry != null) { - entry.getValue().isLatestPartition = false; - } - timePartitionInfoMapForRegion.put(timePartitionInfo.partitionId, timePartitionInfo); // We need to ensure that the following method is called before diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java index 90dc24e89a5..2c2a0517c87 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManagerTest.java @@ -58,29 +58,10 @@ public class TimePartitionManagerTest { timePartitionManager.clear(); } - @Test - public void testRegisterPartitionInfo() { - TimePartitionInfo timePartitionInfo1 = - new TimePartitionInfo(new DataRegionId(1), 0L, true, Long.MAX_VALUE, 0, true); - timePartitionManager.registerTimePartitionInfo(timePartitionInfo1); - - assertEquals( - timePartitionInfo1, timePartitionManager.getTimePartitionInfo(new DataRegionId(1), 0L)); - - TimePartitionInfo timePartitionInfo2 = - new TimePartitionInfo(new DataRegionId(1), 1L, true, Long.MAX_VALUE, 0, true); - timePartitionManager.registerTimePartitionInfo(timePartitionInfo2); - - Assert.assertFalse( - timePartitionManager.getTimePartitionInfo(new DataRegionId(1), 0L).isLatestPartition); - Assert.assertTrue( - timePartitionManager.getTimePartitionInfo(new DataRegionId(1), 1L).isLatestPartition); - } - @Test public void testUpdate() { TimePartitionInfo timePartitionInfo = - new TimePartitionInfo(new DataRegionId(1), 0L, true, Long.MAX_VALUE, 0, true); + new TimePartitionInfo(new DataRegionId(1), 0L, true, Long.MAX_VALUE, 0); timePartitionManager.registerTimePartitionInfo(timePartitionInfo); timePartitionManager.updateAfterFlushing(new DataRegionId(1), 0L, 100L, 100L, false); @@ -88,7 +69,6 @@ public class TimePartitionManagerTest { TimePartitionInfo timePartitionInfo1 = timePartitionManager.getTimePartitionInfo(new DataRegionId(1), 0L); - assertTrue(timePartitionInfo1.isLatestPartition); assertEquals(timePartitionInfo1.lastSystemFlushTime, 100L); assertEquals(timePartitionInfo1.memSize, 100); assertFalse(timePartitionInfo1.isActive); @@ -103,7 +83,7 @@ public class TimePartitionManagerTest { public void testMemoryControl() { for (int i = 0; i < 5; i++) { TimePartitionInfo timePartitionInfo = - new TimePartitionInfo(new DataRegionId(i), 0L, true, Long.MAX_VALUE, 0, true); + new TimePartitionInfo(new DataRegionId(i), 0L, true, Long.MAX_VALUE, 0); timePartitionManager.registerTimePartitionInfo(timePartitionInfo); } timePartitionManager.updateAfterFlushing(new DataRegionId(0), 0L, 100L, 20L, false); @@ -112,7 +92,7 @@ public class TimePartitionManagerTest { timePartitionManager.updateAfterFlushing(new DataRegionId(3), 0L, 103L, 20L, false); timePartitionManager.updateAfterFlushing(new DataRegionId(4), 0L, 104L, 20L, true); timePartitionManager.registerTimePartitionInfo( - new TimePartitionInfo(new DataRegionId(0), 1L, true, Long.MAX_VALUE, 0, true)); + new TimePartitionInfo(new DataRegionId(0), 1L, true, Long.MAX_VALUE, 0)); timePartitionManager.updateAfterFlushing(new DataRegionId(0), 1L, 105L, 20L, true); @@ -130,18 +110,15 @@ public class TimePartitionManagerTest { @Test public void testCompareTimePartitionInfo() { TimePartitionInfo timePartitionInfo = - new TimePartitionInfo(new DataRegionId(1), 0L, true, 100, 0, true); + new TimePartitionInfo(new DataRegionId(1), 0L, true, 100, 0); TimePartitionInfo timePartitionInfo1 = - new TimePartitionInfo(new DataRegionId(1), 0L, false, 100, 0, true); + new TimePartitionInfo(new DataRegionId(1), 0L, false, 100, 0); Assert.assertEquals(1, timePartitionInfo.comparePriority(timePartitionInfo1)); TimePartitionInfo timePartitionInfo2 = - new TimePartitionInfo(new DataRegionId(1), 1L, true, 100, 0, true); + new TimePartitionInfo(new DataRegionId(1), 1L, true, 100, 0); Assert.assertEquals(-1, timePartitionInfo.comparePriority(timePartitionInfo2)); TimePartitionInfo timePartitionInfo3 = - new TimePartitionInfo(new DataRegionId(1), 0L, true, 100, 0, false); - Assert.assertEquals(1, timePartitionInfo.comparePriority(timePartitionInfo3)); - TimePartitionInfo timePartitionInfo4 = - new TimePartitionInfo(new DataRegionId(1), 0L, true, 101, 0, true); - Assert.assertEquals(-1, timePartitionInfo.comparePriority(timePartitionInfo4)); + new TimePartitionInfo(new DataRegionId(1), 0L, true, 101, 0); + Assert.assertEquals(-1, timePartitionInfo.comparePriority(timePartitionInfo3)); } }
