This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_flush_error_due_to_compressionRatio in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b478717935ff566054988500af1838c3bc1454d3 Author: HTHou <[email protected]> AuthorDate: Wed Jul 17 11:23:16 2024 +0800 remove deduplicated timestamp logic --- .../db/storageengine/dataregion/flush/CompressionRatio.java | 4 ---- .../dataregion/memtable/AlignedWritableMemChunk.java | 13 ------------- .../storageengine/dataregion/memtable/WritableMemChunk.java | 6 ------ .../dataregion/flush/CompressionRatioTest.java | 13 ++++++------- 4 files changed, 6 insertions(+), 30 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java index 2e133e44eeb..6253717386e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java @@ -215,10 +215,6 @@ public class CompressionRatio { totalDiskSize = 0L; } - public static void decreaseDuplicatedMemorySize(long size) { - totalMemorySize.addAndGet(-size); - } - public static CompressionRatio getInstance() { return CompressionRatioHolder.INSTANCE; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index ba78113bd03..77b98b8b70e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -19,10 +19,8 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable; -import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALWriteUtils; -import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; @@ -390,17 +388,6 @@ public class AlignedWritableMemChunk implements IWritableMemChunk { list.getValueIndex(sortedRowIndex); } if (timeDuplicateInfo[sortedRowIndex]) { - if (!list.isNullValue(list.getValueIndex(sortedRowIndex), columnIndex)) { - long recordSize = - MemUtils.getRecordSize( - tsDataType, - tsDataType.isBinary() - ? list.getBinaryByValueIndex( - list.getValueIndex(sortedRowIndex), columnIndex) - : null, - true); - CompressionRatio.decreaseDuplicatedMemorySize(recordSize); - } continue; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index 3f05af15fb6..bc665cff5cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -20,9 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; -import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.datastructure.TVList; import org.apache.tsfile.enums.TSDataType; @@ -340,10 +338,6 @@ public class WritableMemChunk implements IWritableMemChunk { // skip duplicated data if ((sortedRowIndex + 1 < list.rowCount() && (time == list.getTime(sortedRowIndex + 1)))) { - long recordSize = - MemUtils.getRecordSize( - tsDataType, tsDataType.isBinary() ? list.getBinary(sortedRowIndex) : null, true); - CompressionRatio.decreaseDuplicatedMemorySize(recordSize); continue; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java index b1da6c32111..f088fee719a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatioTest.java @@ -109,14 +109,14 @@ public class CompressionRatioTest { public void testRestoreIllegal1() throws IOException { Files.createFile( new File( - directory, - String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, 10, 50)) + directory, + String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, 10, 50)) .toPath()); Files.createFile( new File( - directory, - String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, -1000, 100)) + directory, + String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, -1000, 100)) .toPath()); compressionRatio.restore(); @@ -131,8 +131,8 @@ public class CompressionRatioTest { Files.createFile( new File( - directory, - String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, -1000, 100)) + directory, + String.format(Locale.ENGLISH, CompressionRatio.RATIO_FILE_PATH_FORMAT, -1000, 100)) .toPath()); compressionRatio.restore(); @@ -140,5 +140,4 @@ public class CompressionRatioTest { // if compression ratio from file is negative, assume the compression ratio is 0 / 0 = NaN assertEquals(Double.NaN, compressionRatio.getRatio(), 0.1); } - }
