This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.3.2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 36a18f127a4caf2260d0143ddcf756573b49b775 Author: shuwenwei <[email protected]> AuthorDate: Mon Jul 8 22:12:18 2024 +0800 Remove usage of CompressionRatio in compaction (#12875) * add log * add default value * remove empty line * set old file name * remove usage of CompressionRatio in compaction * modify calculatingDataMemoryCost * fix ut --- .../estimator/AbstractCompactionEstimator.java | 6 +++--- .../FastCompactionInnerCompactionEstimator.java | 15 +++++++-------- .../estimator/FastCrossSpaceCompactionEstimator.java | 15 +++++++-------- .../estimator/ReadChunkInnerCompactionEstimator.java | 19 +++++++++---------- .../RepairUnsortedFileCompactionEstimator.java | 11 ++++------- .../dataregion/flush/CompressionRatio.java | 7 +++++++ ...paceCompactionWithFastPerformerValidationTest.java | 4 ++++ ...ompactionWithReadPointPerformerValidationTest.java | 4 ++++ 8 files changed, 45 insertions(+), 36 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java index e72f163a21c..6c28e56a5c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java @@ -21,12 +21,13 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimat import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; +import org.apache.iotdb.tsfile.common.conf.TSFileConfig; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.file.metadata.IDeviceID; import org.apache.iotdb.tsfile.read.TsFileSequenceReader; @@ -58,8 +59,7 @@ public abstract class AbstractCompactionEstimator { protected Map<TsFileResource, DeviceTimeIndex> deviceTimeIndexCache = new HashMap<>(); protected IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - - protected long compressionRatio = (long) CompressionRatio.getInstance().getRatio() + 1; + protected TSFileConfig tsFileConfig = TSFileDescriptor.getInstance().getConfig(); protected abstract long calculatingMetadataMemoryCost(CompactionTaskInfo taskInfo); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java index 8be9b210a52..8530219671e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java @@ -55,21 +55,20 @@ public class FastCompactionInnerCompactionEstimator extends AbstractInnerSpaceEs } long maxConcurrentSeriesNum = Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum()); - long averageUncompressedChunkSize = - taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum(); + long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum(); long maxConcurrentSeriesSizeOfTotalFiles = - averageUncompressedChunkSize - * taskInfo.getFileInfoList().size() - * maxConcurrentSeriesNum - * taskInfo.getMaxChunkMetadataNumInSeries() - / compressionRatio; + averageChunkSize + * taskInfo.getFileInfoList().size() + * maxConcurrentSeriesNum + * taskInfo.getMaxChunkMetadataNumInSeries() + + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte(); long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum; long targetChunkWriterSize = Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize); long maxConcurrentChunkSizeFromSourceFile = - averageUncompressedChunkSize + (averageChunkSize + tsFileConfig.getPageSizeInByte()) * maxConcurrentSeriesNum * calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java index 1ac3b5d5be1..4d9b89fda52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java @@ -56,21 +56,20 @@ public class FastCrossSpaceCompactionEstimator extends AbstractCrossSpaceEstimat long maxConcurrentSeriesNum = Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum()); - long averageUncompressedChunkSize = - taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum(); + long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum(); long maxConcurrentSeriesSizeOfTotalFiles = - averageUncompressedChunkSize - * taskInfo.getFileInfoList().size() - * maxConcurrentSeriesNum - * taskInfo.getMaxChunkMetadataNumInSeries() - / compressionRatio; + averageChunkSize + * taskInfo.getFileInfoList().size() + * maxConcurrentSeriesNum + * taskInfo.getMaxChunkMetadataNumInSeries() + + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte(); long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum; long targetChunkWriterSize = Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize); long maxConcurrentChunkSizeFromSourceFile = - averageUncompressedChunkSize + (averageChunkSize + tsFileConfig.getPageSizeInByte()) * maxConcurrentSeriesNum * calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java index e432b1eac5d..e77a87f4c58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java @@ -51,22 +51,21 @@ public class ReadChunkInnerCompactionEstimator extends AbstractInnerSpaceEstimat if (taskInfo.getTotalChunkNum() == 0) { return taskInfo.getModificationFileSize(); } - long averageUncompressedChunkSize = - taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum(); + long maxConcurrentSeriesNum = taskInfo.getMaxConcurrentSeriesNum(); + long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum(); long maxConcurrentSeriesSizeOfTotalFiles = - averageUncompressedChunkSize - * taskInfo.getFileInfoList().size() - * taskInfo.getMaxConcurrentSeriesNum() - * taskInfo.getMaxChunkMetadataNumInSeries() - / compressionRatio; - long maxTargetChunkWriterSize = - config.getTargetChunkSize() * taskInfo.getMaxConcurrentSeriesNum(); + averageChunkSize + * taskInfo.getFileInfoList().size() + * maxConcurrentSeriesNum + * taskInfo.getMaxChunkMetadataNumInSeries() + + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte(); + long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum; long targetChunkWriterSize = Math.min(maxConcurrentSeriesSizeOfTotalFiles, maxTargetChunkWriterSize); long chunkSizeFromSourceFile = - averageUncompressedChunkSize * taskInfo.getMaxConcurrentSeriesNum(); + (averageChunkSize + tsFileConfig.getPageSizeInByte()) * maxConcurrentSeriesNum; return targetChunkWriterSize + chunkSizeFromSourceFile + taskInfo.getModificationFileSize(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java index 3eadd41ea28..c7852753822 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java @@ -50,19 +50,16 @@ public class RepairUnsortedFileCompactionEstimator extends AbstractInnerSpaceEst } long maxConcurrentSeriesNum = Math.max(config.getSubCompactionTaskNum(), taskInfo.getMaxConcurrentSeriesNum()); - long averageUncompressedChunkSize = - taskInfo.getTotalFileSize() * compressionRatio / taskInfo.getTotalChunkNum(); + long averageChunkSize = taskInfo.getTotalFileSize() / taskInfo.getTotalChunkNum(); long maxConcurrentSeriesSize = - averageUncompressedChunkSize - * maxConcurrentSeriesNum - * taskInfo.getMaxChunkMetadataNumInSeries() - / compressionRatio; + averageChunkSize * maxConcurrentSeriesNum * taskInfo.getMaxChunkMetadataNumInSeries() + + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte(); long maxTargetChunkWriterSize = config.getTargetChunkSize() * maxConcurrentSeriesNum; long targetChunkWriterSize = Math.min(maxConcurrentSeriesSize, maxTargetChunkWriterSize); long inMemorySortedDataSize = - averageUncompressedChunkSize + (averageChunkSize + tsFileConfig.getPageSizeInByte()) * Math.min( taskInfo.getMaxChunkMetadataNumInDevice(), taskInfo.getMaxChunkMetadataNumInSeries() * maxConcurrentSeriesNum); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java index 5d6c40e1487..2f1e173f249 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java @@ -85,6 +85,12 @@ public class CompressionRatio { * call this method. */ public synchronized void updateRatio(long memorySize, long diskSize) throws IOException { + if (memorySize < 0 || totalMemorySize.get() < 0) { + LOGGER.warn( + "The compression ratio is negative, current memTableSize: {}, totalMemTableSize: {}", + memorySize, + totalMemorySize); + } File oldFile = SystemFileFactory.INSTANCE.getFile(directory, oldFileName); totalMemorySize.addAndGet(memorySize); @@ -149,6 +155,7 @@ public class CompressionRatio { "After restoring from compression ratio file, total memory size = {}, total disk size = {}", totalMemorySize, totalDiskSize); + oldFileName = ratioFiles[maxRatioIndex].getName(); deleteRedundantFilesByIndex(ratioFiles, maxRatioIndex); } else { // If there is no new file, try to restore from the old version file File[] ratioFilesBeforeV121 = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java index d1bf1564d2a..cf9358340f6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils; +import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.tools.validate.TsFileValidationTool; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -79,6 +80,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac private final String oldThreadName = Thread.currentThread().getName(); private ICrossCompactionPerformer performer = new FastCompactionPerformer(true); + private long compactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction(); @Before public void setUp() @@ -87,6 +89,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0); IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024); TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30); + SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024); Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); } @@ -95,6 +98,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac super.tearDown(); Thread.currentThread().setName(oldThreadName); FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); + SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory); TsFileValidationTool.badFileNum = 0; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java index c1090e39450..5c0edd7bc24 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionF import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.tools.validate.TsFileValidationTool; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -77,6 +78,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest private final String oldThreadName = Thread.currentThread().getName(); private ICrossCompactionPerformer performer = new ReadPointCompactionPerformer(); + private long compactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction(); @Before public void setUp() @@ -85,6 +87,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0); IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024); TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30); + SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024); Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); } @@ -93,6 +96,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest super.tearDown(); Thread.currentThread().setName(oldThreadName); FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); + SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory); TsFileValidationTool.badFileNum = 0; }
