This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d1fccd50425 Remove usage of CompressionRatio in compaction (#12875)
d1fccd50425 is described below
commit d1fccd504255e10632659e8d4db7fd56fd1819bc
Author: shuwenwei <[email protected]>
AuthorDate: Mon Jul 8 22:12:18 2024 +0800
Remove usage of CompressionRatio in compaction (#12875)
* add log
* add default value
* remove empty line
* set old file name
* remove usage of CompressionRatio in compaction
* modify calculatingDataMemoryCost
* fix ut
---
.../estimator/AbstractCompactionEstimator.java | 6 +++---
.../FastCompactionInnerCompactionEstimator.java | 15 +++++++--------
.../estimator/FastCrossSpaceCompactionEstimator.java | 15 +++++++--------
.../estimator/ReadChunkInnerCompactionEstimator.java | 19 +++++++++----------
.../RepairUnsortedFileCompactionEstimator.java | 11 ++++-------
.../dataregion/flush/CompressionRatio.java | 7 +++++++
...paceCompactionWithFastPerformerValidationTest.java | 4 ++++
...ompactionWithReadPointPerformerValidationTest.java | 4 ++++
8 files changed, 45 insertions(+), 36 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
index 4b7a9a1cbdd..81bb753de7b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
@@ -21,7 +21,6 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimat
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
@@ -29,6 +28,8 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeInd
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
import org.apache.commons.collections4.map.LRUMap;
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.common.conf.TSFileDescriptor;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.read.TsFileSequenceReader;
@@ -58,8 +59,7 @@ public abstract class AbstractCompactionEstimator {
protected Map<TsFileResource, DeviceTimeIndex> deviceTimeIndexCache = new
HashMap<>();
protected IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-
- protected long compressionRatio = (long)
CompressionRatio.getInstance().getRatio() + 1;
+ protected TSFileConfig tsFileConfig =
TSFileDescriptor.getInstance().getConfig();
protected abstract long calculatingMetadataMemoryCost(CompactionTaskInfo
taskInfo);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
index 8be9b210a52..8530219671e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
@@ -55,21 +55,20 @@ public class FastCompactionInnerCompactionEstimator extends
AbstractInnerSpaceEs
}
long maxConcurrentSeriesNum =
Math.max(config.getSubCompactionTaskNum(),
taskInfo.getMaxConcurrentSeriesNum());
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio /
taskInfo.getTotalChunkNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() /
taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSizeOfTotalFiles =
- averageUncompressedChunkSize
- * taskInfo.getFileInfoList().size()
- * maxConcurrentSeriesNum
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
+ averageChunkSize
+ * taskInfo.getFileInfoList().size()
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
long maxTargetChunkWriterSize = config.getTargetChunkSize() *
maxConcurrentSeriesNum;
long targetChunkWriterSize =
Math.min(maxConcurrentSeriesSizeOfTotalFiles,
maxTargetChunkWriterSize);
long maxConcurrentChunkSizeFromSourceFile =
- averageUncompressedChunkSize
+ (averageChunkSize + tsFileConfig.getPageSizeInByte())
* maxConcurrentSeriesNum
*
calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
index 1ac3b5d5be1..4d9b89fda52 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
@@ -56,21 +56,20 @@ public class FastCrossSpaceCompactionEstimator extends
AbstractCrossSpaceEstimat
long maxConcurrentSeriesNum =
Math.max(config.getSubCompactionTaskNum(),
taskInfo.getMaxConcurrentSeriesNum());
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio /
taskInfo.getTotalChunkNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() /
taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSizeOfTotalFiles =
- averageUncompressedChunkSize
- * taskInfo.getFileInfoList().size()
- * maxConcurrentSeriesNum
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
+ averageChunkSize
+ * taskInfo.getFileInfoList().size()
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
long maxTargetChunkWriterSize = config.getTargetChunkSize() *
maxConcurrentSeriesNum;
long targetChunkWriterSize =
Math.min(maxConcurrentSeriesSizeOfTotalFiles,
maxTargetChunkWriterSize);
long maxConcurrentChunkSizeFromSourceFile =
- averageUncompressedChunkSize
+ (averageChunkSize + tsFileConfig.getPageSizeInByte())
* maxConcurrentSeriesNum
*
calculatingMaxOverlapFileNumInSubCompactionTask(taskInfo.getResources());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
index e432b1eac5d..e77a87f4c58 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
@@ -51,22 +51,21 @@ public class ReadChunkInnerCompactionEstimator extends
AbstractInnerSpaceEstimat
if (taskInfo.getTotalChunkNum() == 0) {
return taskInfo.getModificationFileSize();
}
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio /
taskInfo.getTotalChunkNum();
+ long maxConcurrentSeriesNum = taskInfo.getMaxConcurrentSeriesNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() /
taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSizeOfTotalFiles =
- averageUncompressedChunkSize
- * taskInfo.getFileInfoList().size()
- * taskInfo.getMaxConcurrentSeriesNum()
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
- long maxTargetChunkWriterSize =
- config.getTargetChunkSize() * taskInfo.getMaxConcurrentSeriesNum();
+ averageChunkSize
+ * taskInfo.getFileInfoList().size()
+ * maxConcurrentSeriesNum
+ * taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
+ long maxTargetChunkWriterSize = config.getTargetChunkSize() *
maxConcurrentSeriesNum;
long targetChunkWriterSize =
Math.min(maxConcurrentSeriesSizeOfTotalFiles,
maxTargetChunkWriterSize);
long chunkSizeFromSourceFile =
- averageUncompressedChunkSize * taskInfo.getMaxConcurrentSeriesNum();
+ (averageChunkSize + tsFileConfig.getPageSizeInByte()) *
maxConcurrentSeriesNum;
return targetChunkWriterSize + chunkSizeFromSourceFile +
taskInfo.getModificationFileSize();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
index 3eadd41ea28..c7852753822 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/RepairUnsortedFileCompactionEstimator.java
@@ -50,19 +50,16 @@ public class RepairUnsortedFileCompactionEstimator extends
AbstractInnerSpaceEst
}
long maxConcurrentSeriesNum =
Math.max(config.getSubCompactionTaskNum(),
taskInfo.getMaxConcurrentSeriesNum());
- long averageUncompressedChunkSize =
- taskInfo.getTotalFileSize() * compressionRatio /
taskInfo.getTotalChunkNum();
+ long averageChunkSize = taskInfo.getTotalFileSize() /
taskInfo.getTotalChunkNum();
long maxConcurrentSeriesSize =
- averageUncompressedChunkSize
- * maxConcurrentSeriesNum
- * taskInfo.getMaxChunkMetadataNumInSeries()
- / compressionRatio;
+ averageChunkSize * maxConcurrentSeriesNum *
taskInfo.getMaxChunkMetadataNumInSeries()
+ + maxConcurrentSeriesNum * tsFileConfig.getPageSizeInByte();
long maxTargetChunkWriterSize = config.getTargetChunkSize() *
maxConcurrentSeriesNum;
long targetChunkWriterSize = Math.min(maxConcurrentSeriesSize,
maxTargetChunkWriterSize);
long inMemorySortedDataSize =
- averageUncompressedChunkSize
+ (averageChunkSize + tsFileConfig.getPageSizeInByte())
* Math.min(
taskInfo.getMaxChunkMetadataNumInDevice(),
taskInfo.getMaxChunkMetadataNumInSeries() *
maxConcurrentSeriesNum);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
index e40027aa51c..1e37cb9e8ec 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/CompressionRatio.java
@@ -85,6 +85,12 @@ public class CompressionRatio {
* call this method.
*/
public synchronized void updateRatio(long memorySize, long diskSize) throws
IOException {
+ if (memorySize < 0 || totalMemorySize.get() < 0) {
+ LOGGER.warn(
+ "The compression ratio is negative, current memTableSize: {},
totalMemTableSize: {}",
+ memorySize,
+ totalMemorySize);
+ }
File oldFile = SystemFileFactory.INSTANCE.getFile(directory, oldFileName);
totalMemorySize.addAndGet(memorySize);
@@ -149,6 +155,7 @@ public class CompressionRatio {
"After restoring from compression ratio file, total memory size =
{}, total disk size = {}",
totalMemorySize,
totalDiskSize);
+ oldFileName = ratioFiles[maxRatioIndex].getName();
deleteRedundantFilesByIndex(ratioFiles, maxRatioIndex);
} else { // If there is no new file, try to restore from the old version
file
File[] ratioFilesBeforeV121 =
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index b47d2ecc82d..6808c8d23d5 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -47,6 +47,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.tsfile.common.conf.TSFileDescriptor;
@@ -80,6 +81,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
private final String oldThreadName = Thread.currentThread().getName();
private ICrossCompactionPerformer performer = new
FastCompactionPerformer(true);
+ private long compactionMemory =
SystemInfo.getInstance().getMemorySizeForCompaction();
@Before
public void setUp()
@@ -88,6 +90,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+ SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
}
@@ -96,6 +99,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
super.tearDown();
Thread.currentThread().setName(oldThreadName);
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+ SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory);
TsFileValidationTool.badFileNum = 0;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index 95f7745d5bd..ff41c9a9902 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -45,6 +45,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionF
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.tools.validate.TsFileValidationTool;
import org.apache.tsfile.common.conf.TSFileDescriptor;
@@ -78,6 +79,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
private final String oldThreadName = Thread.currentThread().getName();
private ICrossCompactionPerformer performer = new
ReadPointCompactionPerformer();
+ private long compactionMemory =
SystemInfo.getInstance().getMemorySizeForCompaction();
@Before
public void setUp()
@@ -86,6 +88,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
+ SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1");
}
@@ -94,6 +97,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
super.tearDown();
Thread.currentThread().setName(oldThreadName);
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
+ SystemInfo.getInstance().setMemorySizeForCompaction(compactionMemory);
TsFileValidationTool.badFileNum = 0;
}
