This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 88cadb2d800 more compaction config items allow hot load (#12596)
88cadb2d800 is described below
commit 88cadb2d800a7c125ed883e902814b5b3a5e4f29
Author: shuwenwei <[email protected]>
AuthorDate: Mon May 27 18:20:47 2024 +0800
more compaction config items allow hot load (#12596)
---
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 34 ++++++++++++++++++++++
.../impl/RewriteCrossSpaceCompactionSelector.java | 5 ++--
.../impl/SizeTieredCompactionSelector.java | 5 ++--
3 files changed, 40 insertions(+), 4 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 6ac64b6c482..f38bd65f3cc 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1188,6 +1188,40 @@ public class IoTDBDescriptor {
"compaction_read_throughput_mb_per_sec",
Integer.toString(conf.getCompactionReadThroughputMbPerSec()))));
+ // update max_inner_compaction_candidate_file_num
+ conf.setFileLimitPerInnerTask(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_inner_compaction_candidate_file_num",
+ Integer.toString(conf.getFileLimitPerInnerTask()))));
+
+ // update target_compaction_file_size
+ conf.setTargetCompactionFileSize(
+ Long.parseLong(
+ properties.getProperty(
+ "target_compaction_file_size",
Long.toString(conf.getTargetCompactionFileSize()))));
+
+ // update max_cross_compaction_candidate_file_num
+ conf.setFileLimitPerCrossTask(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_cross_compaction_candidate_file_num",
+ Integer.toString(conf.getFileLimitPerCrossTask()))));
+
+ // update max_cross_compaction_candidate_file_size
+ conf.setMaxCrossCompactionCandidateFileSize(
+ Long.parseLong(
+ properties.getProperty(
+ "max_cross_compaction_candidate_file_size",
+
Long.toString(conf.getMaxCrossCompactionCandidateFileSize()))));
+
+ // update min_cross_compaction_unseq_file_level
+ conf.setMinCrossCompactionUnseqFileLevel(
+ Integer.parseInt(
+ properties.getProperty(
+ "min_cross_compaction_unseq_file_level",
+
Integer.toString(conf.getMinCrossCompactionUnseqFileLevel()))));
+
CompactionTaskManager.getInstance()
.setCompactionReadOperationRate(conf.getCompactionReadOperationPerSec());
CompactionTaskManager.getInstance()
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index cc858f2d04b..1904723ce73 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -265,18 +265,19 @@ public class RewriteCrossSpaceCompactionSelector
implements ICrossSpaceSelector
}
TsFileNameGenerator.TsFileName unseqFileName =
TsFileNameGenerator.getTsFileName(unseqFile.getTsFile().getName());
+ long targetCompactionFileSize = config.getTargetCompactionFileSize();
// we add a hard limit for cross compaction that selected unseqFile should
reach a certain size
// or be compacted in inner
// space at least once. This is used to make to improve the priority of
inner compaction and
// avoid too much cross compaction with small files.
- if (unseqFile.getTsFileSize() < config.getTargetCompactionFileSize()
+ if (unseqFile.getTsFileSize() < targetCompactionFileSize
&& unseqFileName.getInnerCompactionCnt() <
config.getMinCrossCompactionUnseqFileLevel()) {
return false;
}
long totalFileSize = unseqFile.getTsFileSize();
for (TsFileResource f : seqFiles) {
- if (f.getTsFileSize() >= config.getTargetCompactionFileSize() * 1.5) {
+ if (f.getTsFileSize() >= targetCompactionFileSize * 1.5) {
// to avoid serious write amplification caused by cross space
compaction, we restrict that
// seq files are no longer be compacted when the size reaches the
threshold.
return false;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
index 2dee4544b01..e7eed32dde8 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
@@ -101,6 +101,7 @@ public class SizeTieredCompactionSelector
List<TsFileResource> selectedFileList = new ArrayList<>();
long selectedFileSize = 0L;
long targetCompactionFileSize = config.getTargetCompactionFileSize();
+ int fileLimit = config.getFileLimitPerInnerTask();
List<List<TsFileResource>> taskList = new ArrayList<>();
for (TsFileResource currentFile : tsFileResources) {
@@ -124,7 +125,7 @@ public class SizeTieredCompactionSelector
long totalSizeIfSelectCurrentFile = selectedFileSize +
currentFile.getTsFileSize();
boolean canNotAddCurrentFileIntoCurrentTask =
totalSizeIfSelectCurrentFile > targetCompactionFileSize
- || selectedFileList.size() >= config.getFileLimitPerInnerTask();
+ || selectedFileList.size() >= fileLimit;
if (canNotAddCurrentFileIntoCurrentTask) {
// total file size or num will beyond the threshold if select current
file, stop the
// selection of current task
@@ -149,7 +150,7 @@ public class SizeTieredCompactionSelector
}
// if the selected file size reach the condition to submit
- if (selectedFileList.size() == config.getFileLimitPerInnerTask()) {
+ if (selectedFileList.size() == fileLimit) {
taskList.add(new ArrayList<>(selectedFileList));
selectedFileList.clear();
selectedFileSize = 0;
