This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.2 by this push:
new 2704c0f1ec0 [To rel/1.2][IOTDB-5972]Limit the number of all source
files in compaction (#10105)
2704c0f1ec0 is described below
commit 2704c0f1ec0a7b668dc6d60ad80a998c72da7c11
Author: 周沛辰 <[email protected]>
AuthorDate: Mon Jun 12 17:18:55 2023 +0800
[To rel/1.2][IOTDB-5972]Limit the number of all source files in compaction
(#10105)
---
docs/UserGuide/Reference/Common-Config-Manual.md | 10 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 10 +-
.../resources/conf/iotdb-common.properties | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 31 +++--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 +-
.../CompactionFileCountExceededException.java | 26 ++++
.../execute/task/AbstractCompactionTask.java | 6 +-
.../execute/task/CrossSpaceCompactionTask.java | 30 +++--
.../execute/task/InnerSpaceCompactionTask.java | 7 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 2 +-
.../impl/SizeTieredCompactionSelector.java | 2 +-
.../db/engine/settle/SettleRequestHandler.java | 2 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 54 ++++++++
.../engine/compaction/AbstractCompactionTest.java | 4 +-
.../engine/compaction/CompactionSchedulerTest.java | 96 +++++++--------
.../CompactionSchedulerWithFastPerformerTest.java | 96 +++++++--------
.../db/engine/compaction/MemoryControlTest.java | 137 ++++++++++++++++++++-
.../cross/CrossSpaceCompactionSelectorTest.java | 47 +++++++
...eCompactionWithFastPerformerValidationTest.java | 22 ++--
...actionWithReadPointPerformerValidationTest.java | 18 +--
.../cross/RewriteCompactionFileSelectorTest.java | 15 +--
.../inner/InnerCompactionSchedulerTest.java | 10 +-
.../inner/InnerSpaceCompactionSelectorTest.java | 2 +-
.../SizeTieredCompactionSelectorTest.java | 9 +-
.../compaction/utils/CompactionConfigRestorer.java | 4 +-
.../db/engine/settle/SettleRequestHandlerTest.java | 6 +-
.../db/engine/storagegroup/DataRegionTest.java | 10 +-
27 files changed, 465 insertions(+), 201 deletions(-)
diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md
b/docs/UserGuide/Reference/Common-Config-Manual.md
index a499c57e1b0..0564bc1f6da 100644
--- a/docs/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/UserGuide/Reference/Common-Config-Manual.md
@@ -974,12 +974,12 @@ Different configuration parameters take effect in the
following three ways:
* max\_cross\_compaction\_file\_num
-|Name| max\_cross\_compaction\_candidate\_file\_num |
-|:---:|:---|
+|Name| max\_cross\_compaction\_candidate\_file\_num |
+|:---:|:---------------------------------------------------------|
|Description| The max num of files encounter in cross space compaction |
-|Type| int32 |
-|Default| 1000 |
-|Effective|After restart system|
+|Type| int32 |
+|Default| 500 |
+|Effective| After restart system |
* max\_cross\_compaction\_file\_size
diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
index d5ac85ad9a0..a8c1a716fbf 100644
--- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md
+++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md
@@ -1023,11 +1023,11 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。
* max\_cross\_compaction\_candidate\_file\_num
|名字| max\_cross\_compaction\_candidate\_file\_num |
-|:---:|:---|
-|描述| 跨空间合并中一次合并最多参与的文件数 |
-|类型| int32 |
-|默认值| 1000|
-|改后生效方式|重启服务生效|
+|:---:|:---------------------------------------------|
+|描述| 跨空间合并中一次合并最多参与的文件数 |
+|类型| int32 |
+|默认值| 500 |
+|改后生效方式| 重启服务生效 |
* max\_cross\_compaction\_candidate\_file\_size
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties
b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 132b4332113..a2d70d60aa8 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -635,7 +635,7 @@ cluster_name=defaultCluster
# The max file when selecting cross space compaction candidate files
# At least one unseq file with it's overlapped seq files will be selected even
exceeded this number
# Datatype: int
-# max_cross_compaction_candidate_file_num=1000
+# max_cross_compaction_candidate_file_num=500
# The max total size when selecting cross space compaction candidate files
# At least one unseq file with it's overlapped seq files will be selected even
exceeded this number
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 2d147943b87..46e8f212bbe 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -494,13 +494,16 @@ public class IoTDBConfig {
*/
private long compactionAcquireWriteLockTimeout = 60_000L;
- /** The max candidate file num in inner space compaction */
- private int maxInnerCompactionCandidateFileNum = 30;
+ /** The max candidate file num in one inner space compaction task */
+ private int fileLimitPerInnerTask = 30;
+
+ /** The max candidate file num in one cross space compaction task */
+ private int fileLimitPerCrossTask = 500;
/** The max candidate file num in cross space compaction */
- private int maxCrossCompactionCandidateFileNum = 1000;
+ private int totalFileLimitForCrossTask = 5000;
- /** The max total size of candidate files in cross space compaction */
+ /** The max total size of candidate files in one cross space compaction task
*/
private long maxCrossCompactionCandidateFileSize = 1024 * 1024 * 1024 * 5L;
/**
@@ -2962,20 +2965,24 @@ public class IoTDBConfig {
this.compactionScheduleIntervalInMs = compactionScheduleIntervalInMs;
}
- public int getMaxInnerCompactionCandidateFileNum() {
- return maxInnerCompactionCandidateFileNum;
+ public int getFileLimitPerInnerTask() {
+ return fileLimitPerInnerTask;
+ }
+
+ public void setFileLimitPerInnerTask(int fileLimitPerInnerTask) {
+ this.fileLimitPerInnerTask = fileLimitPerInnerTask;
}
- public void setMaxInnerCompactionCandidateFileNum(int
maxInnerCompactionCandidateFileNum) {
- this.maxInnerCompactionCandidateFileNum =
maxInnerCompactionCandidateFileNum;
+ public int getFileLimitPerCrossTask() {
+ return fileLimitPerCrossTask;
}
- public int getMaxCrossCompactionCandidateFileNum() {
- return maxCrossCompactionCandidateFileNum;
+ public int getTotalFileLimitForCrossTask() {
+ return totalFileLimitForCrossTask;
}
- public void setMaxCrossCompactionCandidateFileNum(int
maxCrossCompactionCandidateFileNum) {
- this.maxCrossCompactionCandidateFileNum =
maxCrossCompactionCandidateFileNum;
+ public void setFileLimitPerCrossTask(int fileLimitPerCrossTask) {
+ this.fileLimitPerCrossTask = fileLimitPerCrossTask;
}
public long getMaxCrossCompactionCandidateFileSize() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 2f1c9fdee4c..15a78fd231d 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -680,16 +680,16 @@ public class IoTDBDescriptor {
properties.getProperty(
"chunk_size_lower_bound_in_compaction",
Long.toString(conf.getChunkSizeLowerBoundInCompaction()))));
- conf.setMaxInnerCompactionCandidateFileNum(
+ conf.setFileLimitPerInnerTask(
Integer.parseInt(
properties.getProperty(
"max_inner_compaction_candidate_file_num",
-
Integer.toString(conf.getMaxInnerCompactionCandidateFileNum()))));
- conf.setMaxCrossCompactionCandidateFileNum(
+ Integer.toString(conf.getFileLimitPerInnerTask()))));
+ conf.setFileLimitPerCrossTask(
Integer.parseInt(
properties.getProperty(
"max_cross_compaction_candidate_file_num",
-
Integer.toString(conf.getMaxCrossCompactionCandidateFileNum()))));
+ Integer.toString(conf.getFileLimitPerCrossTask()))));
conf.setMaxCrossCompactionCandidateFileSize(
Long.parseLong(
properties.getProperty(
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionFileCountExceededException.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionFileCountExceededException.java
new file mode 100644
index 00000000000..49a7e3d4546
--- /dev/null
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/exception/CompactionFileCountExceededException.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.compaction.execute.exception;
+
+public class CompactionFileCountExceededException extends Exception {
+ public CompactionFileCountExceededException(String message) {
+ super(message);
+ }
+}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/AbstractCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/AbstractCompactionTask.java
index b4ae3660630..fa74353eec1 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/AbstractCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/AbstractCompactionTask.java
@@ -140,7 +140,11 @@ public abstract class AbstractCompactionTask {
return false;
}
- public abstract void resetCompactionCandidateStatusForAllSourceFiles();
+ public void resetCompactionCandidateStatusForAllSourceFiles() {
+ List<TsFileResource> resources = getAllSourceTsFiles();
+ // only reset status of the resources whose status is COMPACTING and
COMPACTING_CANDIDATE
+ resources.forEach(x -> x.setStatus(TsFileResourceStatus.NORMAL));
+ }
public long getTimeCost() {
return summary.getTimeCost();
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
index 76f59f9bf5d..7ce24e887cb 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.engine.compaction.execute.task;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.engine.compaction.execute.exception.CompactionExceptionHandler;
+import
org.apache.iotdb.db.engine.compaction.execute.exception.CompactionFileCountExceededException;
import
org.apache.iotdb.db.engine.compaction.execute.exception.CompactionMemoryNotEnoughException;
import
org.apache.iotdb.db.engine.compaction.execute.performer.ICrossCompactionPerformer;
import
org.apache.iotdb.db.engine.compaction.execute.performer.impl.FastCompactionPerformer;
@@ -344,13 +345,6 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
return equalsOtherTask((CrossSpaceCompactionTask) other);
}
- @Override
- public void resetCompactionCandidateStatusForAllSourceFiles() {
- // Only reset status of the resources whose status is COMPACTING and
COMPACTING_CANDIDATE
- selectedSequenceFiles.forEach(x ->
x.setStatus(TsFileResourceStatus.NORMAL));
- selectedUnsequenceFiles.forEach(x ->
x.setStatus(TsFileResourceStatus.NORMAL));
- }
-
private long[] deleteOldFiles(List<TsFileResource> tsFileResourceList) {
long[] size = new long[tsFileResourceList.size()];
for (int i = 0, length = tsFileResourceList.size(); i < length; ++i) {
@@ -376,21 +370,33 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
@Override
public boolean checkValidAndSetMerging() {
if (!tsFileManager.isAllowCompaction()) {
+ resetCompactionCandidateStatusForAllSourceFiles();
return false;
}
try {
SystemInfo.getInstance().addCompactionMemoryCost(memoryCost, 60);
- } catch (InterruptedException e) {
- LOGGER.error("Interrupted when allocating memory for compaction", e);
- return false;
- } catch (CompactionMemoryNotEnoughException e) {
- LOGGER.error("No enough memory for current compaction task {}", this, e);
+ SystemInfo.getInstance()
+ .addCompactionFileNum(selectedSequenceFiles.size() +
selectedUnsequenceFiles.size(), 60);
+ } catch (Throwable t) {
+ if (t instanceof InterruptedException) {
+ LOGGER.warn("Interrupted when allocating memory for compaction", t);
+ } else if (t instanceof CompactionMemoryNotEnoughException) {
+ LOGGER.info("No enough memory for current compaction task {}", this,
t);
+ } else if (t instanceof CompactionFileCountExceededException) {
+ LOGGER.info("No enough file num for current compaction task {}", this,
t);
+ SystemInfo.getInstance().resetCompactionMemoryCost(memoryCost);
+ }
+ resetCompactionCandidateStatusForAllSourceFiles();
return false;
}
+
boolean addReadLockSuccess =
addReadLock(selectedSequenceFiles) &&
addReadLock(selectedUnsequenceFiles);
if (!addReadLockSuccess) {
SystemInfo.getInstance().resetCompactionMemoryCost(memoryCost);
+ SystemInfo.getInstance()
+ .decreaseCompactionFileNumCost(
+ selectedSequenceFiles.size() + selectedUnsequenceFiles.size());
}
return addReadLockSuccess;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
index 3a84b6d0167..673601fd4e9 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -407,12 +407,6 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
return equalsOtherTask((InnerSpaceCompactionTask) other);
}
- @Override
- public void resetCompactionCandidateStatusForAllSourceFiles() {
- // Only reset status of the resources whose status is COMPACTING and
COMPACTING_CANDIDATE
- selectedTsFileResourceList.forEach(x ->
x.setStatus(TsFileResourceStatus.NORMAL));
- }
-
/**
* release the read lock and write lock of files if it is held, and set the
merging status of
* selected files to false
@@ -433,6 +427,7 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
@Override
public boolean checkValidAndSetMerging() {
if (!tsFileManager.isAllowCompaction()) {
+ resetCompactionCandidateStatusForAllSourceFiles();
return false;
}
try {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index 635df78cd23..60b1deecb07 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -76,7 +76,7 @@ public class RewriteCrossSpaceCompactionSelector implements
ICrossSpaceSelector
/
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount()
* config.getUsableCompactionMemoryProportion());
this.maxCrossCompactionFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerCrossTask();
this.maxCrossCompactionFileSize =
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileSize();
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
index 805abe2f1d8..8ebf6ceb211 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/SizeTieredCompactionSelector.java
@@ -129,7 +129,7 @@ public class SizeTieredCompactionSelector
selectedFileSize);
// if the file size or file num reach threshold
if (selectedFileSize >= targetCompactionFileSize
- || selectedFileList.size() >=
config.getMaxInnerCompactionCandidateFileNum()) {
+ || selectedFileList.size() >= config.getFileLimitPerInnerTask()) {
// submit the task
if (selectedFileList.size() > 1) {
taskPriorityQueue.add(new Pair<>(new ArrayList<>(selectedFileList),
selectedFileSize));
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java
b/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java
index fa84d422f75..a4cec2403e2 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/settle/SettleRequestHandler.java
@@ -107,7 +107,7 @@ public class SettleRequestHandler {
TSStatusCode.ILLEGAL_PARAMETER, "The files to settle is not
offered.");
}
- int maxInnerCompactionCandidateFileNum =
config.getMaxInnerCompactionCandidateFileNum();
+ int maxInnerCompactionCandidateFileNum =
config.getFileLimitPerInnerTask();
if (paths.size() > maxInnerCompactionCandidateFileNum) {
return RpcUtils.getStatus(
TSStatusCode.UNSUPPORTED_OPERATION,
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/SystemInfo.java
b/server/src/main/java/org/apache/iotdb/db/rescon/SystemInfo.java
index c77825ddd61..8b9877ead7c 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/SystemInfo.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/SystemInfo.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import
org.apache.iotdb.db.engine.compaction.execute.exception.CompactionFileCountExceededException;
import
org.apache.iotdb.db.engine.compaction.execute.exception.CompactionMemoryNotEnoughException;
import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.engine.storagegroup.DataRegionInfo;
@@ -37,6 +38,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.PriorityQueue;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
public class SystemInfo {
@@ -54,6 +56,10 @@ public class SystemInfo {
private long flushingMemTablesCost = 0L;
private AtomicLong compactionMemoryCost = new AtomicLong(0L);
+ private AtomicInteger compactionFileNumCost = new AtomicInteger(0);
+
+ private int totalFileLimitForCrossTask =
config.getTotalFileLimitForCrossTask();
+
private ExecutorService flushTaskSubmitThreadPool =
IoTDBThreadPoolFactory.newSingleThreadExecutor(ThreadName.FLUSH_TASK_SUBMIT.getName());
private double FLUSH_THERSHOLD = memorySizeForMemtable *
config.getFlushProportion();
@@ -182,6 +188,30 @@ public class SystemInfo {
this.flushingMemTablesCost -= flushingMemTableCost;
}
+ public void addCompactionFileNum(int fileNum, long timeOutInSecond)
+ throws InterruptedException, CompactionFileCountExceededException {
+ if (fileNum > totalFileLimitForCrossTask) {
+ // source file num is greater than the max file num for compaction
+ throw new CompactionFileCountExceededException(
+ String.format(
+ "Required file num %d is greater than the max file num %d for
compaction.",
+ fileNum, totalFileLimitForCrossTask));
+ }
+ long startTime = System.currentTimeMillis();
+ int originFileNum = this.compactionFileNumCost.get();
+ while (originFileNum + fileNum > totalFileLimitForCrossTask
+ || !compactionFileNumCost.compareAndSet(originFileNum, originFileNum +
fileNum)) {
+ if (System.currentTimeMillis() - startTime >= timeOutInSecond * 1000L) {
+ throw new CompactionFileCountExceededException(
+ String.format(
+ "Failed to allocate %d files for compaction after %d seconds,
max file num for compaction module is %d, %d files is used.",
+ fileNum, timeOutInSecond, totalFileLimitForCrossTask,
originFileNum));
+ }
+ Thread.sleep(100);
+ originFileNum = this.compactionFileNumCost.get();
+ }
+ }
+
public void addCompactionMemoryCost(long memoryCost, long timeOutInSecond)
throws InterruptedException, CompactionMemoryNotEnoughException {
if (!config.isEnableCompactionMemControl()) {
@@ -215,6 +245,10 @@ public class SystemInfo {
this.compactionMemoryCost.addAndGet(-compactionMemoryCost);
}
+ public synchronized void decreaseCompactionFileNumCost(int fileNum) {
+ this.compactionFileNumCost.addAndGet(-fileNum);
+ }
+
public long getMemorySizeForCompaction() {
if (config.isEnableMemControl()) {
return memorySizeForCompaction;
@@ -238,6 +272,26 @@ public class SystemInfo {
memorySizeForCompaction = size;
}
+ @TestOnly
+ public void setTotalFileLimitForCrossTask(int totalFileLimitForCrossTask) {
+ this.totalFileLimitForCrossTask = totalFileLimitForCrossTask;
+ }
+
+ @TestOnly
+ public int getTotalFileLimitForCrossTask() {
+ return totalFileLimitForCrossTask;
+ }
+
+ @TestOnly
+ public AtomicLong getCompactionMemoryCost() {
+ return compactionMemoryCost;
+ }
+
+ @TestOnly
+ public AtomicInteger getCompactionFileNumCost() {
+ return compactionFileNumCost;
+ }
+
private void logCurrentTotalSGMemory() {
logger.debug("Current Sg cost is {}", totalStorageGroupMemCost);
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
index 71f7820dd6f..7176f04b5d6 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
@@ -107,7 +107,7 @@ public class AbstractCompactionTest {
TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage();
private static final int oldMaxCrossCompactionFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerCrossTask();
private final int oldMaxDegreeOfIndexNode =
TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode();
@@ -395,7 +395,7 @@ public class AbstractCompactionTest {
IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(oldTargetChunkPointNum);
IoTDBDescriptor.getInstance()
.getConfig()
- .setMaxCrossCompactionCandidateFileNum(oldMaxCrossCompactionFileNum);
+ .setFileLimitPerCrossTask(oldMaxCrossCompactionFileNum);
IoTDBDescriptor.getInstance()
.getConfig()
.setChunkSizeLowerBoundInCompaction(oldTargetChunkPointNum);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
index 1dbb1bda0c4..2c9ba7aedb5 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
@@ -140,8 +140,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
IoTDBDescriptor.getInstance()
.getConfig()
.setTargetCompactionFileSize(2L * 1024L * 1024L * 1024L);
@@ -237,7 +237,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -259,8 +259,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
long origin = SystemInfo.getInstance().getMemorySizeForCompaction();
SystemInfo.getInstance()
.setMemorySizeForCompaction(
@@ -355,7 +355,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
} finally {
SystemInfo.getInstance().setMemorySizeForCompaction(origin);
@@ -379,8 +379,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test3";
try {
CompactionTaskManager.getInstance().restart();
@@ -453,7 +453,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -474,8 +474,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
IoTDBDescriptor.getInstance()
.getConfig()
.setTargetCompactionFileSize(2L * 1024L * 1024L * 1024L);
@@ -539,7 +539,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -560,8 +560,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test5";
try {
CompactionTaskManager.getInstance().restart();
@@ -653,7 +653,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -675,8 +675,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test6";
try {
CompactionTaskManager.getInstance().restart();
@@ -760,7 +760,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -780,8 +780,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test7";
try {
CompactionTaskManager.getInstance().restart();
@@ -861,7 +861,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -881,8 +881,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test8";
try {
CompactionTaskManager.getInstance().restart();
@@ -948,7 +948,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -968,8 +968,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test9";
try {
CompactionTaskManager.getInstance().restart();
@@ -1048,7 +1048,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -1071,8 +1071,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test10";
try {
CompactionTaskManager.getInstance().restart();
@@ -1155,7 +1155,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
IoTDBDescriptor.getInstance()
.getConfig()
.setEnableCrossSpaceCompaction(prevEnableCrossCompaction);
@@ -1178,8 +1178,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test11";
try {
CompactionTaskManager.getInstance().restart();
@@ -1261,7 +1261,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -1282,8 +1282,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test12";
try {
CompactionTaskManager.getInstance().restart();
@@ -1370,7 +1370,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -1391,8 +1391,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test13";
try {
CompactionTaskManager.getInstance().restart();
@@ -1480,7 +1480,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -1500,8 +1500,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test14";
try {
CompactionTaskManager.getInstance().restart();
@@ -1597,7 +1597,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -1617,8 +1617,8 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test16";
try {
CompactionTaskManager.getInstance().restart();
@@ -1701,7 +1701,7 @@ public class CompactionSchedulerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -1709,8 +1709,8 @@ public class CompactionSchedulerTest {
public void testLargeFileInLowerLevel() throws Exception {
logger.warn("Running test16");
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
long originTargetSize =
IoTDBDescriptor.getInstance().getConfig().getTargetCompactionFileSize();
IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1024
* 1024);
String sgName = COMPACTION_TEST_SG + "test17";
@@ -1801,7 +1801,7 @@ public class CompactionSchedulerTest {
} finally {
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(originTargetSize);
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
index 15dd291aba1..757613e4655 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
@@ -141,8 +141,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
IoTDBDescriptor.getInstance()
.getConfig()
.setTargetCompactionFileSize(2L * 1024L * 1024L * 1024L);
@@ -238,7 +238,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -260,8 +260,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
long origin = SystemInfo.getInstance().getMemorySizeForCompaction();
SystemInfo.getInstance()
.setMemorySizeForCompaction(
@@ -356,7 +356,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
} finally {
SystemInfo.getInstance().setMemorySizeForCompaction(origin);
@@ -380,8 +380,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test3";
try {
CompactionTaskManager.getInstance().restart();
@@ -454,7 +454,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -475,8 +475,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
IoTDBDescriptor.getInstance()
.getConfig()
.setTargetCompactionFileSize(2L * 1024L * 1024L * 1024L);
@@ -540,7 +540,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -561,8 +561,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test5";
try {
CompactionTaskManager.getInstance().restart();
@@ -654,7 +654,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -676,8 +676,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test6";
try {
CompactionTaskManager.getInstance().restart();
@@ -761,7 +761,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -781,8 +781,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test7";
try {
CompactionTaskManager.getInstance().restart();
@@ -862,7 +862,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -882,8 +882,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(100);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(100);
String sgName = COMPACTION_TEST_SG + "test8";
try {
CompactionTaskManager.getInstance().restart();
@@ -949,7 +949,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -969,8 +969,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test9";
try {
CompactionTaskManager.getInstance().restart();
@@ -1049,7 +1049,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -1072,8 +1072,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test10";
try {
CompactionTaskManager.getInstance().restart();
@@ -1156,7 +1156,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
IoTDBDescriptor.getInstance()
.getConfig()
.setEnableCrossSpaceCompaction(prevEnableCrossCompaction);
@@ -1179,8 +1179,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test11";
try {
CompactionTaskManager.getInstance().restart();
@@ -1262,7 +1262,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -1283,8 +1283,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test12";
try {
CompactionTaskManager.getInstance().restart();
@@ -1368,7 +1368,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -1389,8 +1389,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test13";
try {
CompactionTaskManager.getInstance().restart();
@@ -1478,7 +1478,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -1498,8 +1498,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test14";
try {
CompactionTaskManager.getInstance().restart();
@@ -1595,7 +1595,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
/**
@@ -1615,8 +1615,8 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
String sgName = COMPACTION_TEST_SG + "test16";
try {
CompactionTaskManager.getInstance().restart();
@@ -1699,7 +1699,7 @@ public class CompactionSchedulerWithFastPerformerTest {
.setCompactionThreadCount(prevCompactionConcurrentThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
}
}
@@ -1707,8 +1707,8 @@ public class CompactionSchedulerWithFastPerformerTest {
public void testLargeFileInLowerLevel() throws Exception {
logger.warn("Running test16");
int prevMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
long originTargetSize =
IoTDBDescriptor.getInstance().getConfig().getTargetCompactionFileSize();
IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1024
* 1024);
String sgName = COMPACTION_TEST_SG + "test17";
@@ -1799,7 +1799,7 @@ public class CompactionSchedulerWithFastPerformerTest {
} finally {
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(prevMaxCompactionCandidateFileNum);
IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(originTargetSize);
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
index c71356ca9a7..04d8d87a671 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/MemoryControlTest.java
@@ -20,9 +20,11 @@
package org.apache.iotdb.db.engine.compaction;
import
org.apache.iotdb.db.engine.compaction.execute.task.CrossSpaceCompactionTask;
+import
org.apache.iotdb.db.engine.compaction.execute.task.InnerSpaceCompactionTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
+import org.apache.iotdb.db.rescon.SystemInfo;
import org.junit.Assert;
import org.junit.Test;
@@ -40,17 +42,20 @@ public class MemoryControlTest {
for (int i = 1; i <= 10; i++) {
sequenceFiles.add(
new TsFileResource(
- new File(String.format("%d-%d-0-0.tsfile", i, i)),
TsFileResourceStatus.NORMAL));
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
}
List<TsFileResource> unsequenceFiles = new ArrayList<>();
for (int i = 11; i <= 20; i++) {
unsequenceFiles.add(
new TsFileResource(
- new File(String.format("%d-%d-0-0.tsfile", i, i)),
TsFileResourceStatus.NORMAL));
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
}
TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
Mockito.when(tsFileManager.getStorageGroupName()).thenReturn("root.sg");
Mockito.when(tsFileManager.getDataRegionId()).thenReturn("1");
+ Mockito.when(tsFileManager.isAllowCompaction()).thenReturn(true);
CrossSpaceCompactionTask task =
new CrossSpaceCompactionTask(
0L,
@@ -63,6 +68,8 @@ public class MemoryControlTest {
0);
boolean success = task.checkValidAndSetMerging();
Assert.assertFalse(success);
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionMemoryCost().get());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionFileNumCost().get());
for (TsFileResource tsFileResource : sequenceFiles) {
Assert.assertEquals(TsFileResourceStatus.NORMAL,
tsFileResource.getStatus());
Assert.assertTrue(tsFileResource.tryWriteLock());
@@ -72,4 +79,130 @@ public class MemoryControlTest {
Assert.assertTrue(tsFileResource.tryWriteLock());
}
}
+
+ @Test
+ public void testFailedToAllocateFileNumInCrossTask() {
+ int oldMaxCrossCompactionCandidateFileNum =
+ SystemInfo.getInstance().getTotalFileLimitForCrossTask();
+ SystemInfo.getInstance().setTotalFileLimitForCrossTask(2);
+ try {
+ List<TsFileResource> sequenceFiles = new ArrayList<>();
+ for (int i = 1; i <= 10; i++) {
+ sequenceFiles.add(
+ new TsFileResource(
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
+ }
+ List<TsFileResource> unsequenceFiles = new ArrayList<>();
+ for (int i = 11; i <= 30; i++) {
+ unsequenceFiles.add(
+ new TsFileResource(
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
+ }
+
+ TsFileManager tsFileManager = new TsFileManager("root.testsg", "0", "");
+ tsFileManager.addAll(sequenceFiles, true);
+ tsFileManager.addAll(unsequenceFiles, false);
+ CrossSpaceCompactionTask task =
+ new CrossSpaceCompactionTask(
+ 0L,
+ tsFileManager,
+ sequenceFiles,
+ unsequenceFiles,
+ null,
+ new AtomicInteger(0),
+ 1000,
+ 0);
+ boolean success = task.checkValidAndSetMerging();
+ Assert.assertFalse(success);
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionMemoryCost().get());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionFileNumCost().get());
+ for (TsFileResource tsFileResource : sequenceFiles) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL,
tsFileResource.getStatus());
+ Assert.assertTrue(tsFileResource.tryWriteLock());
+ }
+ for (TsFileResource tsFileResource : unsequenceFiles) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL,
tsFileResource.getStatus());
+ Assert.assertTrue(tsFileResource.tryWriteLock());
+ }
+ } finally {
+
SystemInfo.getInstance().setTotalFileLimitForCrossTask(oldMaxCrossCompactionCandidateFileNum);
+ }
+ }
+
+ /**
+ * AllowCompaction is false.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFailedToCheckValidInCrossTask() {
+ List<TsFileResource> sequenceFiles = new ArrayList<>();
+ for (int i = 1; i <= 10; i++) {
+ sequenceFiles.add(
+ new TsFileResource(
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
+ }
+ List<TsFileResource> unsequenceFiles = new ArrayList<>();
+ for (int i = 11; i <= 20; i++) {
+ unsequenceFiles.add(
+ new TsFileResource(
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
+ }
+ TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
+ Mockito.when(tsFileManager.getStorageGroupName()).thenReturn("root.sg");
+ Mockito.when(tsFileManager.getDataRegionId()).thenReturn("1");
+
+ // fail to check valid when tsfile manager is not allowed to compaction in
cross task
+ CrossSpaceCompactionTask task =
+ new CrossSpaceCompactionTask(
+ 0L, tsFileManager, sequenceFiles, unsequenceFiles, null, new
AtomicInteger(0), 1000, 0);
+ boolean success = task.checkValidAndSetMerging();
+ Assert.assertFalse(success);
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionMemoryCost().get());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionFileNumCost().get());
+ for (TsFileResource tsFileResource : sequenceFiles) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL,
tsFileResource.getStatus());
+ Assert.assertTrue(tsFileResource.tryWriteLock());
+ }
+ for (TsFileResource tsFileResource : unsequenceFiles) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL,
tsFileResource.getStatus());
+ Assert.assertTrue(tsFileResource.tryWriteLock());
+ }
+ }
+
+ /**
+ * AllowCompaction is false.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testFailedToCheckValidInInnerTask() {
+ List<TsFileResource> sequenceFiles = new ArrayList<>();
+ for (int i = 1; i <= 10; i++) {
+ sequenceFiles.add(
+ new TsFileResource(
+ new File(String.format("%d-%d-0-0.tsfile", i, i)),
+ TsFileResourceStatus.COMPACTION_CANDIDATE));
+ }
+ TsFileManager tsFileManager = Mockito.mock(TsFileManager.class);
+ Mockito.when(tsFileManager.getStorageGroupName()).thenReturn("root.sg");
+ Mockito.when(tsFileManager.getDataRegionId()).thenReturn("1");
+
+ // fail to check valid when tsfile manager is not allowed to compaction in
inner task
+ InnerSpaceCompactionTask innerTask =
+ new InnerSpaceCompactionTask(
+ 0L, tsFileManager, sequenceFiles, true, null, new
AtomicInteger(0), 0L);
+ boolean success = innerTask.checkValidAndSetMerging();
+ Assert.assertFalse(success);
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionMemoryCost().get());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionFileNumCost().get());
+ for (TsFileResource tsFileResource : sequenceFiles) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL,
tsFileResource.getStatus());
+ Assert.assertTrue(tsFileResource.tryWriteLock());
+ }
+ }
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
index 28d1fbed726..6407418ea08 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionSelectorTest.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.junit.After;
@@ -174,6 +175,52 @@ public class CrossSpaceCompactionSelectorTest extends
AbstractCompactionTest {
Assert.assertEquals(0, selected.size());
}
+ @Test
+ public void testSelectWithTooManySourceFiles()
+ throws IOException, MetadataException, WriteProcessException {
+ int oldMaxFileNumForCompaction =
SystemInfo.getInstance().getTotalFileLimitForCrossTask();
+ SystemInfo.getInstance().setTotalFileLimitForCrossTask(1);
+ try {
+ createFiles(19, 2, 3, 50, 0, 10000, 50, 50, false, true);
+ createFiles(1, 2, 3, 3000, 0, 10000, 50, 50, false, false);
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("", "", 0, null);
+ List<CrossCompactionTaskResource> selected =
+ selector.selectCrossSpaceTask(seqResources, unseqResources);
+ Assert.assertEquals(1, selected.size());
+ Assert.assertEquals(19, selected.get(0).getSeqFiles().size());
+ Assert.assertEquals(1, selected.get(0).getUnseqFiles().size());
+
+ CrossSpaceCompactionTask crossSpaceCompactionTask =
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ selected.get(0).getSeqFiles(),
+ selected.get(0).getUnseqFiles(),
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .getCrossCompactionPerformer()
+ .createInstance(),
+ CompactionTaskManager.currentTaskNum,
+ 1000,
+ tsFileManager.getNextCompactionTaskId());
+ // set file status to COMPACTION_CANDIDATE
+
Assert.assertTrue(crossSpaceCompactionTask.setSourceFilesToCompactionCandidate());
+
+ Assert.assertFalse(crossSpaceCompactionTask.checkValidAndSetMerging());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionMemoryCost().get());
+ Assert.assertEquals(0,
SystemInfo.getInstance().getCompactionFileNumCost().get());
+ for (TsFileResource resource : seqResources) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL, resource.getStatus());
+ }
+ for (TsFileResource resource : unseqResources) {
+ Assert.assertEquals(TsFileResourceStatus.NORMAL, resource.getStatus());
+ }
+ } finally {
+
SystemInfo.getInstance().setTotalFileLimitForCrossTask(oldMaxFileNumForCompaction);
+ }
+ }
+
@Test
public void testSeqFileWithDeviceIndexBeenDeletedBeforeSelection()
throws IOException, MetadataException, WriteProcessException,
InterruptedException {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index 17e80a541d6..4d00447dde2 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -289,7 +289,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
*/
@Test
public void test5() throws MetadataException, IOException,
WriteProcessException, MergeException {
-
IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(7);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerCrossTask(7);
registerTimeseriesInMManger(5, 10, true);
createFiles(7, 5, 10, 1000, 0, 0, 100, 100, true, true);
createFiles(1, 5, 10, 3300, 2150, 2150, 100, 100, true, false);
@@ -2228,8 +2228,8 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
*/
@Test
public void testCompactionSchedule() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(1);
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerCrossTask(1);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(10, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(1, 5, 10, 1000, 4000, 4000, 0, 100, false, false);
createFiles(1, 5, 10, 1000, 5000, 5000, 0, 100, false, false);
@@ -2313,7 +2313,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles1() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
@@ -2396,7 +2396,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles2() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
@@ -2477,7 +2477,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles3() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(4, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
@@ -2559,7 +2559,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles4() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(1, 9, 10, 500, 100, 100, 0, 100, false, false);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
@@ -2643,7 +2643,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles1() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
@@ -2732,7 +2732,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles2() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
@@ -2820,7 +2820,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles3() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(4, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
@@ -2909,7 +2909,7 @@ public class
CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles4() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(1, 9, 10, 500, 100, 100, 0, 100, true, false);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index 8019b237307..eec1057caf9 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -289,7 +289,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
*/
@Test
public void test5() throws MetadataException, IOException,
WriteProcessException, MergeException {
-
IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(7);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerCrossTask(7);
registerTimeseriesInMManger(5, 10, true);
createFiles(7, 5, 10, 1000, 0, 0, 100, 100, true, true);
createFiles(1, 5, 10, 3300, 2150, 2150, 100, 100, true, false);
@@ -2165,7 +2165,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles1() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
@@ -2247,7 +2247,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles2() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
@@ -2328,7 +2328,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles3() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(4, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, false, true);
@@ -2410,7 +2410,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testNonAlignedUnseqFilesNotOverlapWithSeqFiles4() throws
Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, false, true);
createFiles(1, 9, 10, 500, 100, 100, 0, 100, false, false);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, false, false);
@@ -2493,7 +2493,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles1() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
createFiles(3, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
@@ -2582,7 +2582,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles2() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
@@ -2670,7 +2670,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles3() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(4, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
createFiles(1, 10, 5, 1000, 7500, 7500, 100, 100, true, true);
@@ -2759,7 +2759,7 @@ public class
CrossSpaceCompactionWithReadPointPerformerValidationTest
@Test
public void testAlignedUnseqFilesNotOverlapWithSeqFiles4() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
createFiles(5, 10, 5, 1000, 0, 0, 100, 100, true, true);
createFiles(1, 9, 10, 500, 100, 100, 0, 100, true, false);
createFiles(2, 5, 10, 500, 6000, 6000, 0, 100, true, false);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
index c4cb3423dfc..84a0aa76e1c 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
@@ -954,8 +954,8 @@ public class RewriteCompactionFileSelectorTest extends
MergeTest {
@Test
public void testMaxFileSelection() throws MergeException, IOException {
int oldMaxCrossCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(5);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerCrossTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerCrossTask(5);
RewriteCrossSpaceCompactionSelector selector =
new RewriteCrossSpaceCompactionSelector("", "", 0, null);
List<CrossCompactionTaskResource> selected =
@@ -968,14 +968,13 @@ public class RewriteCompactionFileSelectorTest extends
MergeTest {
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxCrossCompactionCandidateFileNum(oldMaxCrossCompactionCandidateFileNum);
+ .setFileLimitPerCrossTask(oldMaxCrossCompactionCandidateFileNum);
}
@Test
public void testAtLeastOneUnseqFileBeenSelected() throws IOException,
MergeException {
- int maxCrossFilesNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxCrossCompactionCandidateFileNum(1);
+ int maxCrossFilesNum =
IoTDBDescriptor.getInstance().getConfig().getFileLimitPerCrossTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerCrossTask(1);
RewriteCrossSpaceCompactionSelector selector =
new RewriteCrossSpaceCompactionSelector("", "", 0, null);
@@ -987,9 +986,7 @@ public class RewriteCompactionFileSelectorTest extends
MergeTest {
assertEquals(1, seqSelected.size());
assertEquals(1, unseqSelected.size());
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setMaxCrossCompactionCandidateFileNum(maxCrossFilesNum);
+
IoTDBDescriptor.getInstance().getConfig().setFileLimitPerCrossTask(maxCrossFilesNum);
}
@Test
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java
index 42eb8c6d1bd..977a6f44b32 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerCompactionSchedulerTest.java
@@ -47,7 +47,7 @@ public class InnerCompactionSchedulerTest extends
AbstractCompactionTest {
int oldConcurrentCompactionThread =
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount();
int oldMaxCompactionCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
@Before
public void setUp()
@@ -73,7 +73,7 @@ public class InnerCompactionSchedulerTest extends
AbstractCompactionTest {
.setCompactionThreadCount(oldConcurrentCompactionThread);
IoTDBDescriptor.getInstance()
.getConfig()
-
.setMaxInnerCompactionCandidateFileNum(oldMaxCompactionCandidateFileNum);
+ .setFileLimitPerInnerTask(oldMaxCompactionCandidateFileNum);
super.tearDown();
}
@@ -83,7 +83,7 @@ public class InnerCompactionSchedulerTest extends
AbstractCompactionTest {
IoTDBDescriptor.getInstance().getConfig().setEnableSeqSpaceCompaction(true);
IoTDBDescriptor.getInstance().getConfig().setEnableUnseqSpaceCompaction(true);
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(4);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(4);
IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1000000);
createFiles(2, 2, 3, 100, 0, 0, 50, 50, false, true);
registerTimeseriesInMManger(2, 3, false);
@@ -107,7 +107,7 @@ public class InnerCompactionSchedulerTest extends
AbstractCompactionTest {
public void testFileSelector2()
throws IOException, MetadataException, WriteProcessException,
InterruptedException {
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(50);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(50);
TsFileResourceList tsFileResources = new TsFileResourceList();
createFiles(2, 2, 3, 100, 0, 0, 50, 50, false, true);
createFiles(2, 3, 5, 50, 250, 250, 50, 50, false, true);
@@ -137,7 +137,7 @@ public class InnerCompactionSchedulerTest extends
AbstractCompactionTest {
public void testFileSelectorWithUnclosedFile()
throws IOException, MetadataException, WriteProcessException,
InterruptedException {
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(50);
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(50);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(50);
TsFileResourceList tsFileResources = new TsFileResourceList();
createFiles(2, 2, 3, 100, 0, 0, 50, 50, false, true);
createFiles(2, 3, 5, 50, 250, 250, 50, 50, false, true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionSelectorTest.java
index 307283e71ca..849bb5477b3 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionSelectorTest.java
@@ -46,7 +46,7 @@ public class InnerSpaceCompactionSelectorTest extends
AbstractCompactionTest {
public void setUp()
throws IOException, WriteProcessException, MetadataException,
InterruptedException {
super.setUp();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(2);
}
@After
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
index de01ccb46e6..5cd387cefb9 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelectorTest.java
@@ -75,9 +75,8 @@ public class SizeTieredCompactionSelectorTest {
public void testSubmitWhenSequenceFileIsEmpty() throws Exception {
DataRegion region = new DataRegion("root.test", "1");
TsFileManager manager = region.getTsFileManager();
- int originCandidate =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(30);
+ int originCandidate =
IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(30);
boolean enableUnseqCompaction =
IoTDBDescriptor.getInstance().getConfig().isEnableUnseqSpaceCompaction();
IoTDBDescriptor.getInstance().getConfig().setEnableUnseqSpaceCompaction(true);
@@ -96,9 +95,7 @@ public class SizeTieredCompactionSelectorTest {
}
Assert.assertEquals(3, region.compact());
} finally {
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setMaxInnerCompactionCandidateFileNum(originCandidate);
+
IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(originCandidate);
IoTDBDescriptor.getInstance()
.getConfig()
.setEnableUnseqSpaceCompaction(enableUnseqCompaction);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
index 53674296362..9798f24efa0 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
@@ -73,8 +73,8 @@ public class CompactionConfigRestorer {
config.setTargetChunkPointNum(targetChunkPointNum);
config.setChunkSizeLowerBoundInCompaction(chunkSizeLowerBoundInCompaction);
config.setChunkPointNumLowerBoundInCompaction(chunkPointNumLowerBoundInCompaction);
-
config.setMaxInnerCompactionCandidateFileNum(maxInnerCompactionCandidateFileNum);
-
config.setMaxCrossCompactionCandidateFileNum(maxCrossCompactionCandidateFileNum);
+ config.setFileLimitPerInnerTask(maxInnerCompactionCandidateFileNum);
+ config.setFileLimitPerCrossTask(maxCrossCompactionCandidateFileNum);
config.setCompactionThreadCount(concurrentCompactionThread);
config.setCompactionScheduleIntervalInMs(compactionScheduleIntervalInMs);
config.setCompactionSubmissionIntervalInMs(compactionSubmissionIntervalInMs);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
index e832df7b398..6cd7556006c 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
@@ -110,15 +110,15 @@ public class SettleRequestHandlerTest {
config.setEnableSeqSpaceCompaction(true);
// compaction candidate file num
- int maxInnerCompactionCandidateFileNum =
config.getMaxInnerCompactionCandidateFileNum();
- config.setMaxInnerCompactionCandidateFileNum(2);
+ int maxInnerCompactionCandidateFileNum = config.getFileLimitPerInnerTask();
+ config.setFileLimitPerInnerTask(2);
result = reqHandler.handleSettleRequest(req);
Assert.assertEquals(result.code,
TSStatusCode.UNSUPPORTED_OPERATION.getStatusCode());
String firstTsFilePath = paths.remove(0);
result = reqHandler.handleSettleRequest(req);
Assert.assertEquals(result.code,
TSStatusCode.SUCCESS_STATUS.getStatusCode());
paths.add(0, firstTsFilePath);
-
config.setMaxInnerCompactionCandidateFileNum(maxInnerCompactionCandidateFileNum);
+ config.setFileLimitPerInnerTask(maxInnerCompactionCandidateFileNum);
// not continuous
paths.remove(1);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
index acf7e6eb672..01f311d1337 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
@@ -716,8 +716,8 @@ public class DataRegionTest {
public void testMerge()
throws WriteProcessException, QueryProcessException,
IllegalPathException {
int originCandidateFileNum =
-
IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(9);
+ IoTDBDescriptor.getInstance().getConfig().getFileLimitPerInnerTask();
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(9);
boolean originEnableSeqSpaceCompaction =
IoTDBDescriptor.getInstance().getConfig().isEnableSeqSpaceCompaction();
boolean originEnableUnseqSpaceCompaction =
@@ -772,9 +772,7 @@ public class DataRegionTest {
for (TsFileResource resource : queryDataSource.getUnseqResources()) {
Assert.assertTrue(resource.isClosed());
}
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setMaxInnerCompactionCandidateFileNum(originCandidateFileNum);
+
IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(originCandidateFileNum);
IoTDBDescriptor.getInstance()
.getConfig()
.setEnableSeqSpaceCompaction(originEnableSeqSpaceCompaction);
@@ -786,7 +784,7 @@ public class DataRegionTest {
@Ignore
@Test
public void testDeleteStorageGroupWhenCompacting() throws Exception {
-
IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(10);
+ IoTDBDescriptor.getInstance().getConfig().setFileLimitPerInnerTask(10);
try {
for (int j = 0; j < 10; j++) {
TSRecord record = new TSRecord(j, deviceId);
