This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch rc/1.3.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rc/1.3.3 by this push:
new 4136b3bbe82 Fix some issues of compaction (#13316) (#13325)
4136b3bbe82 is described below
commit 4136b3bbe8217416e7d064c219a40f824ec469b1
Author: shuwenwei <[email protected]>
AuthorDate: Wed Aug 28 18:02:31 2024 +0800
Fix some issues of compaction (#13316) (#13325)
* modify inner compaction task comparator
* fix a bug
* modify selector
* modify selector
* fix ut
* reset one line
* fix comment
* fix comment
* fix comment
---
.../fast/FastAlignedSeriesCompactionExecutor.java | 3 +-
.../FastNonAlignedSeriesCompactionExecutor.java | 5 +-
.../fast/element/ChunkMetadataElement.java | 8 +-
.../DefaultCompactionTaskComparatorImpl.java | 6 +-
.../estimator/AbstractCompactionEstimator.java | 6 ++
.../FastCompactionInnerCompactionEstimator.java | 12 +--
.../FastCrossSpaceCompactionEstimator.java | 12 +--
.../ReadChunkInnerCompactionEstimator.java | 11 +--
.../impl/NewSizeTieredCompactionSelector.java | 36 ++++++---
.../storageengine/dataregion/tsfile/TsFileID.java | 6 +-
.../cross/RewriteCompactionFileSelectorTest.java | 2 +-
.../NewSizeTieredCompactionSelectorTest.java | 90 ++++++++++++++++++++++
12 files changed, 142 insertions(+), 55 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
index aca9508fbe8..3c9c195f686 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
@@ -165,8 +165,7 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
new ChunkMetadataElement(
alignedChunkMetadataList.get(i),
i == alignedChunkMetadataList.size() - 1,
- fileElement,
- isBatchedCompaction));
+ fileElement));
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
index 01a1ba8b9c0..ab7d509c0ec 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
@@ -162,10 +162,7 @@ public class FastNonAlignedSeriesCompactionExecutor
extends SeriesCompactionExec
// add into queue
chunkMetadataQueue.add(
new ChunkMetadataElement(
- chunkMetadata,
- i == iChunkMetadataList.size() - 1,
- fileElement,
- isBatchedCompaction));
+ chunkMetadata, i == iChunkMetadataList.size() - 1,
fileElement));
}
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
index d5b33554ca5..6828841f1bd 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java
@@ -42,18 +42,12 @@ public class ChunkMetadataElement {
public boolean needForceDecodingPage;
- public boolean isBatchedCompaction;
-
public ChunkMetadataElement(
- IChunkMetadata chunkMetadata,
- boolean isLastChunk,
- FileElement fileElement,
- boolean isBatchedCompaction) {
+ IChunkMetadata chunkMetadata, boolean isLastChunk, FileElement
fileElement) {
this.chunkMetadata = chunkMetadata;
this.startTime = chunkMetadata.getStartTime();
this.isLastChunk = isLastChunk;
this.fileElement = fileElement;
- this.isBatchedCompaction = isBatchedCompaction;
}
public void clearChunks() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
index e2df9e4911c..bf892924d6a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
@@ -122,7 +122,11 @@ public class DefaultCompactionTaskComparatorImpl
implements ICompactionTaskCompa
if (2 * fileNumDiff >= Math.min(selectedFilesOfO1.size(),
selectedFilesOfO2.size())) {
return selectedFilesOfO2.size() - selectedFilesOfO1.size();
}
- return 0;
+
+ // if the number of selected files is roughly the same,
+ // we prefer to execute the one with the smaller total
+ // file size
+ return o2.getSelectedFileSize() > o1.getSelectedFileSize() ? -1 : 1;
}
public int compareCrossSpaceCompactionTask(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
index 366e852bbf1..e8eef38b6c1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/AbstractCompactionEstimator.java
@@ -27,6 +27,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
+import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.commons.collections4.map.LRUMap;
import org.apache.tsfile.common.conf.TSFileConfig;
@@ -61,6 +62,11 @@ public abstract class AbstractCompactionEstimator {
protected IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
protected TSFileConfig tsFileConfig =
TSFileDescriptor.getInstance().getConfig();
+ protected long memoryBudgetForFileWriter =
+ (long)
+ ((double) SystemInfo.getInstance().getMemorySizeForCompaction()
+ /
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount()
+ *
IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
protected abstract long calculatingMetadataMemoryCost(CompactionTaskInfo
taskInfo);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
index a97adf27309..32140f5a4e5 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCompactionInnerCompactionEstimator.java
@@ -19,9 +19,7 @@
package
org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import java.io.IOException;
import java.util.List;
@@ -40,12 +38,7 @@ public class FastCompactionInnerCompactionEstimator extends
AbstractInnerSpaceEs
* taskInfo.getMaxChunkMetadataSize());
// add ChunkMetadata size of targetFileWriter
- long sizeForFileWriter =
- (long)
- ((double) SystemInfo.getInstance().getMemorySizeForCompaction()
- /
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount()
- *
IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
- cost += sizeForFileWriter;
+ cost += memoryBudgetForFileWriter;
return cost;
}
@@ -95,6 +88,7 @@ public class FastCompactionInnerCompactionEstimator extends
AbstractInnerSpaceEs
int maxOverlapFileNum =
calculatingMaxOverlapFileNumInSubCompactionTask(resources);
// source files (chunk + uncompressed page) * overlap file num
// target file (chunk + unsealed page writer)
- return (maxOverlapFileNum + 1) * maxConcurrentSeriesNum * (maxChunkSize +
maxPageSize);
+ return (maxOverlapFileNum + 1) * maxConcurrentSeriesNum * (maxChunkSize +
maxPageSize)
+ + memoryBudgetForFileWriter;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
index e9f5d714408..ffd4ceba1be 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/FastCrossSpaceCompactionEstimator.java
@@ -19,9 +19,7 @@
package
org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import java.io.IOException;
import java.util.ArrayList;
@@ -41,12 +39,7 @@ public class FastCrossSpaceCompactionEstimator extends
AbstractCrossSpaceEstimat
* taskInfo.getMaxChunkMetadataSize());
// add ChunkMetadata size of targetFileWriter
- long sizeForFileWriter =
- (long)
- ((double) SystemInfo.getInstance().getMemorySizeForCompaction()
- /
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount()
- *
IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
- cost += sizeForFileWriter;
+ cost += memoryBudgetForFileWriter;
return cost;
}
@@ -100,6 +93,7 @@ public class FastCrossSpaceCompactionEstimator extends
AbstractCrossSpaceEstimat
int maxOverlapFileNum =
calculatingMaxOverlapFileNumInSubCompactionTask(sourceFiles);
// source files (chunk + uncompressed page) * overlap file num
// target files (chunk + unsealed page writer)
- return (maxOverlapFileNum + 1) * maxConcurrentSeriesNum * (maxChunkSize +
maxPageSize);
+ return (maxOverlapFileNum + 1) * maxConcurrentSeriesNum * (maxChunkSize +
maxPageSize)
+ + memoryBudgetForFileWriter;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
index b9593cb126d..cd90dd334f3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/estimator/ReadChunkInnerCompactionEstimator.java
@@ -19,9 +19,7 @@
package
org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
-import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import java.util.List;
@@ -39,12 +37,7 @@ public class ReadChunkInnerCompactionEstimator extends
AbstractInnerSpaceEstimat
* taskInfo.getMaxChunkMetadataSize());
// add ChunkMetadata size of targetFileWriter
- long sizeForFileWriter =
- (long)
- ((double) SystemInfo.getInstance().getMemorySizeForCompaction()
- /
IoTDBDescriptor.getInstance().getConfig().getCompactionThreadCount()
- *
IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
- cost += sizeForFileWriter;
+ cost += memoryBudgetForFileWriter;
return cost;
}
@@ -86,6 +79,6 @@ public class ReadChunkInnerCompactionEstimator extends
AbstractInnerSpaceEstimat
long maxPageSize = tsFileConfig.getPageSizeInByte();
// source files (chunk + uncompressed page)
// target file (chunk + unsealed page writer)
- return 2 * maxConcurrentSeriesNum * (maxChunkSize + maxPageSize);
+ return 2 * maxConcurrentSeriesNum * (maxChunkSize + maxPageSize) +
memoryBudgetForFileWriter;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
index f2015829251..1dde62cb6a4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
@@ -116,29 +116,32 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
@SuppressWarnings("java:S135")
private List<InnerSpaceCompactionTask> selectTasksByLevel(int level) throws
IOException {
- InnerSpaceCompactionTaskSelection levelTaskSelection = new
InnerSpaceCompactionTaskSelection();
+ InnerSpaceCompactionTaskSelection levelTaskSelection =
+ new InnerSpaceCompactionTaskSelection(level);
int startSelectIndex = 0;
while (startSelectIndex < tsFileResourceCandidateList.size()) {
- for (int i = startSelectIndex; i < tsFileResourceCandidateList.size();
i++) {
- TsFileResourceCandidate currentFile =
tsFileResourceCandidateList.get(i);
+ int idx = 0;
+ for (idx = startSelectIndex; idx < tsFileResourceCandidateList.size();
idx++) {
+ TsFileResourceCandidate currentFile =
tsFileResourceCandidateList.get(idx);
long innerCompactionCount =
currentFile.resource.getTsFileID().getInnerCompactionCount();
if (levelTaskSelection.isCurrentTaskEmpty() && innerCompactionCount !=
level) {
continue;
}
- if (!currentFile.isValidCandidate || Math.abs(innerCompactionCount -
level) > maxLevelGap) {
+ if (!currentFile.isValidCandidate) {
levelTaskSelection.endCurrentTaskSelection();
break;
}
boolean skipCurrentFile =
!levelTaskSelection.haveOverlappedDevices(currentFile);
if (skipCurrentFile) {
- levelTaskSelection.addSkippedResource(currentFile, i);
+ levelTaskSelection.addSkippedResource(currentFile, idx);
continue;
}
- if (!levelTaskSelection.currentFileSatisfied(currentFile)) {
+ if (!levelTaskSelection.currentFileSizeSatisfied(currentFile)
+ || !levelTaskSelection.isFileLevelSatisfied(innerCompactionCount))
{
levelTaskSelection.endCurrentTaskSelection();
break;
}
@@ -147,10 +150,10 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
levelTaskSelection.endCurrentTaskSelection();
break;
}
- levelTaskSelection.addSelectedResource(currentFile, i);
+ levelTaskSelection.addSelectedResource(currentFile, idx);
}
levelTaskSelection.endCurrentTaskSelection();
- startSelectIndex = levelTaskSelection.getNextTaskStartIndex();
+ startSelectIndex = Math.min(idx + 1,
levelTaskSelection.getNextTaskStartIndex());
}
return levelTaskSelection.getSelectedTaskList();
}
@@ -158,6 +161,7 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
private class InnerSpaceCompactionTaskSelection {
List<InnerSpaceCompactionTask> selectedTaskList = new ArrayList<>();
+ long level;
List<TsFileResource> currentSelectedResources = new ArrayList<>();
List<TsFileResource> currentSkippedResources = new ArrayList<>();
List<TsFileResource> lastContinuousSkippedResources = new ArrayList<>();
@@ -168,6 +172,10 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
int lastSelectedFileIndex = -1;
int nextTaskStartIndex = -1;
+ private InnerSpaceCompactionTaskSelection(long level) {
+ this.level = level;
+ }
+
private boolean haveOverlappedDevices(TsFileResourceCandidate
resourceCandidate)
throws IOException {
return currentSelectedDevices.isEmpty()
@@ -194,10 +202,14 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
lastContinuousSkippedResources.add(currentFile.resource);
}
- private boolean currentFileSatisfied(TsFileResourceCandidate currentFile) {
+ private boolean currentFileSizeSatisfied(TsFileResourceCandidate
currentFile) {
return currentFile.resource.getTsFileSize() < totalFileSizeThreshold;
}
+ private boolean isFileLevelSatisfied(long innerCompactionCount) {
+ return Math.abs(innerCompactionCount - level) <= maxLevelGap;
+ }
+
private boolean isCurrentTaskEmpty() {
return currentSelectedResources.isEmpty();
}
@@ -218,6 +230,9 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
}
private void endCurrentTaskSelection() {
+ if (isCurrentTaskEmpty()) {
+ return;
+ }
try {
// When the total files size does not exceed the limit of the
// size of a single file, merge all files together and try to include
@@ -226,7 +241,8 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
nextTaskStartIndex = lastSelectedFileIndex + 1;
for (TsFileResource resource : lastContinuousSkippedResources) {
long currentFileSize = resource.getTsFileSize();
- if (totalFileSize + currentFileSize > singleFileSizeThreshold) {
+ if (totalFileSize + currentFileSize > singleFileSizeThreshold
+ ||
!isFileLevelSatisfied(resource.getTsFileID().getInnerCompactionCount())) {
break;
}
currentSkippedResources.add(resource);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
index edb2eefb73c..e3bb6adc748 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileID.java
@@ -84,9 +84,9 @@ public class TsFileID {
}
/**
- * @return a long array whose length is 2, the first long value is tsfile
version, second long
- * value is compaction version, high 32 bit is in-space compaction
count, low 32 bit is
- * cross-space compaction count
+ * @return a long array whose length is 3, the first long value is tsfile
timestamp, the second
+ * long value is tsfile version, the third long value is compaction
version , high 32 bit is
+ * in-space compaction count, low 32 bit is cross-space compaction count
*/
private static long[] splitAndGetVersionArray(String tsFileName) {
String[] names = tsFileName.split(FILE_NAME_SEPARATOR);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java
index fb41e569463..c28f73810c5 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/RewriteCompactionFileSelectorTest.java
@@ -70,7 +70,7 @@ public class RewriteCompactionFileSelectorTest extends
MergeTest {
super.setUp();
IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setCompactionThreadCount(1);
- SystemInfo.getInstance().setMemorySizeForCompaction(100 * 1024 * 1024);
+ SystemInfo.getInstance().setMemorySizeForCompaction(1000 * 1024 * 1024);
}
@After
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
index e8072491fe3..69b54e04aa5 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
@@ -32,6 +32,7 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.New
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.tsfile.exception.write.WriteProcessException;
import org.apache.tsfile.file.metadata.enums.CompressionType;
@@ -625,6 +626,95 @@ public class NewSizeTieredCompactionSelectorTest extends
AbstractCompactionTest
Assert.assertEquals(6, task2.getAllSourceTsFiles().size());
}
+ @Test
+ public void testSkipToPreviousIndexAndSelectSkippedFiles4() throws
IOException {
+ // TsFiles: [d0], [d1], [d2], [d3](level=100), [d3], [d3], [d3], [d3],
[d3], [d3]
+ for (int i = 0; i < 10; i++) {
+ String device;
+ if (i >= 4) {
+ device = "d3";
+ } else {
+ device = "d" + i;
+ }
+ int level = 0;
+ if (i == 3) {
+ level = 100;
+ }
+ TsFileResource resource =
+ generateSingleNonAlignedSeriesFile(
+ String.format("%d-%d-%d-0.tsfile", i, i, level),
+ new TimeRange[] {new TimeRange(100 * i + 1, 100 * (i + 1))},
+ true,
+ device);
+ seqResources.add(resource);
+ }
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setTargetCompactionFileSize(
+ seqResources.get(0).getTsFileSize()
+ + seqResources.get(1).getTsFileSize()
+ + seqResources.get(2).getTsFileSize()
+ + seqResources.get(3).getTsFileSize());
+ NewSizeTieredCompactionSelector selector =
+ new NewSizeTieredCompactionSelector(
+ COMPACTION_TEST_SG, "0", 0, true, tsFileManager, new
CompactionScheduleContext());
+ List<InnerSpaceCompactionTask> innerSpaceCompactionTasks =
+ selector.selectInnerSpaceTask(seqResources);
+ Assert.assertEquals(2, innerSpaceCompactionTasks.size());
+ // select file0, file1, file2
+ InnerSpaceCompactionTask task1 = innerSpaceCompactionTasks.get(0);
+ Assert.assertEquals(3, task1.getSelectedTsFileResourceList().size());
+ Assert.assertEquals(3, task1.getAllSourceTsFiles().size());
+ InnerSpaceCompactionTask task2 = innerSpaceCompactionTasks.get(1);
+ // select file4 - file9
+ Assert.assertEquals(6, task2.getSelectedTsFileResourceList().size());
+ Assert.assertEquals(6, task2.getAllSourceTsFiles().size());
+ }
+
+ @Test
+ public void testSkipToPreviousIndexAndSelectSkippedFiles5() throws
IOException {
+ // TsFiles: [d0], [d1], [d2], [d3](compacting), [d3], [d3], [d3], [d3],
[d3], [d3]
+ for (int i = 0; i < 10; i++) {
+ String device;
+ if (i >= 4) {
+ device = "d3";
+ } else {
+ device = "d" + i;
+ }
+ TsFileResource resource =
+ generateSingleNonAlignedSeriesFile(
+ String.format("%d-%d-%d-0.tsfile", i, i, 0),
+ new TimeRange[] {new TimeRange(100 * i + 1, 100 * (i + 1))},
+ true,
+ device);
+ if (i == 3) {
+ resource.setStatusForTest(TsFileResourceStatus.COMPACTING);
+ }
+ seqResources.add(resource);
+ }
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setTargetCompactionFileSize(
+ seqResources.get(0).getTsFileSize()
+ + seqResources.get(1).getTsFileSize()
+ + seqResources.get(2).getTsFileSize()
+ + seqResources.get(3).getTsFileSize());
+ NewSizeTieredCompactionSelector selector =
+ new NewSizeTieredCompactionSelector(
+ COMPACTION_TEST_SG, "0", 0, true, tsFileManager, new
CompactionScheduleContext());
+ List<InnerSpaceCompactionTask> innerSpaceCompactionTasks =
+ selector.selectInnerSpaceTask(seqResources);
+ Assert.assertEquals(2, innerSpaceCompactionTasks.size());
+ InnerSpaceCompactionTask task1 = innerSpaceCompactionTasks.get(0);
+ // select file0, file1, file2
+ Assert.assertEquals(3, task1.getSelectedTsFileResourceList().size());
+ Assert.assertEquals(3, task1.getAllSourceTsFiles().size());
+ InnerSpaceCompactionTask task2 = innerSpaceCompactionTasks.get(1);
+ // select file4 - file9
+ Assert.assertEquals(6, task2.getSelectedTsFileResourceList().size());
+ Assert.assertEquals(6, task2.getAllSourceTsFiles().size());
+ }
+
private TsFileResource generateSingleNonAlignedSeriesFile(
String fileName, TimeRange[] chunkTimeRanges, boolean isSeq, String...
devices)
throws IOException {
