This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a04c752bc24 fix compaction task comparator & fix the conflicts between
inner compaction selection conditions (#14495)
a04c752bc24 is described below
commit a04c752bc24c0b36b17aa405d2d8eb3596499e53
Author: shuwenwei <[email protected]>
AuthorDate: Fri Dec 20 11:23:42 2024 +0800
fix compaction task comparator & fix the conflicts between inner compaction
selection conditions (#14495)
---
.../DefaultCompactionTaskComparatorImpl.java | 16 +++++++-----
.../impl/NewSizeTieredCompactionSelector.java | 2 +-
.../compaction/CompactionTaskComparatorTest.java | 2 +-
.../NewSizeTieredCompactionSelectorTest.java | 30 ++++++++++++++++++++++
4 files changed, 41 insertions(+), 9 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
index 724d2748bf0..6eb6686f9f4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
@@ -21,11 +21,11 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.compara
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InsertionCrossSpaceCompactionTask;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionPriority;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -38,6 +38,14 @@ public class DefaultCompactionTaskComparatorImpl implements
ICompactionTaskCompa
@SuppressWarnings({"squid:S3776", "javabugs:S6320"})
@Override
public int compare(AbstractCompactionTask o1, AbstractCompactionTask o2) {
+ if (o1 instanceof RepairUnsortedFileCompactionTask
+ && o2 instanceof RepairUnsortedFileCompactionTask) {
+ return o1.getSerialId() < o2.getSerialId() ? -1 : 1;
+ } else if (o1 instanceof RepairUnsortedFileCompactionTask) {
+ return -1;
+ } else if (o2 instanceof RepairUnsortedFileCompactionTask) {
+ return 1;
+ }
if (o1 instanceof InsertionCrossSpaceCompactionTask
&& o2 instanceof InsertionCrossSpaceCompactionTask) {
return o1.getSerialId() < o2.getSerialId() ? -1 : 1;
@@ -82,12 +90,6 @@ public class DefaultCompactionTaskComparatorImpl implements
ICompactionTaskCompa
public int compareInnerSpaceCompactionTask(
InnerSpaceCompactionTask o1, InnerSpaceCompactionTask o2) {
- // if compactionTaskType of o1 and o2 are different
- // we prefer to execute task type with Repair type
- if (o1.getCompactionTaskType() != o2.getCompactionTaskType()) {
- return o1.getCompactionTaskType() == CompactionTaskType.REPAIR ? -1 : 1;
- }
-
// If the average file size of the two compaction tasks differs by more
than 10%,
// we prefer to execute task with smaller avg file size
double avgFileSize1 = o1.getAvgFileSize();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
index 3583ee90f7d..795841e0e16 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
@@ -241,7 +241,7 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
for (TsFileResource resource : lastContinuousSkippedResources) {
long currentFileSize = resource.getTsFileSize();
if (totalFileSize + currentFileSize > singleFileSizeThreshold
- || totalFileNum > totalFileNumUpperBound
+ || totalFileNum + 1 > totalFileNumUpperBound
||
!isFileLevelSatisfied(resource.getTsFileID().getInnerCompactionCount())) {
break;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
index 42a748779e9..4878948ee91 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
@@ -86,7 +86,7 @@ public class CompactionTaskComparatorTest {
new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), i));
}
compactionTasks[i] =
- new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, true,
resources, 0);
+ new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, i % 2
== 0, resources, 0);
compactionTaskQueue.put(compactionTasks[i]);
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
index 6c073a95430..2ede5aa26f6 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
@@ -717,6 +717,36 @@ public class NewSizeTieredCompactionSelectorTest extends
AbstractCompactionTest
Assert.assertEquals(6, task2.getAllSourceTsFiles().size());
}
+ @Test
+ public void testSelectLastSkippedFilesWithTotalFileNumLimit() throws
IOException {
+
IoTDBDescriptor.getInstance().getConfig().setInnerCompactionCandidateFileNum(10);
+
IoTDBDescriptor.getInstance().getConfig().setInnerCompactionTotalFileNumThreshold(10);
+ // TsFiles: [d0], [d1], [d2], [d3], [d3], [d3], [d3], [d3], [d3], [d3],
[d3]
+ for (int i = 0; i < 11; i++) {
+ String device;
+ if (i >= 4) {
+ device = "d3";
+ } else {
+ device = "d" + i;
+ }
+ TsFileResource resource =
+ generateSingleNonAlignedSeriesFile(
+ String.format("%d-%d-%d-0.tsfile", i, i, 0),
+ new TimeRange[] {new TimeRange(100 * i + 1, 100 * (i + 1))},
+ true,
+ device);
+ seqResources.add(resource);
+ }
+ NewSizeTieredCompactionSelector selector =
+ new NewSizeTieredCompactionSelector(
+ COMPACTION_TEST_SG, "0", 0, true, tsFileManager, new
CompactionScheduleContext());
+ List<InnerSpaceCompactionTask> innerSpaceCompactionTasks =
+ selector.selectInnerSpaceTask(seqResources);
+ Assert.assertEquals(1, innerSpaceCompactionTasks.size());
+ Assert.assertEquals(
+ 10,
innerSpaceCompactionTasks.get(0).getSelectedTsFileResourceList().size());
+ }
+
private TsFileResource generateSingleNonAlignedSeriesFile(
String fileName, TimeRange[] chunkTimeRanges, boolean isSeq, String...
devices)
throws IOException {
