This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new db2b9d959e8 fix compaction task comparator & fix the conflicts between
inner compaction selection conditions (#14496)
db2b9d959e8 is described below
commit db2b9d959e83fab9bad33ea665b274cd1fa1cde5
Author: shuwenwei <[email protected]>
AuthorDate: Thu Dec 19 18:41:27 2024 +0800
fix compaction task comparator & fix the conflicts between inner compaction
selection conditions (#14496)
---
.../DefaultCompactionTaskComparatorImpl.java | 16 +++++++-----
.../impl/NewSizeTieredCompactionSelector.java | 2 +-
.../compaction/CompactionTaskComparatorTest.java | 2 +-
.../NewSizeTieredCompactionSelectorTest.java | 30 ++++++++++++++++++++++
4 files changed, 41 insertions(+), 9 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
index 724d2748bf0..6eb6686f9f4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
@@ -21,11 +21,11 @@ package
org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.compara
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import
org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InsertionCrossSpaceCompactionTask;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionPriority;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -38,6 +38,14 @@ public class DefaultCompactionTaskComparatorImpl implements
ICompactionTaskCompa
@SuppressWarnings({"squid:S3776", "javabugs:S6320"})
@Override
public int compare(AbstractCompactionTask o1, AbstractCompactionTask o2) {
+ if (o1 instanceof RepairUnsortedFileCompactionTask
+ && o2 instanceof RepairUnsortedFileCompactionTask) {
+ return o1.getSerialId() < o2.getSerialId() ? -1 : 1;
+ } else if (o1 instanceof RepairUnsortedFileCompactionTask) {
+ return -1;
+ } else if (o2 instanceof RepairUnsortedFileCompactionTask) {
+ return 1;
+ }
if (o1 instanceof InsertionCrossSpaceCompactionTask
&& o2 instanceof InsertionCrossSpaceCompactionTask) {
return o1.getSerialId() < o2.getSerialId() ? -1 : 1;
@@ -82,12 +90,6 @@ public class DefaultCompactionTaskComparatorImpl implements
ICompactionTaskCompa
public int compareInnerSpaceCompactionTask(
InnerSpaceCompactionTask o1, InnerSpaceCompactionTask o2) {
- // if compactionTaskType of o1 and o2 are different
- // we prefer to execute task type with Repair type
- if (o1.getCompactionTaskType() != o2.getCompactionTaskType()) {
- return o1.getCompactionTaskType() == CompactionTaskType.REPAIR ? -1 : 1;
- }
-
// If the average file size of the two compaction tasks differs by more
than 10%,
// we prefer to execute task with smaller avg file size
double avgFileSize1 = o1.getAvgFileSize();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
index 802e0afd140..b800a1a9815 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/NewSizeTieredCompactionSelector.java
@@ -243,7 +243,7 @@ public class NewSizeTieredCompactionSelector extends
SizeTieredCompactionSelecto
for (TsFileResource resource : lastContinuousSkippedResources) {
long currentFileSize = resource.getTsFileSize();
if (totalFileSize + currentFileSize > singleFileSizeThreshold
- || totalFileNum > totalFileNumUpperBound
+ || totalFileNum + 1 > totalFileNumUpperBound
||
!isFileLevelSatisfied(resource.getTsFileID().getInnerCompactionCount())) {
break;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
index dd868ed805b..4ad08802b68 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
@@ -85,7 +85,7 @@ public class CompactionTaskComparatorTest {
new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), i));
}
compactionTasks[i] =
- new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, true,
resources, 0);
+ new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, i % 2
== 0, resources, 0);
compactionTaskQueue.put(compactionTasks[i]);
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
index 69b54e04aa5..dce350c7895 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/sizetiered/NewSizeTieredCompactionSelectorTest.java
@@ -715,6 +715,36 @@ public class NewSizeTieredCompactionSelectorTest extends
AbstractCompactionTest
Assert.assertEquals(6, task2.getAllSourceTsFiles().size());
}
+ @Test
+ public void testSelectLastSkippedFilesWithTotalFileNumLimit() throws
IOException {
+
IoTDBDescriptor.getInstance().getConfig().setInnerCompactionCandidateFileNum(10);
+
IoTDBDescriptor.getInstance().getConfig().setInnerCompactionTotalFileNumThreshold(10);
+ // TsFiles: [d0], [d1], [d2], [d3], [d3], [d3], [d3], [d3], [d3], [d3],
[d3]
+ for (int i = 0; i < 11; i++) {
+ String device;
+ if (i >= 4) {
+ device = "d3";
+ } else {
+ device = "d" + i;
+ }
+ TsFileResource resource =
+ generateSingleNonAlignedSeriesFile(
+ String.format("%d-%d-%d-0.tsfile", i, i, 0),
+ new TimeRange[] {new TimeRange(100 * i + 1, 100 * (i + 1))},
+ true,
+ device);
+ seqResources.add(resource);
+ }
+ NewSizeTieredCompactionSelector selector =
+ new NewSizeTieredCompactionSelector(
+ COMPACTION_TEST_SG, "0", 0, true, tsFileManager, new
CompactionScheduleContext());
+ List<InnerSpaceCompactionTask> innerSpaceCompactionTasks =
+ selector.selectInnerSpaceTask(seqResources);
+ Assert.assertEquals(1, innerSpaceCompactionTasks.size());
+ Assert.assertEquals(
+ 10,
innerSpaceCompactionTasks.get(0).getSelectedTsFileResourceList().size());
+ }
+
private TsFileResource generateSingleNonAlignedSeriesFile(
String fileName, TimeRange[] chunkTimeRanges, boolean isSeq, String...
devices)
throws IOException {
