This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 7bc541d296e Add avg file size in DefaultCompactionTaskComparatorImpl
(#14270) (#14295)
7bc541d296e is described below
commit 7bc541d296e740aae601807c599155202b31f803
Author: shuwenwei <[email protected]>
AuthorDate: Wed Dec 4 09:40:21 2024 +0800
Add avg file size in DefaultCompactionTaskComparatorImpl (#14270) (#14295)
* add avg file size in DefaultCompactionTaskComparatorImpl
* fix ut
---
.../execute/task/InnerSpaceCompactionTask.java | 4 ++++
.../DefaultCompactionTaskComparatorImpl.java | 12 +++++++++--
.../compaction/CompactionTaskComparatorTest.java | 25 ++++++++++++++++++++--
3 files changed, 37 insertions(+), 4 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index d10297e35c6..cb0e72c3220 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -603,6 +603,10 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
/ filesView.sourceFilesInCompactionPerformer.size();
}
+ public double getAvgFileSize() {
+ return (double) filesView.selectedFileSize /
filesView.sourceFilesInCompactionPerformer.size();
+ }
+
@TestOnly
public void setTargetTsFileResource(TsFileResource targetTsFileResource) {
this.filesView.setTargetFileForRecover(targetTsFileResource);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
index f80b43528f2..724d2748bf0 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
@@ -88,8 +88,16 @@ public class DefaultCompactionTaskComparatorImpl implements
ICompactionTaskCompa
return o1.getCompactionTaskType() == CompactionTaskType.REPAIR ? -1 : 1;
}
- // if the sum of compaction count of the selected files are different
- // we prefer to execute task with smaller compaction count
+ // If the average file size of the two compaction tasks differs by more
than 10%,
+ // we prefer to execute task with smaller avg file size
+ double avgFileSize1 = o1.getAvgFileSize();
+ double avgFileSize2 = o2.getAvgFileSize();
+ if (10 * Math.abs(avgFileSize1 - avgFileSize2) > Math.min(avgFileSize1,
avgFileSize2)) {
+ return Double.compare(avgFileSize1, avgFileSize2);
+ }
+
+ // if the avg of compaction count of the selected files are different
+ // we prefer to execute task with smaller avg compaction count
// this can reduce write amplification
double avgCompactionCount1 = o1.getAvgCompactionCount();
double avgCompactionCount2 = o2.getAvgCompactionCount();
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
index a59f0eb758f..dd868ed805b 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
@@ -74,6 +74,27 @@ public class CompactionTaskComparatorTest {
new CompactionConfigRestorer().restoreCompactionConfig();
}
+ /** Test comparation of tasks with different avg file size */
+ @Test
+ public void testAvgFileSizeCompare() throws InterruptedException {
+ AbstractCompactionTask[] compactionTasks = new AbstractCompactionTask[100];
+ for (int i = 0; i < 10; ++i) {
+ List<TsFileResource> resources = new ArrayList<>();
+ for (int j = i; j < 10; ++j) {
+ resources.add(
+ new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), i));
+ }
+ compactionTasks[i] =
+ new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, true,
resources, 0);
+ compactionTaskQueue.put(compactionTasks[i]);
+ }
+
+ for (int i = 0; i < 10; ++i) {
+ AbstractCompactionTask currentTask = compactionTaskQueue.take();
+ assertTrue(currentTask == compactionTasks[i]);
+ }
+ }
+
/** Test comparation of tasks with different file num */
@Test
public void testFileNumCompare() throws InterruptedException {
@@ -82,7 +103,7 @@ public class CompactionTaskComparatorTest {
List<TsFileResource> resources = new ArrayList<>();
for (int j = i; j < 100; ++j) {
resources.add(
- new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), j));
+ new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), 1));
}
compactionTasks[i] =
new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, true,
resources, 0);
@@ -205,7 +226,7 @@ public class CompactionTaskComparatorTest {
List<TsFileResource> resources = new ArrayList<>();
for (int j = i; j < 100; ++j) {
resources.add(
- new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), j));
+ new FakedTsFileResource(new File(String.format("%d-%d-0-0.tsfile",
i + j, i + j)), 1));
}
innerCompactionTasks[i] =
new FakedInnerSpaceCompactionTask("fakeSg", 0, tsFileManager, true,
resources, 0);
