(iotdb) branch master updated: Strictly check inner space compaction file size (#11919)

[haonan]

This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 22f915f3159 Strictly check inner space compaction file size (#11919)
22f915f3159 is described below

commit 22f915f315941b374d74d67f9b7f76c0da1752b8
Author: shuwenwei <55970239+shuwen...@users.noreply.github.com>
AuthorDate: Wed Jan 17 19:10:06 2024 +0800

    Strictly check inner space compaction file size (#11919)
---
 .../impl/SizeTieredCompactionSelector.java         | 41 +++++++++++++++-------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
index 19623c4f103..23b31ec314e 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java
@@ -118,26 +118,41 @@ public class SizeTieredCompactionSelector
         selectedFileSize = 0L;
         continue;
       }
-      LOGGER.debug("Current File is {}, size is {}", currentFile, 
currentFile.getTsFileSize());
-      selectedFileList.add(currentFile);
-      selectedFileSize += currentFile.getTsFileSize();
-      LOGGER.debug(
-          "Add tsfile {}, current select file num is {}, size is {}",
-          currentFile,
-          selectedFileList.size(),
-          selectedFileSize);
-      // if the file size or file num reach threshold
-      if (selectedFileSize >= targetCompactionFileSize
-          || selectedFileList.size() >= config.getFileLimitPerInnerTask()) {
-        // submit the task
+
+      long totalSizeIfSelectCurrentFile = selectedFileSize + 
currentFile.getTsFileSize();
+      boolean canNotAddCurrentFileIntoCurrentTask =
+          totalSizeIfSelectCurrentFile > targetCompactionFileSize
+              || selectedFileList.size() >= config.getFileLimitPerInnerTask();
+      if (canNotAddCurrentFileIntoCurrentTask) {
+        // total file size or num will beyond the threshold if select current 
file, stop the
+        // selection of current task
         if (selectedFileList.size() > 1) {
+          // submit the task
           taskList.add(new ArrayList<>(selectedFileList));
         }
+        // add current file in a new selected file list
         selectedFileList = new ArrayList<>();
-        selectedFileSize = 0L;
+        selectedFileList.add(currentFile);
+        selectedFileSize = currentFile.getTsFileSize();
+      } else {
+        LOGGER.debug("Current File is {}, size is {}", currentFile, 
currentFile.getTsFileSize());
+        selectedFileList.add(currentFile);
+        selectedFileSize += currentFile.getTsFileSize();
+        LOGGER.debug(
+            "Add tsfile {}, current select file num is {}, size is {}",
+            currentFile,
+            selectedFileList.size(),
+            selectedFileSize);
       }
     }
 
+    // if the selected file size reach the condition to submit
+    if (selectedFileList.size() == config.getFileLimitPerInnerTask()) {
+      taskList.add(new ArrayList<>(selectedFileList));
+      selectedFileList.clear();
+      selectedFileSize = 0;
+    }
+
     // if next time partition exists
     // submit a merge task even it does not meet the requirement for file num 
or file size
     if (hasNextTimePartition && selectedFileList.size() > 1) {