[iotdb] branch research/M4-visualization updated: speed up QueryUtils.modifyChunkMetaData by filtering non-overlap deletes

[leirui]

This is an automated email from the ASF dual-hosted git repository.

leirui pushed a commit to branch research/M4-visualization
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/research/M4-visualization by 
this push:
     new 6ab9f7a84a speed up QueryUtils.modifyChunkMetaData by filtering 
non-overlap deletes
6ab9f7a84a is described below

commit 6ab9f7a84ae3ba8e701af7a48715db536041a675
Author: Lei Rui <1010953...@qq.com>
AuthorDate: Tue Jun 28 14:21:26 2022 +0800

    speed up QueryUtils.modifyChunkMetaData by filtering non-overlap deletes
---
 .../iotdb/db/engine/modification/Deletion.java     |  7 +++-
 .../java/org/apache/iotdb/db/utils/QueryUtils.java | 41 +++++++++++++++++++---
 2 files changed, 42 insertions(+), 6 deletions(-)

diff --git 
a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java 
b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
index 5d41dbee3e..742a544a6e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/modification/Deletion.java
@@ -24,7 +24,7 @@ import org.apache.iotdb.db.metadata.PartialPath;
 import java.util.Objects;
 
 /** Deletion is a delete operation on a timeseries. */
-public class Deletion extends Modification {
+public class Deletion extends Modification implements Comparable<Deletion> {
 
   /** data within the interval [startTime, endTime] are to be deleted. */
   private long startTime;
@@ -104,4 +104,9 @@ public class Deletion extends Modification {
         + fileOffset
         + '}';
   }
+
+  @Override
+  public int compareTo(Deletion deletion) {
+    return Long.compare(startTime, deletion.startTime);
+  }
 }
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java 
b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
index 29fab5acdc..c505895910 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java
@@ -49,26 +49,46 @@ public class QueryUtils {
   @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity 
warning
   public static void modifyChunkMetaData(
       List<ChunkMetadata> chunkMetaData, List<Modification> modifications) {
+
+    // sort deletions by startTime. 
不过这里默认modification就是delete，暂时没有考虑其它modifications。
+    List<Deletion> deletions =
+        modifications.stream().map(e -> (Deletion) 
e).sorted().collect(Collectors.toList());
+
+    // 对于每个chunkMetadata，先过滤掉时间范围上不重叠的删除操作，然后再根据版本高低判断是否应用mod
     for (int metaIndex = 0; metaIndex < chunkMetaData.size(); metaIndex++) {
       ChunkMetadata metaData = chunkMetaData.get(metaIndex);
-      for (Modification modification : modifications) {
+      long startTime = metaData.getStartTime();
+      long endTime = metaData.getEndTime();
+      for (Deletion deletion : deletions) {
+        long deleteStartTime = deletion.getStartTime();
+        long deleteEndTime = deletion.getEndTime();
+        if (deleteStartTime > endTime) {
+          break;
+        }
+        if (startTime > deleteEndTime) {
+          continue;
+        }
+        // then deals with deletes that overlap in time with the chunk
+        // check the version number
+
         // When the chunkMetadata come from an old TsFile, the method 
modification.getFileOffset()
         // is gerVersionNum actually. In this case, we compare the versions of 
modification and
         // mataData to determine whether need to do modify.
         if (metaData.isFromOldTsFile()) {
-          if (modification.getFileOffset() > metaData.getVersion()) {
-            doModifyChunkMetaData(modification, metaData);
+          if (deletion.getFileOffset() > metaData.getVersion()) {
+            doModifyChunkMetaData(deletion, metaData);
           }
           continue;
         }
         // The case modification.getFileOffset() == 
metaData.getOffsetOfChunkHeader()
         // is not supposed to exist as getFileOffset() is offset containing 
full chunk,
         // while getOffsetOfChunkHeader() returns the chunk header offset
-        if (modification.getFileOffset() > metaData.getOffsetOfChunkHeader()) {
-          doModifyChunkMetaData(modification, metaData);
+        if (deletion.getFileOffset() > metaData.getOffsetOfChunkHeader()) {
+          doModifyChunkMetaData(deletion, metaData);
         }
       }
     }
+
     // remove chunks that are completely deleted
     chunkMetaData.removeIf(
         metaData -> {
@@ -92,6 +112,17 @@ public class QueryUtils {
   private static void doModifyChunkMetaData(Modification modification, 
ChunkMetadata metaData) {
     if (modification instanceof Deletion) {
       Deletion deletion = (Deletion) modification;
+      System.out.println(
+          "====DEBUG====: doModifyChunkMetaData/insertIntoSortedDeletions: "
+              + "chunkTime=["
+              + metaData.getStartTime()
+              + ","
+              + metaData.getEndTime()
+              + "],deleteTime=["
+              + deletion.getStartTime()
+              + ","
+              + deletion.getEndTime()
+              + "]");
       metaData.insertIntoSortedDeletions(deletion.getStartTime(), 
deletion.getEndTime());
     }
   }