[iotdb] branch master updated: Pipe: support parameter extractor.history.sloppy-time-range to improve historical data processing efficiency (#11387)

[rong]

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 1448699284e Pipe: support parameter 
extractor.history.sloppy-time-range to improve historical data processing 
efficiency (#11387)
1448699284e is described below

commit 1448699284ef7fe04c48c5c05841aabaf80e642e
Author: Steve Yurong Su <r...@apache.org>
AuthorDate: Wed Oct 25 16:22:16 2023 +0800

    Pipe: support parameter extractor.history.sloppy-time-range to improve 
historical data processing efficiency (#11387)
---
 .../iotdb/db/pipe/config/constant/PipeExtractorConstant.java       | 2 ++
 .../historical/PipeHistoricalDataRegionTsFileExtractor.java        | 7 ++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
index 6a7c095c148..b4be2ed34b9 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
@@ -33,6 +33,8 @@ public class PipeExtractorConstant {
   public static final String EXTRACTOR_HISTORY_ENABLE_KEY = 
"extractor.history.enable";
   public static final String EXTRACTOR_HISTORY_START_TIME = 
"extractor.history.start-time";
   public static final String EXTRACTOR_HISTORY_END_TIME = 
"extractor.history.end-time";
+  public static final String EXTRACTOR_HISTORY_SLOPPY_TIME_RANGE =
+      "extractor.history.sloppy-time-range";
 
   public static final String EXTRACTOR_REALTIME_ENABLE = 
"extractor.realtime.enable";
   public static final String EXTRACTOR_REALTIME_MODE = 
"extractor.realtime.mode";
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java
index b250436622e..1a704b44722 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/historical/PipeHistoricalDataRegionTsFileExtractor.java
@@ -50,6 +50,7 @@ import java.util.stream.Collectors;
 
 import static 
org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY;
 import static 
org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_END_TIME;
+import static 
org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_SLOPPY_TIME_RANGE;
 import static 
org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_START_TIME;
 import static 
org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_DEFAULT_VALUE;
 import static 
org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_PATTERN_KEY;
@@ -74,6 +75,8 @@ public class PipeHistoricalDataRegionTsFileExtractor 
implements PipeHistoricalDa
 
   private long historicalDataExtractionTimeLowerBound; // Arrival time
 
+  private boolean sloppyTimeRange; // true to disable time range filter after 
extraction
+
   private Queue<TsFileResource> pendingQueue;
 
   @Override
@@ -151,6 +154,8 @@ public class PipeHistoricalDataRegionTsFileExtractor 
implements PipeHistoricalDa
         }
       }
     }
+
+    sloppyTimeRange = 
parameters.getBooleanOrDefault(EXTRACTOR_HISTORY_SLOPPY_TIME_RANGE, false);
   }
 
   private void flushDataRegionAllTsFiles() {
@@ -289,7 +294,7 @@ public class PipeHistoricalDataRegionTsFileExtractor 
implements PipeHistoricalDa
             pattern,
             historicalDataExtractionStartTime,
             historicalDataExtractionEndTime,
-            !isTsFileResourceCoveredByTimeRange(resource));
+            !sloppyTimeRange && !isTsFileResourceCoveredByTimeRange(resource));
     
event.increaseReferenceCount(PipeHistoricalDataRegionTsFileExtractor.class.getName());
     try {
       PipeResourceManager.tsfile().unpinTsFileResource(resource);