This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch QueryImprove in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 991124ad4523a74cfadb93a9017168aa16f46d9b Author: JackieTien97 <[email protected]> AuthorDate: Fri Mar 17 15:20:14 2023 +0800 Filter unsatisfied TimePartition while preparing resource list --- .../iotdb/db/engine/storagegroup/DataRegion.java | 3 +- .../engine/storagegroup/HashLastFlushTimeMap.java | 35 +++++++++++++++++----- .../storagegroup/IDTableLastFlushTimeMap.java | 3 +- .../db/engine/storagegroup/ILastFlushTimeMap.java | 4 ++- .../apache/iotdb/db/utils/TimePartitionUtils.java | 11 +++++++ 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java index 1864b0425f..cf994427db 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java @@ -1700,7 +1700,8 @@ public class DataRegion implements IDataRegionForQuery { List<PartialPath> pathList, String singleDeviceId, QueryContext context, Filter timeFilter) throws QueryProcessException { try { - List<Long> timePartitions = lastFlushTimeMap.getAllSatisfiedTimePartitions(singleDeviceId); + List<Long> timePartitions = + lastFlushTimeMap.getAllSatisfiedTimePartitions(singleDeviceId, timeFilter); List<TsFileResource> seqResources = getFileResourceListForQuery( tsFileManager.getTsFileList(timePartitions, true), diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java index d34a52cdad..5c337d4709 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/HashLastFlushTimeMap.java @@ -19,10 +19,12 @@ package org.apache.iotdb.db.engine.storagegroup; +import org.apache.iotdb.db.utils.TimePartitionUtils; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -249,12 +251,29 @@ public class HashLastFlushTimeMap implements ILastFlushTimeMap { } @Override - public List<Long> getAllSatisfiedTimePartitions(String deviceId) { - return deviceId == null - ? new ArrayList<>(newlyFlushedPartitionLatestFlushedTimeForEachDevice.keySet()) - : newlyFlushedPartitionLatestFlushedTimeForEachDevice.entrySet().stream() - .filter(entry -> entry.getValue().containsKey(deviceId)) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); + public List<Long> getAllSatisfiedTimePartitions(String deviceId, Filter timeFilter) { + if (deviceId == null) { + return newlyFlushedPartitionLatestFlushedTimeForEachDevice.keySet().stream() + .filter( + stringLongMap -> { + long[] startAndEndTime = + TimePartitionUtils.getStartAndEndTimeForTimePartition(stringLongMap); + return (timeFilter == null + || timeFilter.satisfy(startAndEndTime[0], startAndEndTime[1])); + }) + .collect(Collectors.toList()); + } else { + return newlyFlushedPartitionLatestFlushedTimeForEachDevice.entrySet().stream() + .filter( + entry -> { + long[] startAndEndTime = + TimePartitionUtils.getStartAndEndTimeForTimePartition(entry.getKey()); + return (timeFilter == null + || timeFilter.satisfy(startAndEndTime[0], startAndEndTime[1])) + && entry.getValue().containsKey(deviceId); + }) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java index 3a56a2dd41..5915839319 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/IDTableLastFlushTimeMap.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.storagegroup; import org.apache.iotdb.db.metadata.idtable.IDTable; import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; import java.util.ArrayList; import java.util.HashMap; @@ -187,7 +188,7 @@ public class IDTableLastFlushTimeMap implements ILastFlushTimeMap { } @Override - public List<Long> getAllSatisfiedTimePartitions(String deviceId) { + public List<Long> getAllSatisfiedTimePartitions(String deviceId, Filter timeFilter) { return new ArrayList<>(partitionSet); } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/ILastFlushTimeMap.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/ILastFlushTimeMap.java index 0da4674073..dbd617b2e9 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/ILastFlushTimeMap.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/ILastFlushTimeMap.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.engine.storagegroup; +import org.apache.iotdb.tsfile.read.filter.basic.Filter; + import java.util.List; import java.util.Map; @@ -72,5 +74,5 @@ public interface ILastFlushTimeMap { long getMemSize(long partitionId); - List<Long> getAllSatisfiedTimePartitions(String deviceId); + List<Long> getAllSatisfiedTimePartitions(String deviceId, Filter timeFilter); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java index 763b7e0d5a..2c21eb7d79 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/TimePartitionUtils.java @@ -38,6 +38,17 @@ public class TimePartitionUtils { return timePartitionInterval; } + /** + * get start and end time(included) for one time partition + * + * @param timePartition time partition id + * @return long[2] {startTime, endTime(included)} + */ + public static long[] getStartAndEndTimeForTimePartition(long timePartition) { + long startTime = timePartition * timePartitionInterval; + return new long[] {startTime, startTime + timePartitionInterval - 1}; + } + @TestOnly public static void setTimePartitionInterval(long timePartitionInterval) { TimePartitionUtils.timePartitionInterval = timePartitionInterval;
