This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch OffsetPushDown-1.2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dd82b79418ebcd199f0431048ed55fa11a008328 Author: Jackie Tien <[email protected]> AuthorDate: Tue Aug 15 09:51:38 2023 +0800 [IOTDB-6112] Fix Limit & Offset push down doesn't take effect while there exist time filter (cherry picked from commit cdcb4a691d3563f40904db3f2d9b314d573d9c55) --- .../operator/source/AlignedSeriesScanUtil.java | 18 ++++++++---------- .../read/reader/chunk/MemAlignedPageReader.java | 14 ++++++++------ .../tsfile/read/reader/page/AlignedPageReader.java | 16 +++++++++------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index 360211805bb..d36de2e4bf7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -141,12 +141,11 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { && !isFileOverlapped() && !firstTimeSeriesMetadata.isModified()) { Filter queryFilter = scanOptions.getQueryFilter(); - if (queryFilter != null) { - if (!queryFilter.satisfy(firstTimeSeriesMetadata.getStatistics())) { - skipCurrentFile(); - } - } else { + Statistics statistics = firstTimeSeriesMetadata.getStatistics(); + if (queryFilter == null || queryFilter.allSatisfy(statistics)) { skipOffsetByTimeSeriesMetadata(); + } else if (!queryFilter.satisfy(statistics)) { + skipCurrentFile(); } } } @@ -178,12 +177,11 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { protected void filterFirstChunkMetadata() throws IOException { if (firstChunkMetadata != null && !isChunkOverlapped() && !firstChunkMetadata.isModified()) { Filter queryFilter = scanOptions.getQueryFilter(); - if (queryFilter != null) { - if (!queryFilter.satisfy(firstChunkMetadata.getStatistics())) { - skipCurrentChunk(); - } - } else { + Statistics statistics = firstChunkMetadata.getStatistics(); + if (queryFilter == null || queryFilter.allSatisfy(statistics)) { skipOffsetByChunkMetadata(); + } else if (!queryFilter.satisfy(statistics)) { + skipCurrentChunk(); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 6bce32c0a08..b1bba47f129 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -99,17 +99,16 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader { } private boolean pageSatisfy() { - if (valueFilter != null) { - return valueFilter.satisfy(getStatistics()); - } else { + Statistics<? extends Serializable> statistics = getStatistics(); + if (valueFilter == null || valueFilter.allSatisfy(statistics)) { // For aligned series, When we only read some measurements under an aligned device, if the // values of these queried measurements at a timestamp are all null, the timestamp will not be // selected. // NOTE: if we change the read semantic in the future for aligned series, we need to remove // this check here. long rowCount = getTimeStatistics().getCount(); - for (Statistics<? extends Serializable> statistics : getValueStatisticsList()) { - if (statistics == null || statistics.hasNullValue(rowCount)) { + for (Statistics<? extends Serializable> vStatistics : getValueStatisticsList()) { + if (vStatistics == null || vStatistics.hasNullValue(rowCount)) { return true; } } @@ -118,9 +117,12 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader { if (paginationController.hasCurOffset(rowCount)) { paginationController.consumeOffset(rowCount); return false; + } else { + return true; } + } else { + return valueFilter.satisfy(statistics); } - return true; } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java index 8064db9748f..0a93e92f423 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java @@ -118,18 +118,16 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { } private boolean pageSatisfy() { - if (filter != null) { - // TODO accept valueStatisticsList to filter - return filter.satisfy(getStatistics()); - } else { + Statistics statistics = getStatistics(); + if (filter == null || filter.allSatisfy(statistics)) { // For aligned series, When we only query some measurements under an aligned device, if the // values of these queried measurements at a timestamp are all null, the timestamp will not be // selected. // NOTE: if we change the query semantic in the future for aligned series, we need to remove // this check here. long rowCount = getTimeStatistics().getCount(); - for (Statistics statistics : getValueStatisticsList()) { - if (statistics == null || statistics.hasNullValue(rowCount)) { + for (Statistics vStatistics : getValueStatisticsList()) { + if (vStatistics == null || vStatistics.hasNullValue(rowCount)) { return true; } } @@ -138,9 +136,13 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { if (paginationController.hasCurOffset(rowCount)) { paginationController.consumeOffset(rowCount); return false; + } else { + return true; } + } else { + // TODO accept valueStatisticsList to filter + return filter.satisfy(statistics); } - return true; } @Override
