This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/extendFilter in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit af0a8a6d69e75fb3a2c2cb26ab100f945a6d0478 Author: Minghui Liu <[email protected]> AuthorDate: Fri Nov 24 16:57:13 2023 +0800 rename satisfy to canSkip --- .../operator/source/AlignedSeriesScanUtil.java | 4 +- .../execution/operator/source/SeriesScanUtil.java | 4 +- .../read/reader/chunk/MemAlignedPageReader.java | 68 +++++------ .../read/reader/chunk/MemPageReader.java | 10 +- .../iotdb/tsfile/read/filter/basic/Filter.java | 5 +- .../basic/IDisableStatisticsValueFilter.java | 8 +- .../tsfile/read/filter/basic/ITimeFilter.java | 8 +- .../iotdb/tsfile/read/filter/operator/And.java | 9 +- .../iotdb/tsfile/read/filter/operator/Not.java | 13 ++- .../iotdb/tsfile/read/filter/operator/Or.java | 9 +- .../read/filter/operator/ValueFilterOperators.java | 124 ++++++++++++++------- .../read/reader/chunk/AlignedChunkReader.java | 32 +++--- .../tsfile/read/reader/chunk/ChunkReader.java | 14 +-- .../tsfile/read/reader/page/AlignedPageReader.java | 78 +++++++------ .../iotdb/tsfile/read/reader/page/PageReader.java | 12 +- .../tsfile/read/reader/page/ValuePageReader.java | 3 +- .../reader/series/AbstractFileSeriesReader.java | 16 +-- .../read/reader/series/EmptyFileSeriesReader.java | 4 +- .../read/reader/series/FileSeriesReader.java | 4 +- .../tsfile/read/filter/GroupByFilterTest.java | 22 ++-- .../tsfile/read/filter/GroupByMonthFilterTest.java | 14 +-- .../tsfile/read/filter/StatisticsFilterTest.java | 72 ++++++------ 22 files changed, 301 insertions(+), 232 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index b5e7e2486e2..b4b8fc11fe4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -165,7 +165,7 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { Statistics statistics = firstTimeSeriesMetadata.getStatistics(); if (queryFilter == null || queryFilter.allSatisfy(statistics)) { skipOffsetByTimeSeriesMetadata(); - } else if (!queryFilter.satisfy(statistics)) { + } else if (queryFilter.canSkip(statistics)) { skipCurrentFile(); } } @@ -213,7 +213,7 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { Statistics statistics = firstChunkMetadata.getStatistics(); if (queryFilter == null || queryFilter.allSatisfy(statistics)) { skipOffsetByChunkMetadata(); - } else if (!queryFilter.satisfy(statistics)) { + } else if (queryFilter.canSkip(statistics)) { skipCurrentChunk(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 77f1c100f4d..15806de7fbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -304,7 +304,7 @@ public class SeriesScanUtil { skipCurrentChunk(); paginationController.consumeOffset(rowCount); } - } else if (!queryFilter.satisfy(statistics)) { + } else if (queryFilter.canSkip(statistics)) { skipCurrentChunk(); } } @@ -1059,7 +1059,7 @@ public class SeriesScanUtil { skipCurrentFile(); paginationController.consumeOffset(rowCount); } - } else if (!queryFilter.satisfy(statistics)) { + } else if (queryFilter.canSkip(statistics)) { skipCurrentFile(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 7070f84bc4d..4f3dd7bd7be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -47,8 +47,7 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader { private final AlignedChunkMetadata chunkMetadata; // only used for limit and offset push down optimizer, if we select all columns from aligned - // device, we - // can use statistics to skip. + // device, we can use statistics to skip. // it's only exact while using limit & offset push down private final boolean queryAllSensors; @@ -106,44 +105,51 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader { } } - private boolean pageSatisfy() { + private boolean pageCanSkip() { Statistics<? extends Serializable> statistics = getStatistics(); - if (valueFilter == null || valueFilter.allSatisfy(statistics)) { - // For aligned series, When we only query some measurements under an aligned device, if any - // values of these queried measurements has the same value count as the time column, the - // timestamp will be selected. - // NOTE: if we change the query semantic in the future for aligned series, we need to remove - // this check here. - long rowCount = getTimeStatistics().getCount(); - boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty(); - if (!canUse) { - for (Statistics<? extends Serializable> vStatistics : getValueStatisticsList()) { - if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) { - canUse = true; - break; - } - } - } - if (!canUse) { - return true; - } - // When the number of points in all value pages is the same as that in the time page, it means - // that there is no null value, and all timestamps will be selected. - if (paginationController.hasCurOffset(rowCount)) { - paginationController.consumeOffset(rowCount); + if (valueFilter != null && !valueFilter.allSatisfy(statistics)) { + return valueFilter.canSkip(statistics); + } + + if (!canSkipOffsetByStatistics()) { + return false; + } + + long rowCount = getTimeStatistics().getCount(); + if (paginationController.hasCurOffset(rowCount)) { + paginationController.consumeOffset(rowCount); + return true; + } else { + return false; + } + } + + private boolean canSkipOffsetByStatistics() { + if (queryAllSensors || getValueStatisticsList().isEmpty()) { + return true; + } + + // For aligned series, When we only query some measurements under an aligned device, if any + // values of these queried measurements has the same value count as the time column, the + // timestamp will be selected. + // NOTE: if we change the query semantic in the future for aligned series, we need to remove + // this check here. + long rowCount = getTimeStatistics().getCount(); + for (Statistics<? extends Serializable> vStatistics : getValueStatisticsList()) { + if (vStatistics != null && vStatistics.hasNullValue(rowCount)) { return false; - } else { - return true; } - } else { - return valueFilter.satisfy(statistics); } + + // When the number of points in all value pages is the same as that in the time page, it means + // that there is no null value, and all timestamps will be selected. + return true; } @Override public TsBlock getAllSatisfiedData() { builder.reset(); - if (!pageSatisfy()) { + if (pageCanSkip()) { return builder.build(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java index fa71bc3d894..2c7705bbc5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java @@ -95,18 +95,18 @@ public class MemPageReader implements IPageReader { return batchData.flip(); } - private boolean pageSatisfy() { + private boolean pageCanSkip() { Statistics<? extends Serializable> statistics = getStatistics(); if (valueFilter == null || valueFilter.allSatisfy(statistics)) { long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { paginationController.consumeOffset(rowCount); - return false; - } else { return true; + } else { + return false; } } else { - return valueFilter.satisfy(statistics); + return valueFilter.canSkip(statistics); } } @@ -116,7 +116,7 @@ public class MemPageReader implements IPageReader { TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(dataType)); TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); ColumnBuilder valueBuilder = builder.getColumnBuilder(0); - if (pageSatisfy()) { + if (!pageCanSkip()) { switch (dataType) { case BOOLEAN: doWithBoolean(builder, timeBuilder, valueBuilder); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java index 263f1d9e6fe..fe3b2f93329 100755 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java @@ -36,6 +36,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.List; @@ -70,7 +71,7 @@ public interface Filter { * @param statistics statistics with min time, max time, min value, max value. * @return false if there are no data points satisfied with the filter */ - boolean satisfy(Statistics statistics); + boolean canSkip(Statistics<? extends Serializable> statistics); /** * To examine whether all data points are satisfied with the filter. @@ -78,7 +79,7 @@ public interface Filter { * @param statistics statistics with min time, max time, min value, max value. * @return true if all data points are satisfied with the filter */ - boolean allSatisfy(Statistics statistics); + boolean allSatisfy(Statistics<? extends Serializable> statistics); /** * To examine whether the min time and max time are satisfied with the filter. diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IDisableStatisticsValueFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IDisableStatisticsValueFilter.java index 9de9704ad42..ca96b77ffd2 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IDisableStatisticsValueFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IDisableStatisticsValueFilter.java @@ -21,13 +21,15 @@ package org.apache.iotdb.tsfile.read.filter.basic; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; +import java.io.Serializable; + public interface IDisableStatisticsValueFilter extends IValueFilter { - default boolean satisfy(Statistics statistics) { - return true; + default boolean canSkip(Statistics<? extends Serializable> statistics) { + return false; } - default boolean allSatisfy(Statistics statistics) { + default boolean allSatisfy(Statistics<? extends Serializable> statistics) { return false; } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java index 2c96c897f2e..6f445c8361b 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java @@ -21,13 +21,15 @@ package org.apache.iotdb.tsfile.read.filter.basic; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; +import java.io.Serializable; + public interface ITimeFilter extends Filter { - default boolean satisfy(Statistics statistics) { - return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + default boolean canSkip(Statistics<? extends Serializable> statistics) { + return !satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } - default boolean allSatisfy(Statistics statistics) { + default boolean allSatisfy(Statistics<? extends Serializable> statistics) { return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java index 7b144a6ed01..a6474aa1f4f 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java @@ -25,6 +25,7 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.basic.OperatorType; import org.apache.iotdb.tsfile.read.filter.operator.base.BinaryLogicalFilter; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -45,12 +46,14 @@ public class And extends BinaryLogicalFilter { } @Override - public boolean satisfy(Statistics statistics) { - return left.satisfy(statistics) && right.satisfy(statistics); + public boolean canSkip(Statistics<? extends Serializable> statistics) { + // we can drop a chunk of records if we know that either the left or the right predicate agrees + // that no matter what we don't need this chunk. + return left.canSkip(statistics) || right.canSkip(statistics); } @Override - public boolean allSatisfy(Statistics statistics) { + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { return left.allSatisfy(statistics) && right.allSatisfy(statistics); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java index 2246edaa7b4..a8894eebd4f 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java @@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -54,23 +55,23 @@ public class Not implements Filter { } @Override - public boolean satisfy(Statistics statistics) { - throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); + public boolean canSkip(Statistics<? extends Serializable> statistics) { + throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } @Override - public boolean allSatisfy(Statistics statistics) { - throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { + throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } @Override public boolean satisfyStartEndTime(long startTime, long endTime) { - throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); + throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } @Override public boolean containStartEndTime(long startTime, long endTime) { - throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); + throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } public Filter getFilter() { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java index aa4adab9809..3d5777c2e22 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java @@ -25,6 +25,7 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.basic.OperatorType; import org.apache.iotdb.tsfile.read.filter.operator.base.BinaryLogicalFilter; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -45,12 +46,14 @@ public class Or extends BinaryLogicalFilter { } @Override - public boolean satisfy(Statistics statistics) { - return left.satisfy(statistics) || right.satisfy(statistics); + public boolean canSkip(Statistics<? extends Serializable> statistics) { + // we can only drop a chunk of records if we know that both the left and right predicates agree + // that no matter what we don't need this chunk. + return left.canSkip(statistics) && right.canSkip(statistics); } @Override - public boolean allSatisfy(Statistics statistics) { + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { return left.allSatisfy(statistics) || right.allSatisfy(statistics); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java index 7ef646a8775..26c5b22232e 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java @@ -35,6 +35,7 @@ import org.apache.iotdb.tsfile.utils.RegexUtils; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Objects; import java.util.Set; @@ -101,6 +102,7 @@ public final class ValueFilterOperators { super(measurement, constant); } + @SuppressWarnings("unchecked") public ValueEq(ByteBuffer buffer) { this(ReadWriteIOUtils.readString(buffer), (T) ReadWriteIOUtils.readObject(buffer)); } @@ -111,16 +113,20 @@ public final class ValueFilterOperators { } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return constant.compareTo((T) statistics.getMinValue()) >= 0 - && constant.compareTo((T) statistics.getMaxValue()) <= 0; + + // drop if value < min || value > max + return constant.compareTo((T) statistics.getMinValue()) < 0 + || constant.compareTo((T) statistics.getMaxValue()) > 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } @@ -153,6 +159,7 @@ public final class ValueFilterOperators { super(measurement, constant); } + @SuppressWarnings("unchecked") public ValueNotEq(ByteBuffer buffer) { this(ReadWriteIOUtils.readString(buffer), (T) ReadWriteIOUtils.readObject(buffer)); } @@ -163,16 +170,20 @@ public final class ValueFilterOperators { } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return !(constant.compareTo((T) statistics.getMinValue()) == 0 - && constant.compareTo((T) statistics.getMaxValue()) == 0); + + // drop if this is a column where min = max = value + return constant.compareTo((T) statistics.getMinValue()) == 0 + && constant.compareTo((T) statistics.getMaxValue()) == 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } @@ -203,25 +214,31 @@ public final class ValueFilterOperators { super(measurement, Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG)); } + @SuppressWarnings("unchecked") public ValueLt(ByteBuffer buffer) { this(ReadWriteIOUtils.readString(buffer), (T) ReadWriteIOUtils.readObject(buffer)); } @Override + @SuppressWarnings("unchecked") public boolean satisfy(long time, Object value) { return constant.compareTo((T) value) > 0; } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return constant.compareTo((T) statistics.getMinValue()) > 0; + + // drop if value <= min + return constant.compareTo((T) statistics.getMinValue()) <= 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } @@ -251,25 +268,31 @@ public final class ValueFilterOperators { super(measurement, Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG)); } + @SuppressWarnings("unchecked") public ValueLtEq(ByteBuffer buffer) { this(ReadWriteIOUtils.readString(buffer), (T) ReadWriteIOUtils.readObject(buffer)); } @Override + @SuppressWarnings("unchecked") public boolean satisfy(long time, Object value) { return constant.compareTo((T) value) >= 0; } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return constant.compareTo((T) statistics.getMinValue()) >= 0; + + // drop if value < min + return constant.compareTo((T) statistics.getMinValue()) < 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } @@ -299,25 +322,31 @@ public final class ValueFilterOperators { super(measurement, Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG)); } + @SuppressWarnings("unchecked") public ValueGt(ByteBuffer buffer) { this(ReadWriteIOUtils.readString(buffer), (T) ReadWriteIOUtils.readObject(buffer)); } @Override + @SuppressWarnings("unchecked") public boolean satisfy(long time, Object value) { return constant.compareTo((T) value) < 0; } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return constant.compareTo((T) statistics.getMaxValue()) < 0; + + // drop if value >= max + return constant.compareTo((T) statistics.getMaxValue()) >= 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } @@ -347,25 +376,31 @@ public final class ValueFilterOperators { super(measurement, Objects.requireNonNull(constant, CONSTANT_CANNOT_BE_NULL_MSG)); } + @SuppressWarnings("unchecked") public ValueGtEq(ByteBuffer buffer) { this(ReadWriteIOUtils.readString(buffer), (T) ReadWriteIOUtils.readObject(buffer)); } @Override + @SuppressWarnings("unchecked") public boolean satisfy(long time, Object value) { return constant.compareTo((T) value) <= 0; } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return constant.compareTo((T) statistics.getMaxValue()) <= 0; + + // drop if value > max + return constant.compareTo((T) statistics.getMaxValue()) > 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } @@ -439,6 +474,7 @@ public final class ValueFilterOperators { super(measurement, min, max); } + @SuppressWarnings("unchecked") public ValueBetweenAnd(ByteBuffer buffer) { this( ReadWriteIOUtils.readString(buffer), @@ -447,26 +483,29 @@ public final class ValueFilterOperators { } @Override + @SuppressWarnings("unchecked") public boolean satisfy(long time, Object value) { return min.compareTo((T) value) <= 0 && max.compareTo((T) value) >= 0; } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return (((T) statistics.getMaxValue()).compareTo(min) >= 0 - && ((T) statistics.getMinValue()).compareTo(max) <= 0); + return ((T) statistics.getMaxValue()).compareTo(min) >= 0 + && ((T) statistics.getMinValue()).compareTo(max) <= 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } - return (((T) statistics.getMinValue()).compareTo(min) >= 0 - && ((T) statistics.getMaxValue()).compareTo(max) <= 0); + return ((T) statistics.getMinValue()).compareTo(min) >= 0 + && ((T) statistics.getMaxValue()).compareTo(max) <= 0; } @Override @@ -492,6 +531,7 @@ public final class ValueFilterOperators { super(measurement, min, max); } + @SuppressWarnings("unchecked") public ValueNotBetweenAnd(ByteBuffer buffer) { this( ReadWriteIOUtils.readString(buffer), @@ -500,26 +540,30 @@ public final class ValueFilterOperators { } @Override + @SuppressWarnings("unchecked") public boolean satisfy(long time, Object value) { return min.compareTo((T) value) > 0 || max.compareTo((T) value) < 0; } @Override - public boolean satisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean canSkip(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; + return false; } - return (((T) statistics.getMinValue()).compareTo(min) < 0 - || ((T) statistics.getMaxValue()).compareTo(max) > 0); + + return ((T) statistics.getMinValue()).compareTo(min) >= 0 + && ((T) statistics.getMaxValue()).compareTo(max) <= 0; } @Override - public boolean allSatisfy(Statistics statistics) { + @SuppressWarnings("unchecked") + public boolean allSatisfy(Statistics<? extends Serializable> statistics) { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return false; } - return (((T) statistics.getMinValue()).compareTo(max) > 0 - || ((T) statistics.getMaxValue()).compareTo(min) < 0); + return ((T) statistics.getMinValue()).compareTo(max) > 0 + || ((T) statistics.getMaxValue()).compareTo(min) < 0; } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java index bbbb5fc8864..914efc3db58 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java @@ -197,22 +197,22 @@ public class AlignedChunkReader implements IChunkReader { } /** used for value page filter */ - protected boolean pageSatisfied(PageHeader pageHeader, List<TimeRange> valueDeleteInterval) { + protected boolean pageCanSkip(PageHeader pageHeader, List<TimeRange> valueDeleteInterval) { if (currentTimestamp > pageHeader.getEndTime()) { // used for chunk reader by timestamp - return false; + return true; } if (valueDeleteInterval != null) { for (TimeRange range : valueDeleteInterval) { if (range.contains(pageHeader.getStartTime(), pageHeader.getEndTime())) { - return false; + return true; } if (range.overlaps(new TimeRange(pageHeader.getStartTime(), pageHeader.getEndTime()))) { pageHeader.setModified(true); } } } - return filter == null || filter.satisfy(pageHeader.getStatistics()); + return filter != null && filter.canSkip(pageHeader.getStatistics()); } private AlignedPageReader constructPageReaderForNextPage( @@ -233,9 +233,17 @@ public class AlignedChunkReader implements IChunkReader { valuePageDataList.add(null); valueDataTypeList.add(null); valueDecoderList.add(null); - } else if (pageSatisfied( - valuePageHeader.get(i), - valueDeleteIntervalList.get(i))) { // if the page is satisfied, deserialize it + } else if (pageCanSkip(valuePageHeader.get(i), valueDeleteIntervalList.get(i))) { + valueChunkDataBufferList + .get(i) + .position( + valueChunkDataBufferList.get(i).position() + + valuePageHeader.get(i).getCompressedSize()); + valuePageHeaderList.add(null); + valuePageDataList.add(null); + valueDataTypeList.add(null); + valueDecoderList.add(null); + } else { getPageInfo( valuePageHeader.get(i), valueChunkDataBufferList.get(i), @@ -246,16 +254,6 @@ public class AlignedChunkReader implements IChunkReader { valueDataTypeList.add(valuePageInfo.dataType); valueDecoderList.add(valuePageInfo.decoder); exist = true; - } else { // if the page is not satisfied, just skip it - valueChunkDataBufferList - .get(i) - .position( - valueChunkDataBufferList.get(i).position() - + valuePageHeader.get(i).getCompressedSize()); - valuePageHeaderList.add(null); - valuePageDataList.add(null); - valueDataTypeList.add(null); - valueDecoderList.add(null); } } if (!exist) { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java index da73339cd1e..c3557cfee8a 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java @@ -114,10 +114,10 @@ public class ChunkReader implements IChunkReader { pageHeader = PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType()); } // if the current page satisfies - if (pageSatisfied(pageHeader)) { - pageReaderList.add(constructPageReaderForNextPage(pageHeader)); - } else { + if (pageCanSkip(pageHeader)) { skipBytesInStreamByLength(pageHeader.getCompressedSize()); + } else { + pageReaderList.add(constructPageReaderForNextPage(pageHeader)); } } } @@ -146,22 +146,22 @@ public class ChunkReader implements IChunkReader { chunkDataBuffer.position(chunkDataBuffer.position() + length); } - protected boolean pageSatisfied(PageHeader pageHeader) { + protected boolean pageCanSkip(PageHeader pageHeader) { if (currentTimestamp > pageHeader.getEndTime()) { // used for chunk reader by timestamp - return false; + return true; } if (deleteIntervalList != null) { for (TimeRange range : deleteIntervalList) { if (range.contains(pageHeader.getStartTime(), pageHeader.getEndTime())) { - return false; + return true; } if (range.overlaps(new TimeRange(pageHeader.getStartTime(), pageHeader.getEndTime()))) { pageHeader.setModified(true); } } } - return filter == null || filter.satisfy(pageHeader.getStatistics()); + return filter != null && filter.canSkip(pageHeader.getStatistics()); } private PageReader constructPageReaderForNextPage(PageHeader pageHeader) throws IOException { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java index 67058a9661f..fc39a2cb07f 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java @@ -126,39 +126,45 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { return pageData.flip(); } - private boolean pageSatisfy() { - Statistics statistics = getStatistics(); - if (filter == null || filter.allSatisfy(statistics)) { - // For aligned series, When we only query some measurements under an aligned device, if any - // values of these queried measurements has the same value count as the time column, the - // timestamp will be selected. - // NOTE: if we change the query semantic in the future for aligned series, we need to remove - // this check here. - long rowCount = getTimeStatistics().getCount(); - boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty(); - if (!canUse) { - for (Statistics vStatistics : getValueStatisticsList()) { - if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) { - canUse = true; - break; - } - } - } - if (!canUse) { - return true; - } - // When the number of points in all value pages is the same as that in the time page, it means - // that there is no null value, and all timestamps will be selected. - if (paginationController.hasCurOffset(rowCount)) { - paginationController.consumeOffset(rowCount); + private boolean pageCanSkip() { + Statistics<? extends Serializable> statistics = getStatistics(); + if (filter != null && !filter.allSatisfy(statistics)) { + return filter.canSkip(statistics); + } + + if (!canSkipOffsetByStatistics()) { + return false; + } + + long rowCount = getTimeStatistics().getCount(); + if (paginationController.hasCurOffset(rowCount)) { + paginationController.consumeOffset(rowCount); + return true; + } else { + return false; + } + } + + private boolean canSkipOffsetByStatistics() { + if (queryAllSensors || getValueStatisticsList().isEmpty()) { + return true; + } + + // For aligned series, When we only query some measurements under an aligned device, if any + // values of these queried measurements has the same value count as the time column, the + // timestamp will be selected. + // NOTE: if we change the query semantic in the future for aligned series, we need to remove + // this check here. + long rowCount = getTimeStatistics().getCount(); + for (Statistics<? extends Serializable> vStatistics : getValueStatisticsList()) { + if (vStatistics != null && vStatistics.hasNullValue(rowCount)) { return false; - } else { - return true; } - } else { - // TODO accept valueStatisticsList to filter - return filter.satisfy(statistics); } + + // When the number of points in all value pages is the same as that in the time page, it means + // that there is no null value, and all timestamps will be selected. + return true; } public IPointReader getLazyPointReader() throws IOException { @@ -184,7 +190,7 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { @Override public TsBlock getAllSatisfiedData() throws IOException { - if (!pageSatisfy()) { + if (pageCanSkip()) { return builder.build(); } @@ -374,25 +380,25 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader { } @Override - public Statistics getStatistics() { + public Statistics<? extends Serializable> getStatistics() { return valuePageReaderList.size() == 1 && valuePageReaderList.get(0) != null ? valuePageReaderList.get(0).getStatistics() : timePageReader.getStatistics(); } @Override - public Statistics getStatistics(int index) { + public Statistics<? extends Serializable> getStatistics(int index) { ValuePageReader valuePageReader = valuePageReaderList.get(index); return valuePageReader == null ? null : valuePageReader.getStatistics(); } @Override - public Statistics getTimeStatistics() { + public Statistics<? extends Serializable> getTimeStatistics() { return timePageReader.getStatistics(); } - private List<Statistics<Serializable>> getValueStatisticsList() { - List<Statistics<Serializable>> valueStatisticsList = new ArrayList<>(); + private List<Statistics<? extends Serializable>> getValueStatisticsList() { + List<Statistics<? extends Serializable>> valueStatisticsList = new ArrayList<>(); for (ValuePageReader v : valuePageReaderList) { valueStatisticsList.add(v == null ? null : v.getStatistics()); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index b5cd89eb047..572c0bb9162 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -116,7 +116,7 @@ public class PageReader implements IPageReader { @Override public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { BatchData pageData = BatchDataFactory.createBatchData(dataType, ascending, false); - if (filter == null || filter.satisfy(getStatistics())) { + if (filter == null || !filter.canSkip(getStatistics())) { while (timeDecoder.hasNext(timeBuffer)) { long timestamp = timeDecoder.readLong(timeBuffer); switch (dataType) { @@ -164,18 +164,18 @@ public class PageReader implements IPageReader { return pageData.flip(); } - private boolean pageSatisfy() { + private boolean pageCanSkip() { Statistics statistics = getStatistics(); if (filter == null || filter.allSatisfy(statistics)) { long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { paginationController.consumeOffset(rowCount); - return false; - } else { return true; + } else { + return false; } } else { - return filter.satisfy(statistics); + return filter.canSkip(statistics); } } @@ -184,7 +184,7 @@ public class PageReader implements IPageReader { TsBlockBuilder builder = new TsBlockBuilder(Collections.singletonList(dataType)); TimeColumnBuilder timeBuilder = builder.getTimeColumnBuilder(); ColumnBuilder valueBuilder = builder.getColumnBuilder(0); - if (pageSatisfy()) { + if (!pageCanSkip()) { switch (dataType) { case BOOLEAN: while (timeDecoder.hasNext(timeBuffer)) { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java index 57a38795103..d18818c5476 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/ValuePageReader.java @@ -33,6 +33,7 @@ import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; @@ -510,7 +511,7 @@ public class ValuePageReader { } } - public Statistics getStatistics() { + public Statistics<? extends Serializable> getStatistics() { return pageHeader.getStatistics(); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/AbstractFileSeriesReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/AbstractFileSeriesReader.java index 24ff3fdb390..648e0a6a1c0 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/AbstractFileSeriesReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/AbstractFileSeriesReader.java @@ -60,13 +60,15 @@ public abstract class AbstractFileSeriesReader implements IBatchReader { while (chunkToRead < chunkMetadataList.size()) { IChunkMetadata chunkMetaData = nextChunkMeta(); - if (chunkSatisfied(chunkMetaData)) { - // chunk metadata satisfy the condition - initChunkReader(chunkMetaData); + if (chunkCanSkip(chunkMetaData)) { + continue; + } + + // chunk metadata satisfy the condition + initChunkReader(chunkMetaData); - if (chunkReader.hasNextSatisfiedPage()) { - return true; - } + if (chunkReader.hasNextSatisfiedPage()) { + return true; } } return false; @@ -79,7 +81,7 @@ public abstract class AbstractFileSeriesReader implements IBatchReader { protected abstract void initChunkReader(IChunkMetadata chunkMetaData) throws IOException; - protected abstract boolean chunkSatisfied(IChunkMetadata chunkMetaData); + protected abstract boolean chunkCanSkip(IChunkMetadata chunkMetaData); @Override public void close() throws IOException { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java index 0316987b19c..ed3bda8cf8e 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/EmptyFileSeriesReader.java @@ -36,8 +36,8 @@ public class EmptyFileSeriesReader extends AbstractFileSeriesReader { } @Override - protected boolean chunkSatisfied(IChunkMetadata chunkMetaData) { - return false; + protected boolean chunkCanSkip(IChunkMetadata chunkMetaData) { + return true; } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java index 2e1e5090bd8..19fa52baf20 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java @@ -61,7 +61,7 @@ public class FileSeriesReader extends AbstractFileSeriesReader { } @Override - protected boolean chunkSatisfied(IChunkMetadata chunkMetaData) { - return filter == null || filter.satisfy(chunkMetaData.getStatistics()); + protected boolean chunkCanSkip(IChunkMetadata chunkMetaData) { + return filter != null && filter.canSkip(chunkMetaData.getStatistics()); } } diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByFilterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByFilterTest.java index 9604821c4e3..b7b13abd5ba 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByFilterTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByFilterTest.java @@ -44,47 +44,47 @@ public class GroupByFilterTest { Statistics statistics = new LongStatistics(); statistics.setStartTime(0); statistics.setEndTime(7); - assertFalse(groupByFilter.satisfy(statistics)); + assertTrue(groupByFilter.canSkip(statistics)); statistics.setStartTime(8 + 30 * 24 + 3 + 6 + 1); statistics.setEndTime(8 + 30 * 24 + 3 + 6 + 2); - assertFalse(groupByFilter.satisfy(statistics)); + assertTrue(groupByFilter.canSkip(statistics)); statistics.setStartTime(0); statistics.setEndTime(9); - assertTrue(groupByFilter.satisfy(statistics)); + assertFalse(groupByFilter.canSkip(statistics)); statistics.setStartTime(32); statistics.setEndTime(34); - assertTrue(groupByFilter.satisfy(statistics)); + assertFalse(groupByFilter.canSkip(statistics)); statistics.setStartTime(32); statistics.setEndTime(36); - assertTrue(groupByFilter.satisfy(statistics)); + assertFalse(groupByFilter.canSkip(statistics)); statistics.setStartTime(36); statistics.setEndTime(37); - assertFalse(groupByFilter.satisfy(statistics)); + assertTrue(groupByFilter.canSkip(statistics)); statistics.setStartTime(36); statistics.setEndTime(55); - assertFalse(groupByFilter.satisfy(statistics)); + assertTrue(groupByFilter.canSkip(statistics)); statistics.setStartTime(35); statistics.setEndTime(56); - assertTrue(groupByFilter.satisfy(statistics)); + assertFalse(groupByFilter.canSkip(statistics)); statistics.setStartTime(35); statistics.setEndTime(58); - assertTrue(groupByFilter.satisfy(statistics)); + assertFalse(groupByFilter.canSkip(statistics)); statistics.setStartTime(8 + 30 * 24 + 3 + 1); statistics.setEndTime(8 + 30 * 24 + 5); - assertFalse(groupByFilter.satisfy(statistics)); + assertTrue(groupByFilter.canSkip(statistics)); statistics.setStartTime(8 + 30 * 24 + 3 + 1); statistics.setEndTime(8 + 30 * 24 + 8); - assertFalse(groupByFilter.satisfy(statistics)); + assertTrue(groupByFilter.canSkip(statistics)); } @Test diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java index 0f44cb2a89a..6a68fc279a0 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByMonthFilterTest.java @@ -194,37 +194,37 @@ public class GroupByMonthFilterTest { Statistics statistics = new LongStatistics(); statistics.setStartTime(0); statistics.setEndTime(MS_TO_DAY); - assertTrue(filter.satisfy(statistics)); + assertFalse(filter.canSkip(statistics)); // 1970-01-01 20:00:00 - 1970-01-02 08:00:00 statistics.setStartTime(MS_TO_DAY / 2); statistics.setEndTime(MS_TO_DAY); - assertTrue(filter.satisfy(statistics)); + assertFalse(filter.canSkip(statistics)); // 1970-01-01 20:00:00 - 1970-01-03 08:00:00 statistics.setStartTime(MS_TO_DAY / 2); statistics.setEndTime(MS_TO_DAY * 2); - assertTrue(filter.satisfy(statistics)); + assertFalse(filter.canSkip(statistics)); // 1970-01-02 08:00:00 - 1970-01-03 08:00:00 statistics.setStartTime(MS_TO_DAY); statistics.setEndTime(MS_TO_DAY * 2); - assertFalse(filter.satisfy(statistics)); + assertTrue(filter.canSkip(statistics)); // 1970-02-28 08:00:00 - 1970-03-01 07:59:59 statistics.setStartTime(5011200000L); statistics.setEndTime(5097599000L); - assertFalse(filter.satisfy(statistics)); + assertTrue(filter.canSkip(statistics)); // 1970-03-01 09:00:00 - 1970-03-01 10:00:00 statistics.setStartTime(5101200000L); statistics.setEndTime(5104800000L); - assertTrue(filter.satisfy(statistics)); + assertFalse(filter.canSkip(statistics)); // 1970-05-01 07:00:00 - 1970-05-01 08:00:00 statistics.setStartTime(10364400000L); statistics.setEndTime(10368000000L); - assertTrue(filter.satisfy(statistics)); + assertFalse(filter.canSkip(statistics)); // 1970-01-02 07:59:59 assertTrue(filter.satisfy(86399000L, null)); diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java index 56e4ef45fa5..3ad79ed636c 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java @@ -55,15 +55,15 @@ public class StatisticsFilterTest { @Test public void testEq() { Filter timeEq = TimeFilter.eq(10L); - Assert.assertTrue(timeEq.satisfy(statistics1)); - Assert.assertFalse(timeEq.satisfy(statistics2)); + Assert.assertFalse(timeEq.canSkip(statistics1)); + Assert.assertTrue(timeEq.canSkip(statistics2)); Assert.assertFalse(timeEq.allSatisfy(statistics1)); Assert.assertFalse(timeEq.allSatisfy(statistics2)); Assert.assertTrue(timeEq.allSatisfy(statistics3)); Filter valueEq = ValueFilter.eq(101L); - Assert.assertFalse(valueEq.satisfy(statistics1)); - Assert.assertTrue(valueEq.satisfy(statistics2)); + Assert.assertTrue(valueEq.canSkip(statistics1)); + Assert.assertFalse(valueEq.canSkip(statistics2)); Assert.assertFalse(valueEq.allSatisfy(statistics1)); Assert.assertFalse(valueEq.allSatisfy(statistics2)); Assert.assertFalse(valueEq.allSatisfy(statistics3)); @@ -72,15 +72,15 @@ public class StatisticsFilterTest { @Test public void testNotEq() { Filter timeNotEq = TimeFilter.notEq(10L); - Assert.assertTrue(timeNotEq.satisfy(statistics1)); - Assert.assertTrue(timeNotEq.satisfy(statistics2)); + Assert.assertFalse(timeNotEq.canSkip(statistics1)); + Assert.assertFalse(timeNotEq.canSkip(statistics2)); Assert.assertFalse(timeNotEq.allSatisfy(statistics1)); Assert.assertTrue(timeNotEq.allSatisfy(statistics2)); Assert.assertFalse(timeNotEq.allSatisfy(statistics3)); Filter valueNotEq = ValueFilter.notEq(101L); - Assert.assertTrue(valueNotEq.satisfy(statistics1)); - Assert.assertTrue(valueNotEq.satisfy(statistics2)); + Assert.assertFalse(valueNotEq.canSkip(statistics1)); + Assert.assertFalse(valueNotEq.canSkip(statistics2)); Assert.assertTrue(valueNotEq.allSatisfy(statistics1)); Assert.assertFalse(valueNotEq.allSatisfy(statistics2)); Assert.assertTrue(valueNotEq.allSatisfy(statistics3)); @@ -89,14 +89,14 @@ public class StatisticsFilterTest { @Test public void testGt() { Filter timeGt = TimeFilter.gt(10L); - Assert.assertTrue(timeGt.satisfy(statistics1)); - Assert.assertTrue(timeGt.satisfy(statistics2)); + Assert.assertFalse(timeGt.canSkip(statistics1)); + Assert.assertFalse(timeGt.canSkip(statistics2)); Assert.assertFalse(timeGt.allSatisfy(statistics1)); Assert.assertTrue(timeGt.allSatisfy(statistics2)); Filter valueGt = ValueFilter.gt(100L); - Assert.assertFalse(valueGt.satisfy(statistics1)); - Assert.assertTrue(valueGt.satisfy(statistics2)); + Assert.assertTrue(valueGt.canSkip(statistics1)); + Assert.assertFalse(valueGt.canSkip(statistics2)); Assert.assertFalse(valueGt.allSatisfy(statistics1)); Assert.assertTrue(valueGt.allSatisfy(statistics2)); } @@ -104,14 +104,14 @@ public class StatisticsFilterTest { @Test public void testGtEq() { Filter timeGtEq = TimeFilter.gtEq(10L); - Assert.assertTrue(timeGtEq.satisfy(statistics1)); - Assert.assertTrue(timeGtEq.satisfy(statistics2)); + Assert.assertFalse(timeGtEq.canSkip(statistics1)); + Assert.assertFalse(timeGtEq.canSkip(statistics2)); Assert.assertFalse(timeGtEq.allSatisfy(statistics1)); Assert.assertTrue(timeGtEq.allSatisfy(statistics2)); Filter valueGtEq = ValueFilter.gtEq(100L); - Assert.assertTrue(valueGtEq.satisfy(statistics1)); - Assert.assertTrue(valueGtEq.satisfy(statistics2)); + Assert.assertFalse(valueGtEq.canSkip(statistics1)); + Assert.assertFalse(valueGtEq.canSkip(statistics2)); Assert.assertFalse(valueGtEq.allSatisfy(statistics1)); Assert.assertTrue(valueGtEq.allSatisfy(statistics2)); } @@ -119,14 +119,14 @@ public class StatisticsFilterTest { @Test public void testLt() { Filter timeLt = TimeFilter.lt(101L); - Assert.assertTrue(timeLt.satisfy(statistics1)); - Assert.assertFalse(timeLt.satisfy(statistics2)); + Assert.assertFalse(timeLt.canSkip(statistics1)); + Assert.assertTrue(timeLt.canSkip(statistics2)); Assert.assertTrue(timeLt.allSatisfy(statistics1)); Assert.assertFalse(timeLt.allSatisfy(statistics2)); Filter valueLt = ValueFilter.lt(11L); - Assert.assertTrue(valueLt.satisfy(statistics1)); - Assert.assertFalse(valueLt.satisfy(statistics2)); + Assert.assertFalse(valueLt.canSkip(statistics1)); + Assert.assertTrue(valueLt.canSkip(statistics2)); Assert.assertFalse(valueLt.allSatisfy(statistics1)); Assert.assertFalse(valueLt.allSatisfy(statistics2)); } @@ -134,12 +134,12 @@ public class StatisticsFilterTest { @Test public void testLtEq() { Filter timeLtEq = TimeFilter.ltEq(101L); - Assert.assertTrue(timeLtEq.satisfy(statistics1)); - Assert.assertTrue(timeLtEq.satisfy(statistics2)); + Assert.assertFalse(timeLtEq.canSkip(statistics1)); + Assert.assertFalse(timeLtEq.canSkip(statistics2)); Filter valueLtEq = ValueFilter.ltEq(11L); - Assert.assertTrue(valueLtEq.satisfy(statistics1)); - Assert.assertFalse(valueLtEq.satisfy(statistics2)); + Assert.assertFalse(valueLtEq.canSkip(statistics1)); + Assert.assertTrue(valueLtEq.canSkip(statistics2)); Assert.assertFalse(valueLtEq.allSatisfy(statistics1)); Assert.assertFalse(valueLtEq.allSatisfy(statistics2)); } @@ -147,19 +147,19 @@ public class StatisticsFilterTest { @Test public void testAndOr() { Filter andFilter = FilterFactory.and(TimeFilter.gt(10L), ValueFilter.lt(50L)); - Assert.assertTrue(andFilter.satisfy(statistics1)); - Assert.assertFalse(andFilter.satisfy(statistics2)); + Assert.assertFalse(andFilter.canSkip(statistics1)); + Assert.assertTrue(andFilter.canSkip(statistics2)); Filter orFilter = FilterFactory.or(andFilter, TimeFilter.eq(200L)); - Assert.assertTrue(orFilter.satisfy(statistics1)); - Assert.assertTrue(orFilter.satisfy(statistics2)); + Assert.assertFalse(orFilter.canSkip(statistics1)); + Assert.assertFalse(orFilter.canSkip(statistics2)); } @Test public void testNot() { Filter timeNotEq = FilterFactory.not(TimeFilter.eq(10L)); try { - timeNotEq.satisfy(statistics1); + timeNotEq.canSkip(statistics1); fail(); } catch (Exception e) { Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); @@ -173,7 +173,7 @@ public class StatisticsFilterTest { Filter valueNotEq = FilterFactory.not(ValueFilter.eq(101L)); try { - valueNotEq.satisfy(statistics1); + valueNotEq.canSkip(statistics1); fail(); } catch (Exception e) { Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); @@ -189,17 +189,17 @@ public class StatisticsFilterTest { @Test public void testBetweenAnd() { Filter timeBetweenAnd = TimeFilter.between(0, 20); - Assert.assertTrue(timeBetweenAnd.satisfy(statistics1)); - Assert.assertFalse(timeBetweenAnd.satisfy(statistics2)); - Assert.assertTrue(timeBetweenAnd.satisfy(statistics3)); + Assert.assertFalse(timeBetweenAnd.canSkip(statistics1)); + Assert.assertTrue(timeBetweenAnd.canSkip(statistics2)); + Assert.assertFalse(timeBetweenAnd.canSkip(statistics3)); Assert.assertFalse(timeBetweenAnd.allSatisfy(statistics1)); Assert.assertFalse(timeBetweenAnd.allSatisfy(statistics2)); Assert.assertTrue(timeBetweenAnd.allSatisfy(statistics3)); Filter timeNotBetweenAnd = TimeFilter.notBetween(0, 20); - Assert.assertTrue(timeNotBetweenAnd.satisfy(statistics1)); - Assert.assertTrue(timeNotBetweenAnd.satisfy(statistics2)); - Assert.assertFalse(timeNotBetweenAnd.satisfy(statistics3)); + Assert.assertFalse(timeNotBetweenAnd.canSkip(statistics1)); + Assert.assertFalse(timeNotBetweenAnd.canSkip(statistics2)); + Assert.assertTrue(timeNotBetweenAnd.canSkip(statistics3)); Assert.assertFalse(timeNotBetweenAnd.allSatisfy(statistics1)); Assert.assertTrue(timeNotBetweenAnd.allSatisfy(statistics2)); Assert.assertFalse(timeNotBetweenAnd.allSatisfy(statistics3));
