This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/extendFilter in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 76c4bc69e743df226e37f1306b27655667a66732 Author: Minghui Liu <[email protected]> AuthorDate: Mon Nov 27 20:21:32 2023 +0800 add batch interface (tmp save) --- .../iotdb/tsfile/file/header/ChunkGroupHeader.java | 2 +- .../iotdb/tsfile/file/header/PageHeader.java | 2 +- .../metadata/IAlignedMetadata.java} | 20 ++- .../iotdb/tsfile/read/filter/basic/Filter.java | 5 + .../tsfile/read/filter/basic/ITimeFilter.java | 9 ++ .../tsfile/read/filter/basic/IValueFilter.java | 11 ++ .../iotdb/tsfile/read/filter/operator/And.java | 11 ++ .../iotdb/tsfile/read/filter/operator/Not.java | 7 + .../iotdb/tsfile/read/filter/operator/Or.java | 7 + .../read/filter/operator/ValueFilterOperators.java | 154 ++++++++++++++++----- .../iotdb/tsfile/read/filter/OperatorTest.java | 31 ++--- 11 files changed, 193 insertions(+), 66 deletions(-) diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkGroupHeader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkGroupHeader.java index 406b1dfe9c4..ec289156125 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkGroupHeader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/ChunkGroupHeader.java @@ -36,7 +36,7 @@ public class ChunkGroupHeader { private final String deviceID; // this field does not need to be serialized. - private int serializedSize; + private final int serializedSize; /** * constructor of CHUNK_GROUP_HEADER. diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java index 1863c986b72..73fdfec9237 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/header/PageHeader.java @@ -33,7 +33,7 @@ public class PageHeader { private int uncompressedSize; private int compressedSize; - private Statistics<? extends Serializable> statistics; + private final Statistics<? extends Serializable> statistics; private boolean modified; public PageHeader( diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IAlignedMetadata.java similarity index 62% copy from iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java copy to iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IAlignedMetadata.java index 6f445c8361b..78425a00825 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IAlignedMetadata.java @@ -17,19 +17,27 @@ * under the License. */ -package org.apache.iotdb.tsfile.read.filter.basic; +package org.apache.iotdb.tsfile.file.metadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; +import org.apache.iotdb.tsfile.file.metadata.statistics.TimeStatistics; import java.io.Serializable; -public interface ITimeFilter extends Filter { +public interface IAlignedMetadata { - default boolean canSkip(Statistics<? extends Serializable> statistics) { - return !satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + TimeStatistics getTimeStatistics(); + + Statistics<Serializable> getMeasurementStatistics(String measurement); + + /** @return whether there are any nulls. */ + default boolean hasNullValue(String measurement) { + return getMeasurementStatistics(measurement).hasNullValue(getRowCount()); } - default boolean allSatisfy(Statistics<? extends Serializable> statistics) { - return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + default long getRowCount() { + return getTimeStatistics().getCount(); } + + boolean isAllNulls(String measurement); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java index fe3b2f93329..890527800ee 100755 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.basic; +import org.apache.iotdb.tsfile.file.metadata.IAlignedMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; @@ -73,6 +74,8 @@ public interface Filter { */ boolean canSkip(Statistics<? extends Serializable> statistics); + boolean canSkip(IAlignedMetadata alignedMetadata); + /** * To examine whether all data points are satisfied with the filter. * @@ -81,6 +84,8 @@ public interface Filter { */ boolean allSatisfy(Statistics<? extends Serializable> statistics); + boolean allSatisfy(IAlignedMetadata alignedMetadata); + /** * To examine whether the min time and max time are satisfied with the filter. * diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java index 6f445c8361b..5a387ab3724 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.basic; +import org.apache.iotdb.tsfile.file.metadata.IAlignedMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import java.io.Serializable; @@ -29,7 +30,15 @@ public interface ITimeFilter extends Filter { return !satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } + default boolean canSkip(IAlignedMetadata alignedMetadata) { + return canSkip(alignedMetadata.getTimeStatistics()); + } + default boolean allSatisfy(Statistics<? extends Serializable> statistics) { return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } + + default boolean allSatisfy(IAlignedMetadata alignedMetadata) { + return allSatisfy(alignedMetadata.getTimeStatistics()); + } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java index 59e0f9f1302..3c306149fb9 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java @@ -19,12 +19,23 @@ package org.apache.iotdb.tsfile.read.filter.basic; +import org.apache.iotdb.tsfile.file.metadata.IAlignedMetadata; import org.apache.iotdb.tsfile.read.common.TimeRange; import java.util.List; public interface IValueFilter extends Filter { + String getMeasurement(); + + default boolean canSkip(IAlignedMetadata alignedMetadata) { + return canSkip(alignedMetadata.getMeasurementStatistics(getMeasurement())); + } + + default boolean allSatisfy(IAlignedMetadata alignedMetadata) { + return allSatisfy(alignedMetadata.getMeasurementStatistics(getMeasurement())); + } + default boolean satisfyStartEndTime(long startTime, long endTime) { return true; } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java index a6474aa1f4f..6a6480c882b 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.operator; +import org.apache.iotdb.tsfile.file.metadata.IAlignedMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -52,11 +53,21 @@ public class And extends BinaryLogicalFilter { return left.canSkip(statistics) || right.canSkip(statistics); } + @Override + public boolean canSkip(IAlignedMetadata alignedMetadata) { + return left.canSkip(alignedMetadata) || right.canSkip(alignedMetadata); + } + @Override public boolean allSatisfy(Statistics<? extends Serializable> statistics) { return left.allSatisfy(statistics) && right.allSatisfy(statistics); } + @Override + public boolean allSatisfy(IAlignedMetadata alignedMetadata) { + return left.allSatisfy(alignedMetadata) && right.allSatisfy(alignedMetadata); + } + @Override public boolean satisfyStartEndTime(long startTime, long endTime) { return left.satisfyStartEndTime(startTime, endTime) diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java index a8894eebd4f..90ea0a53edf 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java @@ -31,6 +31,7 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; public class Not implements Filter { @@ -59,6 +60,12 @@ public class Not implements Filter { throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } + @Override + public boolean canSkip( + Map<String, Statistics<? extends Serializable>> measurementToStatisticsMap) { + throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); + } + @Override public boolean allSatisfy(Statistics<? extends Serializable> statistics) { throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java index 3d5777c2e22..20d6f9f38e6 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java @@ -29,6 +29,7 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class Or extends BinaryLogicalFilter { @@ -52,6 +53,12 @@ public class Or extends BinaryLogicalFilter { return left.canSkip(statistics) && right.canSkip(statistics); } + @Override + public boolean canSkip( + Map<String, Statistics<? extends Serializable>> measurementToStatisticsMap) { + return left.canSkip(measurementToStatisticsMap) && right.canSkip(measurementToStatisticsMap); + } + @Override public boolean allSatisfy(Statistics<? extends Serializable> statistics) { return left.allSatisfy(statistics) || right.allSatisfy(statistics); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java index 26c5b22232e..e186c9d0988 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java @@ -20,6 +20,7 @@ package org.apache.iotdb.tsfile.read.filter.operator; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.IAlignedMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -54,6 +55,9 @@ public final class ValueFilterOperators { private static final String MEASUREMENT_CANNOT_BE_NULL_MSG = "measurement cannot be null"; private static final String CONSTANT_CANNOT_BE_NULL_MSG = "constant cannot be null"; + private static final boolean BLOCK_MIGHT_MATCH = false; + private static final boolean BLOCK_CANNOT_MATCH = true; + // base class for ValueEq, ValueNotEq, ValueLt, ValueGt, ValueLtEq, ValueGtEq abstract static class ValueColumnCompareFilter<T extends Comparable<T>> extends ColumnCompareFilter<T> implements IValueFilter { @@ -65,6 +69,7 @@ public final class ValueFilterOperators { this.measurement = Objects.requireNonNull(measurement, MEASUREMENT_CANNOT_BE_NULL_MSG); } + @Override public String getMeasurement() { return measurement; } @@ -97,7 +102,6 @@ public final class ValueFilterOperators { public static final class ValueEq<T extends Comparable<T>> extends ValueColumnCompareFilter<T> { // constant can be null - // TODO: consider support IS NULL public ValueEq(String measurement, T constant) { super(measurement, constant); } @@ -109,14 +113,44 @@ public final class ValueFilterOperators { @Override public boolean satisfy(long time, Object value) { - return constant.equals(value); + return Objects.equals(value, constant); } @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; + } + + // drop if value < min || value > max + return constant.compareTo((T) statistics.getMinValue()) < 0 + || constant.compareTo((T) statistics.getMaxValue()) > 0; + } + + @Override + @SuppressWarnings("unchecked") + public boolean canSkip(IAlignedMetadata alignedMetadata) { + Statistics<? extends Serializable> statistics = + alignedMetadata.getMeasurementStatistics(measurement); + + if (statistics == null) { + // the measurement isn't in this block so all values are null. + if (constant != null) { + // non-null is never null + return BLOCK_CANNOT_MATCH; + } + return BLOCK_MIGHT_MATCH; + } + + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; + } + + if (constant == null) { + // we are looking for records where v eq(null) + // so drop if there are no nulls in this chunk + return !alignedMetadata.hasNullValue(measurement); } // drop if value < min || value > max @@ -127,9 +161,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return constant.compareTo((T) statistics.getMinValue()) == 0 && constant.compareTo((T) statistics.getMaxValue()) == 0; } @@ -154,7 +189,6 @@ public final class ValueFilterOperators { extends ValueColumnCompareFilter<T> { // constant can be null - // TODO: consider support IS NOT NULL public ValueNotEq(String measurement, T constant) { super(measurement, constant); } @@ -166,14 +200,43 @@ public final class ValueFilterOperators { @Override public boolean satisfy(long time, Object value) { - return !constant.equals(value); + return !Objects.equals(value, constant); } @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; + } + + // drop if this is a column where min = max = value + return constant.compareTo((T) statistics.getMinValue()) == 0 + && constant.compareTo((T) statistics.getMaxValue()) == 0; + } + + @Override + @SuppressWarnings("unchecked") + public boolean canSkip(IAlignedMetadata alignedMetadata) { + Statistics<? extends Serializable> statistics = + alignedMetadata.getMeasurementStatistics(measurement); + + if (statistics == null) { + if (constant == null) { + // null is always equal to null + return BLOCK_CANNOT_MATCH; + } + return BLOCK_MIGHT_MATCH; + } + + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; + } + + if (constant == null) { + // we are looking for records where v notEq(null) + // so, if this is a column of all nulls, we can drop it + return alignedMetadata.isAllNulls(measurement); } // drop if this is a column where min = max = value @@ -184,9 +247,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return constant.compareTo((T) statistics.getMinValue()) < 0 || constant.compareTo((T) statistics.getMaxValue()) > 0; } @@ -228,8 +292,8 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } // drop if value <= min @@ -239,9 +303,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return constant.compareTo((T) statistics.getMaxValue()) > 0; } @@ -282,8 +347,8 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } // drop if value < min @@ -293,9 +358,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return constant.compareTo((T) statistics.getMaxValue()) >= 0; } @@ -336,8 +402,8 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } // drop if value >= max @@ -347,9 +413,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return constant.compareTo((T) statistics.getMinValue()) < 0; } @@ -390,8 +457,8 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } // drop if value > max @@ -401,9 +468,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return constant.compareTo((T) statistics.getMinValue()) <= 0; } @@ -434,6 +502,7 @@ public final class ValueFilterOperators { this.measurement = Objects.requireNonNull(measurement, MEASUREMENT_CANNOT_BE_NULL_MSG); } + @Override public String getMeasurement() { return measurement; } @@ -491,9 +560,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return ((T) statistics.getMaxValue()).compareTo(min) >= 0 && ((T) statistics.getMinValue()).compareTo(max) <= 0; } @@ -501,9 +571,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return ((T) statistics.getMinValue()).compareTo(min) >= 0 && ((T) statistics.getMaxValue()).compareTo(max) <= 0; } @@ -548,8 +619,8 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean canSkip(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } return ((T) statistics.getMinValue()).compareTo(min) >= 0 @@ -559,9 +630,10 @@ public final class ValueFilterOperators { @Override @SuppressWarnings("unchecked") public boolean allSatisfy(Statistics<? extends Serializable> statistics) { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return false; + if (statisticsNotAvailable(statistics)) { + return BLOCK_MIGHT_MATCH; } + return ((T) statistics.getMinValue()).compareTo(max) > 0 || ((T) statistics.getMaxValue()).compareTo(min) < 0; } @@ -582,6 +654,14 @@ public final class ValueFilterOperators { } } + // we have no statistics available, we cannot drop any blocks + private static boolean statisticsNotAvailable(Statistics<?> statistics) { + return statistics == null + || statistics.getType() == TSDataType.TEXT + || statistics.getType() == TSDataType.BOOLEAN + || statistics.isEmpty(); + } + // base class for ValueIn, ValueNotIn abstract static class ValueColumnSetFilter<T> extends ColumnSetFilter<T> implements IDisableStatisticsValueFilter { @@ -593,6 +673,7 @@ public final class ValueFilterOperators { this.measurement = Objects.requireNonNull(measurement, MEASUREMENT_CANNOT_BE_NULL_MSG); } + @Override public String getMeasurement() { return measurement; } @@ -698,6 +779,7 @@ public final class ValueFilterOperators { this.measurement = Objects.requireNonNull(measurement, MEASUREMENT_CANNOT_BE_NULL_MSG); } + @Override public String getMeasurement() { return measurement; } diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java index b8603231f26..a3b92690fc2 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java @@ -30,7 +30,6 @@ import org.junit.Test; public class OperatorTest { - private static final long EFFICIENCY_TEST_COUNT = 10000000; private static final long TESTED_TIMESTAMP = 1513585371L; @Test @@ -46,6 +45,10 @@ public class OperatorTest { Filter filter3 = ValueFilter.eq(true); Assert.assertTrue(filter3.satisfy(100, true)); Assert.assertFalse(filter3.satisfy(100, false)); + + Filter isNullFilter = ValueFilter.eq(null); + Assert.assertTrue(isNullFilter.satisfy(100, null)); + Assert.assertFalse(isNullFilter.satisfy(100, 1)); } @Test @@ -128,6 +131,10 @@ public class OperatorTest { Filter valueNotEq = ValueFilter.notEq(50); Assert.assertFalse(valueNotEq.satisfy(100, 50)); Assert.assertTrue(valueNotEq.satisfy(100, 51)); + + Filter isNotNullFilter = ValueFilter.notEq(null); + Assert.assertFalse(isNotNullFilter.satisfy(100, null)); + Assert.assertTrue(isNotNullFilter.satisfy(100, 1)); } @Test @@ -154,28 +161,8 @@ public class OperatorTest { try { andFilter.satisfy(101L, 50); Assert.fail(); - } catch (ClassCastException e) { - - } - } - - @Test - public void efficiencyTest() { - Filter andFilter = FilterFactory.and(TimeFilter.gt(100L), ValueFilter.lt(50.9)); - Filter orFilter = FilterFactory.or(andFilter, TimeFilter.eq(1000L)); + } catch (ClassCastException ignored) { - long startTime = System.currentTimeMillis(); - for (long i = 0; i < EFFICIENCY_TEST_COUNT; i++) { - orFilter.satisfy(i, i + 0.1); } - long endTime = System.currentTimeMillis(); - System.out.println( - "EfficiencyTest for Filter: \n\tFilter Expression = " - + orFilter - + "\n\tCOUNT = " - + EFFICIENCY_TEST_COUNT - + "\n\tTotal Time = " - + (endTime - startTime) - + "ms."); } }
