This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/FixOffsetPushDown1.1 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1e7ef5a7beba84d531c929db32400add8b3029e2 Author: liuminghui233 <[email protected]> AuthorDate: Sat Apr 22 21:31:05 2023 +0800 [IOTDB-5784] Incorrect result when querying with offset push-down and time filter (cherry picked from commit 9b194dcac6fbe8315aa5ad174a7f310e67c21731) --- .../iotdb/db/it/query/IoTDBPaginationIT.java | 59 ++++++- .../execution/operator/source/SeriesScanUtil.java | 24 ++- .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 4 + .../db/mpp/plan/analyze/ExpressionAnalyzer.java | 11 +- .../iotdb/db/mpp/plan/analyze/ExpressionUtils.java | 3 +- .../db/mpp/plan/planner/LogicalPlanBuilder.java | 8 + .../db/mpp/plan/planner/OperatorTreeGenerator.java | 12 +- .../source/AlignedSeriesAggregationScanNode.java | 50 +++--- .../plan/node/source/AlignedSeriesScanNode.java | 4 + .../node/source/SeriesAggregationScanNode.java | 50 +++--- .../node/source/SeriesAggregationSourceNode.java | 49 +++++- .../planner/plan/node/source/SeriesScanNode.java | 4 + .../planner/plan/parameter/SeriesScanOptions.java | 5 +- .../iotdb/db/query/reader/chunk/MemPageReader.java | 12 +- .../mpp/plan/analyze/QueryTimePartitionTest.java | 14 +- .../node/process/AggregationNodeSerdeTest.java | 7 +- .../node/process/GroupByLevelNodeSerdeTest.java | 8 +- .../plan/node/process/GroupByTagNodeSerdeTest.java | 5 +- .../source/SeriesAggregationScanNodeSerdeTest.java | 7 +- .../iotdb/tsfile/read/filter/GroupByFilter.java | 13 +- .../read/filter/PredicateRemoveNotRewriter.java | 50 ++++++ .../iotdb/tsfile/read/filter/TimeFilter.java | 158 +++++++++--------- .../iotdb/tsfile/read/filter/ValueFilter.java | 179 ++++++++++++--------- .../tsfile/read/filter/basic/BinaryFilter.java | 5 - .../iotdb/tsfile/read/filter/basic/Filter.java | 11 +- .../tsfile/read/filter/factory/FilterFactory.java | 10 +- .../tsfile/read/filter/operator/AndFilter.java | 11 ++ .../iotdb/tsfile/read/filter/operator/Between.java | 127 +++++++++------ .../iotdb/tsfile/read/filter/operator/Eq.java | 25 ++- .../iotdb/tsfile/read/filter/operator/Gt.java | 23 ++- .../iotdb/tsfile/read/filter/operator/GtEq.java | 23 ++- .../iotdb/tsfile/read/filter/operator/In.java | 34 +++- .../iotdb/tsfile/read/filter/operator/Like.java | 55 +++++-- .../iotdb/tsfile/read/filter/operator/Lt.java | 23 ++- .../iotdb/tsfile/read/filter/operator/LtEq.java | 23 ++- .../iotdb/tsfile/read/filter/operator/NotEq.java | 25 ++- .../tsfile/read/filter/operator/NotFilter.java | 26 +-- .../tsfile/read/filter/operator/OrFilter.java | 11 ++ .../iotdb/tsfile/read/filter/operator/Regexp.java | 57 +++++-- .../iotdb/tsfile/read/reader/page/PageReader.java | 12 +- .../tsfile/read/filter/FilterSerializeTest.java | 21 ++- .../read/filter/MinTimeMaxTimeFilterTest.java | 42 +++-- .../iotdb/tsfile/read/filter/OperatorTest.java | 4 +- .../filter/PredicateRemoveNotRewriterTest.java | 121 ++++++++++++++ .../tsfile/read/filter/StatisticsFilterTest.java | 113 ++++++++++++- 45 files changed, 1109 insertions(+), 429 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java index 0e1fb007f5..f9d9a70a8a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/query/IoTDBPaginationIT.java @@ -76,11 +76,36 @@ public class IoTDBPaginationIT { "insert into root.vehicle.d0(timestamp,s2) values(102,10.00)", "insert into root.vehicle.d0(timestamp,s2) values(105,11.11)", "insert into root.vehicle.d0(timestamp,s2) values(1000,1000.11)", - "insert into root.vehicle.d0(timestamp,s1) values(2000-01-01T08:00:00+08:00, 100)" + "insert into root.vehicle.d0(timestamp,s1) values(2000-01-01T08:00:00+08:00, 100)", + "CREATE DATABASE root.db", + "CREATE TIMESERIES root.db.d1.s1 INT32", + "insert into root.db.d1(timestamp,s1) values(0, 0)", + "insert into root.db.d1(timestamp,s1) values(1, 1)", + "insert into root.db.d1(timestamp,s1) values(2, 2)", + "insert into root.db.d1(timestamp,s1) values(3, 3)", + "insert into root.db.d1(timestamp,s1) values(4, 4)", + "flush", + "insert into root.db.d1(timestamp,s1) values(5, 5)", + "insert into root.db.d1(timestamp,s1) values(6, 6)", + "insert into root.db.d1(timestamp,s1) values(7, 7)", + "insert into root.db.d1(timestamp,s1) values(8, 8)", + "insert into root.db.d1(timestamp,s1) values(9, 9)", + "flush", + "insert into root.db.d1(timestamp,s1) values(10, 10)", + "insert into root.db.d1(timestamp,s1) values(11, 11)", + "insert into root.db.d1(timestamp,s1) values(12, 12)", + "insert into root.db.d1(timestamp,s1) values(13, 13)", + "insert into root.db.d1(timestamp,s1) values(14, 14)" }; @BeforeClass public static void setUp() throws InterruptedException { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false); EnvFactory.getEnv().initClusterEnvironment(); prepareData(SQLs); } @@ -135,4 +160,36 @@ public class IoTDBPaginationIT { resultSetEqualTest(querySQLs.get(0), expectHeaders.get(0), retArrays.get(0)); } } + + @Test + public void limitOffsetPushDownTest() { + String expectedHeader = "Time,root.db.d1.s1,"; + String[] retArray = + new String[] { + "3,3,", + }; + resultSetEqualTest( + "select s1 from root.db.d1 where time > 1 offset 1 limit 1", expectedHeader, retArray); + + retArray = + new String[] { + "5,5,", + }; + resultSetEqualTest( + "select s1 from root.db.d1 where time > 1 offset 3 limit 1", expectedHeader, retArray); + + retArray = + new String[] { + "7,7,", + }; + resultSetEqualTest( + "select s1 from root.db.d1 where time > 1 offset 5 limit 1", expectedHeader, retArray); + + retArray = + new String[] { + "12,12,", + }; + resultSetEqualTest( + "select s1 from root.db.d1 where time > 1 offset 10 limit 1", expectedHeader, retArray); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java index d42c2ff931..d3988be366 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/source/SeriesScanUtil.java @@ -288,17 +288,15 @@ public class SeriesScanUtil { protected void filterFirstChunkMetadata() throws IOException { if (firstChunkMetadata != null && !isChunkOverlapped() && !firstChunkMetadata.isModified()) { Filter queryFilter = scanOptions.getQueryFilter(); - if (queryFilter != null) { - if (!queryFilter.satisfy(firstChunkMetadata.getStatistics())) { - skipCurrentChunk(); - } - // TODO implement allSatisfied interface for filter, then we can still skip offset. - } else { - long rowCount = firstChunkMetadata.getStatistics().getCount(); + Statistics statistics = firstChunkMetadata.getStatistics(); + if (queryFilter == null || queryFilter.allSatisfy(statistics)) { + long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { skipCurrentChunk(); paginationController.consumeOffset(rowCount); } + } else if (!queryFilter.satisfy(statistics)) { + skipCurrentChunk(); } } } @@ -1041,17 +1039,15 @@ public class SeriesScanUtil { && !isFileOverlapped() && !firstTimeSeriesMetadata.isModified()) { Filter queryFilter = scanOptions.getQueryFilter(); - if (queryFilter != null) { - if (!queryFilter.satisfy(firstTimeSeriesMetadata.getStatistics())) { - skipCurrentFile(); - } - // TODO implement allSatisfied interface for filter, then we can still skip offset. - } else { - long rowCount = firstTimeSeriesMetadata.getStatistics().getCount(); + Statistics statistics = firstTimeSeriesMetadata.getStatistics(); + if (queryFilter == null || queryFilter.allSatisfy(statistics)) { + long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { skipCurrentFile(); paginationController.consumeOffset(rowCount); } + } else if (!queryFilter.satisfy(statistics)) { + skipCurrentFile(); } } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index f929573fb1..d2b44cfdd4 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -144,6 +144,7 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.GroupByFilter; import org.apache.iotdb.tsfile.read.filter.GroupByMonthFilter; +import org.apache.iotdb.tsfile.read.filter.PredicateRemoveNotRewriter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; import org.apache.iotdb.tsfile.utils.Pair; @@ -373,6 +374,9 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> Pair<Filter, Boolean> resultPair = ExpressionAnalyzer.extractGlobalTimeFilter(predicate, true, true); globalTimeFilter = resultPair.left; + if (globalTimeFilter != null) { + globalTimeFilter = PredicateRemoveNotRewriter.rewrite(globalTimeFilter); + } hasValueFilter = resultPair.right; predicate = ExpressionAnalyzer.evaluatePredicate(predicate); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java index 915afc8b06..d274ae03ea 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionAnalyzer.java @@ -930,12 +930,11 @@ public class ExpressionAnalyzer { } else if (predicate.getExpressionType().equals(ExpressionType.IN)) { Expression timeExpression = ((InExpression) predicate).getExpression(); if (timeExpression.getExpressionType().equals(ExpressionType.TIMESTAMP)) { - return new Pair<>( - TimeFilter.in( - ((InExpression) predicate) - .getValues().stream().map(Long::parseLong).collect(Collectors.toSet()), - ((InExpression) predicate).isNotIn()), - false); + boolean not = ((InExpression) predicate).isNotIn(); + Set<Long> values = + ((InExpression) predicate) + .getValues().stream().map(Long::parseLong).collect(Collectors.toSet()); + return new Pair<>(not ? TimeFilter.notIn(values) : TimeFilter.in(values), false); } return new Pair<>(null, true); } else if (predicate.getExpressionType().equals(ExpressionType.TIMESERIES) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java index 62df923970..593c3fb278 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/ExpressionUtils.java @@ -365,7 +365,8 @@ public class ExpressionUtils { && ((ConstantOperand) secondExpression).getDataType() == TSDataType.INT64) { long value1 = Long.parseLong(((ConstantOperand) firstExpression).getValueString()); long value2 = Long.parseLong(((ConstantOperand) secondExpression).getValueString()); - return new Pair<>(TimeFilter.between(value1, value2, not), false); + return new Pair<>( + not ? TimeFilter.notBetween(value1, value2) : TimeFilter.between(value1, value2), false); } else { return new Pair<>(null, true); } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java index 84d121be41..e36e29aece 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanBuilder.java @@ -177,12 +177,16 @@ public class LogicalPlanBuilder { new SeriesScanNode( context.getQueryId().genPlanNodeId(), (MeasurementPath) path, scanOrder); seriesScanNode.setTimeFilter(timeFilter); + // TODO: push down value filter + seriesScanNode.setValueFilter(timeFilter); sourceNodeList.add(seriesScanNode); } else if (path instanceof AlignedPath) { // aligned series AlignedSeriesScanNode alignedSeriesScanNode = new AlignedSeriesScanNode( context.getQueryId().genPlanNodeId(), (AlignedPath) path, scanOrder); alignedSeriesScanNode.setTimeFilter(timeFilter); + // TODO: push down value filter + alignedSeriesScanNode.setValueFilter(timeFilter); sourceNodeList.add(alignedSeriesScanNode); } else { throw new IllegalArgumentException("unexpected path type"); @@ -790,6 +794,8 @@ public class LogicalPlanBuilder { scanOrder, groupByTimeParameter); seriesAggregationScanNode.setTimeFilter(timeFilter); + // TODO: push down value filter + seriesAggregationScanNode.setValueFilter(timeFilter); return seriesAggregationScanNode; } else if (selectPath instanceof AlignedPath) { // aligned series AlignedSeriesAggregationScanNode alignedSeriesAggregationScanNode = @@ -800,6 +806,8 @@ public class LogicalPlanBuilder { scanOrder, groupByTimeParameter); alignedSeriesAggregationScanNode.setTimeFilter(timeFilter); + // TODO: push down value filter + alignedSeriesAggregationScanNode.setValueFilter(timeFilter); return alignedSeriesAggregationScanNode; } else { throw new IllegalArgumentException("unexpected path type"); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java index 2742b4bce2..cd5ea260f4 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/OperatorTreeGenerator.java @@ -285,7 +285,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP seriesScanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy()); } if (valueFilter != null) { - seriesScanOptionsBuilder.withGlobalTimeFilter(valueFilter.copy()); + seriesScanOptionsBuilder.withQueryFilter(valueFilter.copy()); } seriesScanOptionsBuilder.withAllSensors( context.getAllSensors(seriesPath.getDevice(), seriesPath.getMeasurement())); @@ -327,7 +327,7 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP seriesScanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy()); } if (valueFilter != null) { - seriesScanOptionsBuilder.withGlobalTimeFilter(valueFilter.copy()); + seriesScanOptionsBuilder.withQueryFilter(valueFilter.copy()); } seriesScanOptionsBuilder.withLimit(node.getLimit()); seriesScanOptionsBuilder.withOffset(node.getOffset()); @@ -385,12 +385,16 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP node.getAggregationDescriptorList(), timeRangeIterator, context.getTypeProvider()); Filter timeFilter = node.getTimeFilter(); + Filter valueFilter = node.getValueFilter(); SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); scanOptionsBuilder.withAllSensors( context.getAllSensors(seriesPath.getDevice(), seriesPath.getMeasurement())); if (timeFilter != null) { scanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy()); } + if (valueFilter != null) { + scanOptionsBuilder.withQueryFilter(valueFilter.copy()); + } SeriesAggregationScanOperator aggregateScanOperator = new SeriesAggregationScanOperator( @@ -458,11 +462,15 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP node.getAggregationDescriptorList(), timeRangeIterator, context.getTypeProvider()); Filter timeFilter = node.getTimeFilter(); + Filter valueFilter = node.getValueFilter(); SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); scanOptionsBuilder.withAllSensors(new HashSet<>(seriesPath.getMeasurementList())); if (timeFilter != null) { scanOptionsBuilder.withGlobalTimeFilter(timeFilter.copy()); } + if (valueFilter != null) { + scanOptionsBuilder.withQueryFilter(valueFilter.copy()); + } AlignedSeriesAggregationScanOperator seriesAggregationScanOperator = new AlignedSeriesAggregationScanOperator( diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java index 319390f6c0..19dc93a5e8 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesAggregationScanNode.java @@ -83,10 +83,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod List<AggregationDescriptor> aggregationDescriptorList, Ordering scanOrder, @Nullable Filter timeFilter, + @Nullable Filter valueFilter, @Nullable GroupByTimeParameter groupByTimeParameter, TRegionReplicaSet dataRegionReplicaSet) { this(id, alignedPath, aggregationDescriptorList, scanOrder, groupByTimeParameter); this.timeFilter = timeFilter; + this.valueFilter = valueFilter; this.regionReplicaSet = dataRegionReplicaSet; } @@ -99,22 +101,6 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod return scanOrder; } - @Override - @Nullable - public Filter getTimeFilter() { - return timeFilter; - } - - public void setTimeFilter(@Nullable Filter timeFilter) { - this.timeFilter = timeFilter; - } - - @Override - @Nullable - public GroupByTimeParameter getGroupByTimeParameter() { - return groupByTimeParameter; - } - @Override public void open() throws Exception {} @@ -155,6 +141,7 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod getAggregationDescriptorList(), getScanOrder(), getTimeFilter(), + getValueFilter(), getGroupByTimeParameter(), getRegionReplicaSet()); } @@ -187,6 +174,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod ReadWriteIOUtils.write((byte) 1, byteBuffer); timeFilter.serialize(byteBuffer); } + if (valueFilter == null) { + ReadWriteIOUtils.write((byte) 0, byteBuffer); + } else { + ReadWriteIOUtils.write((byte) 1, byteBuffer); + valueFilter.serialize(byteBuffer); + } if (groupByTimeParameter == null) { ReadWriteIOUtils.write((byte) 0, byteBuffer); } else { @@ -210,6 +203,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod ReadWriteIOUtils.write((byte) 1, stream); timeFilter.serialize(stream); } + if (valueFilter == null) { + ReadWriteIOUtils.write((byte) 0, stream); + } else { + ReadWriteIOUtils.write((byte) 1, stream); + valueFilter.serialize(stream); + } if (groupByTimeParameter == null) { ReadWriteIOUtils.write((byte) 0, stream); } else { @@ -232,6 +231,11 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod timeFilter = FilterFactory.deserialize(byteBuffer); } isNull = ReadWriteIOUtils.readByte(byteBuffer); + Filter valueFilter = null; + if (isNull == 1) { + valueFilter = FilterFactory.deserialize(byteBuffer); + } + isNull = ReadWriteIOUtils.readByte(byteBuffer); GroupByTimeParameter groupByTimeParameter = null; if (isNull == 1) { groupByTimeParameter = GroupByTimeParameter.deserialize(byteBuffer); @@ -243,6 +247,7 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod aggregationDescriptorList, scanOrder, timeFilter, + valueFilter, groupByTimeParameter, null); } @@ -260,23 +265,12 @@ public class AlignedSeriesAggregationScanNode extends SeriesAggregationSourceNod } AlignedSeriesAggregationScanNode that = (AlignedSeriesAggregationScanNode) o; return alignedPath.equals(that.alignedPath) - && aggregationDescriptorList.equals(that.aggregationDescriptorList) - && scanOrder == that.scanOrder - && Objects.equals(timeFilter, that.timeFilter) - && Objects.equals(groupByTimeParameter, that.groupByTimeParameter) && Objects.equals(regionReplicaSet, that.regionReplicaSet); } @Override public int hashCode() { - return Objects.hash( - super.hashCode(), - alignedPath, - aggregationDescriptorList, - scanOrder, - timeFilter, - groupByTimeParameter, - regionReplicaSet); + return Objects.hash(super.hashCode(), alignedPath, regionReplicaSet); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java index 09387d0e5b..6629a287a1 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/AlignedSeriesScanNode.java @@ -119,6 +119,10 @@ public class AlignedSeriesScanNode extends SeriesSourceNode { return valueFilter; } + public void setValueFilter(@Nullable Filter valueFilter) { + this.valueFilter = valueFilter; + } + public long getLimit() { return limit; } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java index 2ea730b747..c5b6ac8727 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationScanNode.java @@ -95,10 +95,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { List<AggregationDescriptor> aggregationDescriptorList, Ordering scanOrder, @Nullable Filter timeFilter, + @Nullable Filter valueFilter, @Nullable GroupByTimeParameter groupByTimeParameter, TRegionReplicaSet dataRegionReplicaSet) { this(id, seriesPath, aggregationDescriptorList, scanOrder, groupByTimeParameter); this.timeFilter = timeFilter; + this.valueFilter = valueFilter; this.regionReplicaSet = dataRegionReplicaSet; } @@ -107,22 +109,6 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { return scanOrder; } - @Override - @Nullable - public Filter getTimeFilter() { - return timeFilter; - } - - public void setTimeFilter(@Nullable Filter timeFilter) { - this.timeFilter = timeFilter; - } - - @Override - @Nullable - public GroupByTimeParameter getGroupByTimeParameter() { - return groupByTimeParameter; - } - public MeasurementPath getSeriesPath() { return seriesPath; } @@ -150,6 +136,7 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { getAggregationDescriptorList(), getScanOrder(), getTimeFilter(), + getValueFilter(), getGroupByTimeParameter(), getRegionReplicaSet()); } @@ -198,6 +185,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { ReadWriteIOUtils.write((byte) 1, byteBuffer); timeFilter.serialize(byteBuffer); } + if (valueFilter == null) { + ReadWriteIOUtils.write((byte) 0, byteBuffer); + } else { + ReadWriteIOUtils.write((byte) 1, byteBuffer); + valueFilter.serialize(byteBuffer); + } if (groupByTimeParameter == null) { ReadWriteIOUtils.write((byte) 0, byteBuffer); } else { @@ -221,6 +214,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { ReadWriteIOUtils.write((byte) 1, stream); timeFilter.serialize(stream); } + if (valueFilter == null) { + ReadWriteIOUtils.write((byte) 0, stream); + } else { + ReadWriteIOUtils.write((byte) 1, stream); + valueFilter.serialize(stream); + } if (groupByTimeParameter == null) { ReadWriteIOUtils.write((byte) 0, stream); } else { @@ -243,6 +242,11 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { timeFilter = FilterFactory.deserialize(byteBuffer); } isNull = ReadWriteIOUtils.readByte(byteBuffer); + Filter valueFilter = null; + if (isNull == 1) { + valueFilter = FilterFactory.deserialize(byteBuffer); + } + isNull = ReadWriteIOUtils.readByte(byteBuffer); GroupByTimeParameter groupByTimeParameter = null; if (isNull == 1) { groupByTimeParameter = GroupByTimeParameter.deserialize(byteBuffer); @@ -254,6 +258,7 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { aggregationDescriptorList, scanOrder, timeFilter, + valueFilter, groupByTimeParameter, null); } @@ -271,23 +276,12 @@ public class SeriesAggregationScanNode extends SeriesAggregationSourceNode { } SeriesAggregationScanNode that = (SeriesAggregationScanNode) o; return seriesPath.equals(that.seriesPath) - && aggregationDescriptorList.equals(that.aggregationDescriptorList) - && scanOrder == that.scanOrder - && Objects.equals(timeFilter, that.timeFilter) - && Objects.equals(groupByTimeParameter, that.groupByTimeParameter) && Objects.equals(regionReplicaSet, that.regionReplicaSet); } @Override public int hashCode() { - return Objects.hash( - super.hashCode(), - seriesPath, - aggregationDescriptorList, - scanOrder, - timeFilter, - groupByTimeParameter, - regionReplicaSet); + return Objects.hash(super.hashCode(), seriesPath, regionReplicaSet); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java index 183c44cdc9..4857292fde 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesAggregationSourceNode.java @@ -28,6 +28,7 @@ import org.apache.iotdb.tsfile.read.filter.basic.Filter; import javax.annotation.Nullable; import java.util.List; +import java.util.Objects; public abstract class SeriesAggregationSourceNode extends SeriesSourceNode { @@ -40,9 +41,12 @@ public abstract class SeriesAggregationSourceNode extends SeriesSourceNode { // The default order is TIMESTAMP_ASC, which means "order by timestamp asc" protected Ordering scanOrder = Ordering.ASC; - // time filter for current series, could be null if doesn't exist + // time filter for current series, could be null if it doesn't exist @Nullable protected Filter timeFilter; + // push-downing query filter for current series, could be null if it doesn't exist + @Nullable protected Filter valueFilter; + // The parameter of `group by time` // Its value will be null if there is no `group by time` clause, @Nullable protected GroupByTimeParameter groupByTimeParameter; @@ -70,8 +74,51 @@ public abstract class SeriesAggregationSourceNode extends SeriesSourceNode { return timeFilter; } + public void setTimeFilter(@Nullable Filter timeFilter) { + this.timeFilter = timeFilter; + } + + @Nullable + public Filter getValueFilter() { + return valueFilter; + } + + public void setValueFilter(@Nullable Filter valueFilter) { + this.valueFilter = valueFilter; + } + @Nullable public GroupByTimeParameter getGroupByTimeParameter() { return groupByTimeParameter; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + SeriesAggregationSourceNode that = (SeriesAggregationSourceNode) o; + return aggregationDescriptorList.equals(that.aggregationDescriptorList) + && scanOrder == that.scanOrder + && Objects.equals(timeFilter, that.timeFilter) + && Objects.equals(valueFilter, that.valueFilter) + && Objects.equals(groupByTimeParameter, that.groupByTimeParameter); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + aggregationDescriptorList, + scanOrder, + timeFilter, + valueFilter, + groupByTimeParameter); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java index 6458d5af0b..0572544430 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/source/SeriesScanNode.java @@ -159,6 +159,10 @@ public class SeriesScanNode extends SeriesSourceNode { return valueFilter; } + public void setValueFilter(@Nullable Filter valueFilter) { + this.valueFilter = valueFilter; + } + @Override public List<PlanNode> getChildren() { return ImmutableList.of(); diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java index 7e3bcd6f29..c5032a0786 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/parameter/SeriesScanOptions.java @@ -29,7 +29,6 @@ import org.apache.iotdb.tsfile.read.reader.series.PaginationController; import java.util.Collections; import java.util.HashSet; -import java.util.Objects; import java.util.Set; public class SeriesScanOptions { @@ -50,9 +49,7 @@ public class SeriesScanOptions { long offset, Set<String> allSensors) { this.globalTimeFilter = globalTimeFilter; - if (!Objects.equals(globalTimeFilter, queryFilter)) { - this.queryFilter = queryFilter; - } + this.queryFilter = queryFilter; this.limit = limit; this.offset = offset; this.allSensors = allSensors; diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java index ad6916103b..4e4c4916ed 100644 --- a/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java +++ b/server/src/main/java/org/apache/iotdb/db/query/reader/chunk/MemPageReader.java @@ -94,16 +94,18 @@ public class MemPageReader implements IPageReader { } private boolean pageSatisfy() { - if (valueFilter != null) { - return valueFilter.satisfy(getStatistics()); - } else { - long rowCount = getStatistics().getCount(); + Statistics statistics = getStatistics(); + if (valueFilter == null || valueFilter.allSatisfy(statistics)) { + long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { paginationController.consumeOffset(rowCount); return false; + } else { + return true; } + } else { + return valueFilter.satisfy(statistics); } - return true; } @Override diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java index e795634438..c7951571a0 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/analyze/QueryTimePartitionTest.java @@ -186,14 +186,14 @@ public class QueryTimePartitionTest { public void testBetweenTimeFilter() { // time between 10 and 20 - TimeFilter.TimeBetween filter = TimeFilter.between(10, 20, false); + TimeFilter.TimeBetween filter = TimeFilter.between(10, 20); List<TimeRange> timeRangeList = filter.getTimeRanges(); assertEquals(1, timeRangeList.size()); assertEquals(10, timeRangeList.get(0).getMin()); assertEquals(20, timeRangeList.get(0).getMax()); // time not between 10 and 20 - filter = TimeFilter.between(10, 20, true); + filter = TimeFilter.notBetween(10, 20); timeRangeList = filter.getTimeRanges(); assertEquals(2, timeRangeList.size()); assertEquals(Long.MIN_VALUE, timeRangeList.get(0).getMin()); @@ -202,14 +202,14 @@ public class QueryTimePartitionTest { assertEquals(Long.MAX_VALUE, timeRangeList.get(1).getMax()); // time not between 10 and Long.MAX_VALUE - filter = TimeFilter.between(10, Long.MAX_VALUE, true); + filter = TimeFilter.notBetween(10, Long.MAX_VALUE); timeRangeList = filter.getTimeRanges(); assertEquals(1, timeRangeList.size()); assertEquals(Long.MIN_VALUE, timeRangeList.get(0).getMin()); assertEquals(9, timeRangeList.get(0).getMax()); // time not between Long.MIN_VALUE and 20 - filter = TimeFilter.between(Long.MIN_VALUE, 20, true); + filter = TimeFilter.notBetween(Long.MIN_VALUE, 20); timeRangeList = filter.getTimeRanges(); assertEquals(1, timeRangeList.size()); assertEquals(21, timeRangeList.get(0).getMin()); @@ -441,8 +441,7 @@ public class QueryTimePartitionTest { getTimePartitionSlotList( TimeFilter.between( IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() - 1, - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(), - false)); + IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval())); expected = Arrays.asList( new TTimePartitionSlot(0), @@ -481,8 +480,7 @@ public class QueryTimePartitionTest { getTimePartitionSlotList( TimeFilter.between( IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval(), - IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1, - false)); + IoTDBDescriptor.getInstance().getConfig().getTimePartitionInterval() + 1)); expected = Collections.singletonList( new TTimePartitionSlot( diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java index 5e78bd06d6..abee7fea83 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/AggregationNodeSerdeTest.java @@ -32,7 +32,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep; import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter; import org.apache.iotdb.db.mpp.plan.statement.component.Ordering; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.filter.operator.In; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.ValueFilter; import org.apache.commons.compress.utils.Sets; import org.junit.Test; @@ -42,7 +43,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.apache.iotdb.tsfile.read.filter.factory.FilterType.VALUE_FILTER; import static org.junit.Assert.assertEquals; public class AggregationNodeSerdeTest { @@ -62,7 +62,8 @@ public class AggregationNodeSerdeTest { Collections.singletonList( new TimeSeriesOperand(new PartialPath("root.sg.d1.s1"))))), Ordering.ASC, - new In<>(Sets.newHashSet("s1", "s2"), VALUE_FILTER, true), + TimeFilter.gt(100L), + ValueFilter.in(Sets.newHashSet("s1", "s2")), groupByTimeParameter, null); AggregationNode aggregationNode = diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java index 87af66e497..23346894f3 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByLevelNodeSerdeTest.java @@ -33,6 +33,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.CrossSeriesAggregatio import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter; import org.apache.iotdb.db.mpp.plan.statement.component.Ordering; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.ValueFilter; import org.junit.Test; @@ -62,7 +64,8 @@ public class GroupByLevelNodeSerdeTest { Collections.singletonList( new TimeSeriesOperand(new PartialPath("root.sg.d1.s1"))))), Ordering.ASC, - null, + TimeFilter.gt(100L), + ValueFilter.gt(100), groupByTimeParameter, null); SeriesAggregationScanNode seriesAggregationScanNode2 = @@ -76,7 +79,8 @@ public class GroupByLevelNodeSerdeTest { Collections.singletonList( new TimeSeriesOperand(new PartialPath("root.sg.d2.s1"))))), Ordering.ASC, - null, + TimeFilter.gt(100L), + ValueFilter.gt(100), groupByTimeParameter, null); diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java index b0c0c139e5..5c38ff3aaf 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/GroupByTagNodeSerdeTest.java @@ -34,6 +34,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.CrossSeriesAggregatio import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter; import org.apache.iotdb.db.mpp.plan.statement.component.Ordering; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.ValueFilter; import org.junit.Assert; import org.junit.Test; @@ -102,7 +104,8 @@ public class GroupByTagNodeSerdeTest { new MeasurementPath("root.sg.d1.s1", TSDataType.INT32), Arrays.asList(s1MaxTimePartial, s1AvgTimePartial), Ordering.ASC, - null, + TimeFilter.gt(100L), + ValueFilter.gt(100), groupByTimeParameter, null)), groupByTimeParameter, diff --git a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java index d7f1a614b6..a0ac6ff2c3 100644 --- a/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java +++ b/server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/source/SeriesAggregationScanNodeSerdeTest.java @@ -32,7 +32,8 @@ import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.AggregationStep; import org.apache.iotdb.db.mpp.plan.planner.plan.parameter.GroupByTimeParameter; import org.apache.iotdb.db.mpp.plan.statement.component.Ordering; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.filter.operator.In; +import org.apache.iotdb.tsfile.read.filter.TimeFilter; +import org.apache.iotdb.tsfile.read.filter.ValueFilter; import org.apache.commons.compress.utils.Sets; import org.junit.Test; @@ -42,7 +43,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import static org.apache.iotdb.tsfile.read.filter.factory.FilterType.VALUE_FILTER; import static org.junit.Assert.assertEquals; public class SeriesAggregationScanNodeSerdeTest { @@ -62,7 +62,8 @@ public class SeriesAggregationScanNodeSerdeTest { new MeasurementPath("root.sg.d1.s1", TSDataType.BOOLEAN), aggregationDescriptorList, Ordering.ASC, - new In<>(Sets.newHashSet("s1", "s2"), VALUE_FILTER, true), + TimeFilter.gt(100L), + ValueFilter.in(Sets.newHashSet("s1", "s2")), groupByTimeParameter, null); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java index 851d122bab..9ef5750de6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java @@ -53,6 +53,11 @@ public class GroupByFilter implements Filter, Serializable { return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } + @Override + public boolean allSatisfy(Statistics statistics) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } + @Override public boolean satisfy(long time, Object value) { if (time < startTime || time >= endTime) { @@ -101,7 +106,8 @@ public class GroupByFilter implements Filter, Serializable { @Override public String toString() { - return "GroupByFilter{}"; + return String.format( + "GroupByFilter{[%d, %d], %d, %d}", startTime, endTime, interval, slidingStep); } @Override @@ -161,4 +167,9 @@ public class GroupByFilter implements Filter, Serializable { ? Collections.emptyList() : Collections.singletonList(new TimeRange(startTime, endTime - 1)); } + + @Override + public Filter reverse() { + throw new UnsupportedOperationException(); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriter.java new file mode 100644 index 0000000000..4a66d032f9 --- /dev/null +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriter.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.read.filter; + +import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; +import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId; +import org.apache.iotdb.tsfile.read.filter.operator.AndFilter; +import org.apache.iotdb.tsfile.read.filter.operator.NotFilter; +import org.apache.iotdb.tsfile.read.filter.operator.OrFilter; + +public class PredicateRemoveNotRewriter { + + public static Filter rewrite(Filter filter) { + return removeNot(filter); + } + + private static Filter removeNot(Filter filter) { + FilterSerializeId filterType = filter.getSerializeId(); + switch (filterType) { + case AND: + return FilterFactory.and( + removeNot(((AndFilter) filter).getLeft()), removeNot(((AndFilter) filter).getRight())); + case OR: + return FilterFactory.or( + removeNot(((OrFilter) filter).getLeft()), removeNot(((OrFilter) filter).getRight())); + case NOT: + return ((NotFilter) filter).getFilter().reverse(); + default: + return filter; + } + } +} diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java index e33b617e60..0cbd94dc0a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/TimeFilter.java @@ -29,7 +29,6 @@ import org.apache.iotdb.tsfile.read.filter.operator.In; import org.apache.iotdb.tsfile.read.filter.operator.Lt; import org.apache.iotdb.tsfile.read.filter.operator.LtEq; import org.apache.iotdb.tsfile.read.filter.operator.NotEq; -import org.apache.iotdb.tsfile.read.filter.operator.NotFilter; import java.util.ArrayList; import java.util.Arrays; @@ -42,10 +41,6 @@ public class TimeFilter { private TimeFilter() {} - public static TimeEq eq(long value) { - return new TimeEq(value); - } - public static TimeGt gt(long value) { return new TimeGt(value); } @@ -62,160 +57,163 @@ public class TimeFilter { return new TimeLtEq(value); } - public static TimeNotFilter not(Filter filter) { - return new TimeNotFilter(filter); + public static TimeEq eq(long value) { + return new TimeEq(value); } public static TimeNotEq notEq(long value) { return new TimeNotEq(value); } - public static TimeIn in(Set<Long> values, boolean not) { - return new TimeIn(values, not); + public static TimeBetween between(long value1, long value2) { + return new TimeBetween(value1, value2, false); } - public static TimeBetween between(long value1, long value2, boolean not) { - return new TimeBetween(value1, value2, not); + public static TimeBetween notBetween(long value1, long value2) { + return new TimeBetween(value1, value2, true); } - public static class TimeBetween extends Between { - - private TimeBetween(long value1, long value2, boolean not) { - super(value1, value2, FilterType.TIME_FILTER, not); - } + public static TimeIn in(Set<Long> values) { + return new TimeIn(values, false); + } - @Override - public List<TimeRange> getTimeRanges() { - long left = (long) value1; - long right = (long) value2; - if (not) { - List<TimeRange> res = new ArrayList<>(); - if (left != Long.MIN_VALUE) { - res.add(new TimeRange(Long.MIN_VALUE, left - 1)); - } - if (right != Long.MAX_VALUE) { - res.add(new TimeRange(right + 1, Long.MAX_VALUE)); - } - return res; - } else { - return Collections.singletonList(new TimeRange(left, right)); - } - } + public static TimeIn notIn(Set<Long> values) { + return new TimeIn(values, true); } - public static class TimeIn extends In { + public static class TimeGt extends Gt<Long> { - private TimeIn(Set<Long> values, boolean not) { - super(values, FilterType.TIME_FILTER, not); + private TimeGt(long value) { + super(value, FilterType.TIME_FILTER); } @Override public List<TimeRange> getTimeRanges() { - return ((Set<Long>) values) - .stream() - .map( - l -> { - long time = l; - return new TimeRange(time, time); - }) - .collect(Collectors.toList()); + long left = value; + if (left != Long.MAX_VALUE) { + return Collections.singletonList(new TimeRange(left + 1, Long.MAX_VALUE)); + } else { + return Collections.emptyList(); + } } } - public static class TimeEq extends Eq { + public static class TimeGtEq extends GtEq<Long> { - private TimeEq(long value) { + private TimeGtEq(long value) { super(value, FilterType.TIME_FILTER); } @Override public List<TimeRange> getTimeRanges() { - return Collections.singletonList(new TimeRange((long) value, (long) value)); + return Collections.singletonList(new TimeRange(value, Long.MAX_VALUE)); } } - public static class TimeNotEq extends NotEq { + public static class TimeLt extends Lt<Long> { - private TimeNotEq(long value) { + private TimeLt(long value) { super(value, FilterType.TIME_FILTER); } @Override public List<TimeRange> getTimeRanges() { - long time = (long) value; - if (time == Long.MIN_VALUE) { - return Collections.singletonList(new TimeRange(time + 1, Long.MAX_VALUE)); - } else if (time == Long.MAX_VALUE) { - return Collections.singletonList(new TimeRange(Long.MIN_VALUE, time - 1)); + long right = value; + if (right != Long.MIN_VALUE) { + return Collections.singletonList(new TimeRange(Long.MIN_VALUE, right - 1)); } else { - return Arrays.asList( - new TimeRange(Long.MIN_VALUE, time - 1), new TimeRange(time + 1, Long.MAX_VALUE)); + return Collections.emptyList(); } } } - public static class TimeGt extends Gt { + public static class TimeLtEq extends LtEq<Long> { - private TimeGt(long value) { + private TimeLtEq(long value) { super(value, FilterType.TIME_FILTER); } @Override public List<TimeRange> getTimeRanges() { - long left = (long) value; - if (left != Long.MAX_VALUE) { - return Collections.singletonList(new TimeRange(left + 1, Long.MAX_VALUE)); - } else { - return Collections.emptyList(); - } + return Collections.singletonList(new TimeRange(Long.MIN_VALUE, value)); } } - public static class TimeGtEq extends GtEq { + public static class TimeEq extends Eq<Long> { - private TimeGtEq(long value) { + private TimeEq(long value) { super(value, FilterType.TIME_FILTER); } @Override public List<TimeRange> getTimeRanges() { - return Collections.singletonList(new TimeRange((long) value, Long.MAX_VALUE)); + return Collections.singletonList(new TimeRange(value, value)); } } - public static class TimeLt extends Lt { + public static class TimeNotEq extends NotEq<Long> { - private TimeLt(long value) { + private TimeNotEq(long value) { super(value, FilterType.TIME_FILTER); } @Override public List<TimeRange> getTimeRanges() { - long right = (long) value; - if (right != Long.MIN_VALUE) { - return Collections.singletonList(new TimeRange(Long.MIN_VALUE, right - 1)); + long time = value; + if (time == Long.MIN_VALUE) { + return Collections.singletonList(new TimeRange(time + 1, Long.MAX_VALUE)); + } else if (time == Long.MAX_VALUE) { + return Collections.singletonList(new TimeRange(Long.MIN_VALUE, time - 1)); } else { - return Collections.emptyList(); + return Arrays.asList( + new TimeRange(Long.MIN_VALUE, time - 1), new TimeRange(time + 1, Long.MAX_VALUE)); } } } - public static class TimeLtEq extends LtEq { + public static class TimeBetween extends Between<Long> { - private TimeLtEq(long value) { - super(value, FilterType.TIME_FILTER); + private TimeBetween(long value1, long value2, boolean not) { + super(value1, value2, FilterType.TIME_FILTER, not); } @Override public List<TimeRange> getTimeRanges() { - return Collections.singletonList(new TimeRange(Long.MIN_VALUE, (long) value)); + long left = value1, right = value2; + if (not) { + List<TimeRange> res = new ArrayList<>(); + if (left != Long.MIN_VALUE) { + res.add(new TimeRange(Long.MIN_VALUE, left - 1)); + } + if (right != Long.MAX_VALUE) { + res.add(new TimeRange(right + 1, Long.MAX_VALUE)); + } + return res; + } else { + return Collections.singletonList(new TimeRange(left, right)); + } } } - public static class TimeNotFilter extends NotFilter { + public static class TimeIn extends In<Long> { + + private TimeIn(Set<Long> values, boolean not) { + super(values, FilterType.TIME_FILTER, not); + } - private TimeNotFilter(Filter filter) { - super(filter); + @Override + public List<TimeRange> getTimeRanges() { + if (not) { + return Collections.singletonList(new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)); + } else { + return values.stream() + .map( + l -> { + long time = l; + return new TimeRange(time, time); + }) + .collect(Collectors.toList()); + } } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java index 3dcfbe94f8..79a4bdfd09 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/ValueFilter.java @@ -18,8 +18,8 @@ */ package org.apache.iotdb.tsfile.read.filter; -import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterType; +import org.apache.iotdb.tsfile.read.filter.operator.Between; import org.apache.iotdb.tsfile.read.filter.operator.Eq; import org.apache.iotdb.tsfile.read.filter.operator.Gt; import org.apache.iotdb.tsfile.read.filter.operator.GtEq; @@ -28,7 +28,6 @@ import org.apache.iotdb.tsfile.read.filter.operator.Like; import org.apache.iotdb.tsfile.read.filter.operator.Lt; import org.apache.iotdb.tsfile.read.filter.operator.LtEq; import org.apache.iotdb.tsfile.read.filter.operator.NotEq; -import org.apache.iotdb.tsfile.read.filter.operator.NotFilter; import org.apache.iotdb.tsfile.read.filter.operator.Regexp; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -38,88 +37,60 @@ public class ValueFilter { private ValueFilter() {} - public static <T extends Comparable<T>> ValueEq<T> eq(T value) { - return new ValueEq(value); - } - public static <T extends Comparable<T>> ValueGt<T> gt(T value) { - return new ValueGt(value); + return new ValueGt<>(value); } public static <T extends Comparable<T>> ValueGtEq<T> gtEq(T value) { - return new ValueGtEq(value); + return new ValueGtEq<>(value); } public static <T extends Comparable<T>> ValueLt<T> lt(T value) { - return new ValueLt(value); + return new ValueLt<>(value); } public static <T extends Comparable<T>> ValueLtEq<T> ltEq(T value) { - return new ValueLtEq(value); + return new ValueLtEq<>(value); } - public static <T extends Comparable<T>> ValueIn<T> in(Set<T> values, boolean not) { - return new ValueIn(values, not); + public static <T extends Comparable<T>> ValueEq<T> eq(T value) { + return new ValueEq<>(value); } - public static ValueNotFilter not(Filter filter) { - return new ValueNotFilter(filter); + public static <T extends Comparable<T>> ValueNotEq<T> notEq(T value) { + return new ValueNotEq<>(value); } - public static <T extends Comparable<T>> ValueNotEq<T> notEq(T value) { - return new ValueNotEq(value); + public static <T extends Comparable<T>> ValueBetween<T> between(T value1, T value2) { + return new ValueBetween<>(value1, value2, false); } - public static <T extends Comparable<T>> ValueRegexp<T> regexp(String value) { - return new ValueRegexp(value); + public static <T extends Comparable<T>> ValueBetween<T> notBetween(T value1, T value2) { + return new ValueBetween<>(value1, value2, true); } public static <T extends Comparable<T>> ValueLike<T> like(String value) { - return new ValueLike(value); + return new ValueLike<>(value, false); } - public static class ValueIn<T extends Comparable<T>> extends In<T> { - - private ValueIn(Set<T> values, boolean not) { - super(values, FilterType.VALUE_FILTER, not); - } + public static <T extends Comparable<T>> ValueLike<T> notLike(String value) { + return new ValueLike<>(value, true); } - public static class VectorValueIn<T extends Comparable<T>> extends ValueIn<T> { - - private final int index; - - private VectorValueIn(Set<T> values, boolean not, int index) { - super(values, not); - this.index = index; - } - - public boolean satisfy(long time, TsPrimitiveType[] values) { - Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue(); - return this.values.contains(v) != not; - } + public static <T extends Comparable<T>> ValueRegexp<T> regexp(String value) { + return new ValueRegexp<>(value, false); } - public static class ValueEq<T extends Comparable<T>> extends Eq<T> { - - private ValueEq(T value) { - super(value, FilterType.VALUE_FILTER); - } + public static <T extends Comparable<T>> ValueRegexp<T> notRegexp(String value) { + return new ValueRegexp<>(value, true); } - public static class VectorValueEq<T extends Comparable<T>> extends ValueEq<T> { - - private final int index; - - private VectorValueEq(T value, int index) { - super(value); - this.index = index; - } + public static <T extends Comparable<T>> ValueIn<T> in(Set<T> values) { + return new ValueIn<>(values, false); + } - public boolean satisfy(long time, TsPrimitiveType[] values) { - Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue(); - return this.value.equals(v); - } + public static <T extends Comparable<T>> ValueIn<T> notIn(Set<T> values) { + return new ValueIn<>(values, true); } public static class ValueGt<T extends Comparable<T>> extends Gt<T> { @@ -210,15 +181,25 @@ public class ValueFilter { } } - public static class ValueNotFilter extends NotFilter { + public static class ValueEq<T extends Comparable<T>> extends Eq<T> { - private ValueNotFilter(Filter filter) { - super(filter); + private ValueEq(T value) { + super(value, FilterType.VALUE_FILTER); } + } + + public static class VectorValueEq<T extends Comparable<T>> extends ValueEq<T> { - @Override - public String toString() { - return FilterType.VALUE_FILTER + super.toString(); + private final int index; + + private VectorValueEq(T value, int index) { + super(value); + this.index = index; + } + + public boolean satisfy(long time, TsPrimitiveType[] values) { + Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue(); + return this.value.equals(v); } } @@ -244,32 +225,35 @@ public class ValueFilter { } } - public static class ValueRegexp<T extends Comparable<T>> extends Regexp<T> { + public static class ValueBetween<T extends Comparable<T>> extends Between<T> { - private ValueRegexp(String value) { - super(value, FilterType.VALUE_FILTER); + private ValueBetween(T value1, T value2, boolean not) { + super(value1, value2, FilterType.VALUE_FILTER, not); } } - public static class VectorValueRegexp<T extends Comparable<T>> extends ValueRegexp<T> { + public static class VectorValueBetween<T extends Comparable<T>> extends ValueBetween<T> { private final int index; - private VectorValueRegexp(String value, int index) { - super(value); + private VectorValueBetween(T value1, T value2, boolean not, int index) { + super(value1, value2, not); this.index = index; } public boolean satisfy(long time, TsPrimitiveType[] values) { - Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue(); - return this.value.equals(v); + if (filterType != FilterType.VALUE_FILTER) { + return false; + } + Object v = values[index].getValue(); + return (value1.compareTo((T) v) <= 0 && ((T) v).compareTo(value2) <= 0) ^ not; } } public static class ValueLike<T extends Comparable<T>> extends Like<T> { - private ValueLike(String value) { - super(value, FilterType.VALUE_FILTER); + private ValueLike(String value, boolean not) { + super(value, FilterType.VALUE_FILTER, not); } } @@ -277,14 +261,63 @@ public class ValueFilter { private final int index; - private VectorValueLike(String value, int index) { - super(value); + private VectorValueLike(String value, int index, boolean not) { + super(value, not); + this.index = index; + } + + public boolean satisfy(long time, TsPrimitiveType[] values) { + if (filterType != FilterType.VALUE_FILTER) { + return false; + } + Object value = values[index].getValue(); + return pattern.matcher(value.toString()).find() != not; + } + } + + public static class ValueRegexp<T extends Comparable<T>> extends Regexp<T> { + private ValueRegexp(String value, boolean not) { + super(value, FilterType.VALUE_FILTER, not); + } + } + + public static class VectorValueRegexp<T extends Comparable<T>> extends ValueRegexp<T> { + + private final int index; + + private VectorValueRegexp(String value, int index, boolean not) { + super(value, not); + this.index = index; + } + + public boolean satisfy(long time, TsPrimitiveType[] values) { + if (filterType != FilterType.VALUE_FILTER) { + return false; + } + Object value = values[index].getValue(); + return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find() != not; + } + } + + public static class ValueIn<T extends Comparable<T>> extends In<T> { + + private ValueIn(Set<T> values, boolean not) { + super(values, FilterType.VALUE_FILTER, not); + } + } + + public static class VectorValueIn<T extends Comparable<T>> extends ValueIn<T> { + + private final int index; + + private VectorValueIn(Set<T> values, boolean not, int index) { + super(values, not); this.index = index; } public boolean satisfy(long time, TsPrimitiveType[] values) { Object v = filterType == FilterType.TIME_FILTER ? time : values[index].getValue(); - return this.value.equals(v); + return this.values.contains(v) != not; } } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java index 1cba9c66bd..0d91b58b08 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/BinaryFilter.java @@ -57,11 +57,6 @@ public abstract class BinaryFilter implements Filter, Serializable { return right; } - @Override - public String toString() { - return "( " + left + "," + right + " )"; - } - @Override public abstract Filter copy(); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java index dc0d479c83..691cf1030d 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java @@ -32,12 +32,19 @@ import java.util.List; public interface Filter { /** - * To examine whether the statistics is satisfied with the filter. + * To examine whether there are data points satisfied with the filter. * * @param statistics statistics with min time, max time, min value, max value. */ boolean satisfy(Statistics statistics); + /** + * To examine whether all data points are satisfied with the filter. + * + * @param statistics statistics with min time, max time, min value, max value. + */ + boolean allSatisfy(Statistics statistics); + /** * To examine whether the single point(with time and value) is satisfied with the filter. * @@ -80,4 +87,6 @@ public interface Filter { default List<TimeRange> getTimeRanges() { return Collections.emptyList(); } + + Filter reverse(); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java index 4dff4e6337..6d647b96c5 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/FilterFactory.java @@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.read.filter.operator.Eq; import org.apache.iotdb.tsfile.read.filter.operator.Gt; import org.apache.iotdb.tsfile.read.filter.operator.GtEq; import org.apache.iotdb.tsfile.read.filter.operator.In; +import org.apache.iotdb.tsfile.read.filter.operator.Like; import org.apache.iotdb.tsfile.read.filter.operator.Lt; import org.apache.iotdb.tsfile.read.filter.operator.LtEq; import org.apache.iotdb.tsfile.read.filter.operator.NotEq; @@ -82,9 +83,6 @@ public class FilterFactory { case LTEQ: filter = new LtEq<>(); break; - case BETWEEN: - filter = new Between<>(); - break; case IN: filter = new In<>(); break; @@ -97,6 +95,12 @@ public class FilterFactory { case REGEXP: filter = new Regexp<>(); break; + case LIKE: + filter = new Like<>(); + break; + case BETWEEN: + filter = new Between<>(); + break; default: throw new UnsupportedOperationException("Unknown filter type " + id); } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java index 6c5120a3f7..1488c70135 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/AndFilter.java @@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.BinaryFilter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId; import java.util.ArrayList; @@ -43,6 +44,11 @@ public class AndFilter extends BinaryFilter { return left.satisfy(statistics) && right.satisfy(statistics); } + @Override + public boolean allSatisfy(Statistics statistics) { + return left.allSatisfy(statistics) && right.allSatisfy(statistics); + } + @Override public boolean satisfy(long time, Object value) { return left.satisfy(time, value) && right.satisfy(time, value); @@ -110,4 +116,9 @@ public class AndFilter extends BinaryFilter { return result; } + + @Override + public Filter reverse() { + return FilterFactory.or(left.reverse(), right.reverse()); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java index 0ca994f315..9a0281a772 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Between.java @@ -28,9 +28,11 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; +import java.util.Objects; -public class Between<T extends Comparable<T>> implements Filter { +public class Between<T extends Comparable<T>> implements Filter, Serializable { private static final long serialVersionUID = -537390606419370764L; @@ -51,6 +53,66 @@ public class Between<T extends Comparable<T>> implements Filter { this.not = not; } + @Override + public boolean satisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return true; + } + return not + ? (((T) statistics.getMinValue()).compareTo(value1) < 0 + || ((T) statistics.getMaxValue()).compareTo(value2) > 0) + : (((T) statistics.getMaxValue()).compareTo(value1) >= 0 + && ((T) statistics.getMinValue()).compareTo(value2) <= 0); + } + } + + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return not + ? (((T) statistics.getMinValue()).compareTo(value2) > 0 + || ((T) statistics.getMaxValue()).compareTo(value1) < 0) + : (((T) statistics.getMinValue()).compareTo(value1) >= 0 + && ((T) statistics.getMaxValue()).compareTo(value2) <= 0); + } + } + + @Override + public boolean satisfy(long time, Object value) { + Object v = filterType == FilterType.TIME_FILTER ? time : value; + return (value1.compareTo((T) v) <= 0 && ((T) v).compareTo(value2) <= 0) ^ not; + } + + @Override + public boolean satisfyStartEndTime(long startTime, long endTime) { + if (filterType == FilterType.TIME_FILTER) { + return not + ? (startTime < (Long) value1 || endTime > (Long) value2) + : (endTime >= (Long) value1 && startTime <= (Long) value2); + } else { + return true; + } + } + + @Override + public boolean containStartEndTime(long startTime, long endTime) { + if (filterType == FilterType.TIME_FILTER) { + return not + ? (startTime > (Long) value2 || endTime < (Long) value1) + : (startTime >= (Long) value1 && endTime <= (Long) value2); + } else { + return false; + } + } + @Override public void serialize(DataOutputStream outputStream) { try { @@ -78,64 +140,29 @@ public class Between<T extends Comparable<T>> implements Filter { } @Override - public boolean satisfy(Statistics statistics) { - if (filterType == FilterType.TIME_FILTER) { - long time1 = (Long) value1, time2 = (Long) value2; - if (not) { - return statistics.getStartTime() < time1 || statistics.getEndTime() > time2; - } else { - return statistics.getEndTime() >= time1 || statistics.getStartTime() <= time2; - } - } else { - if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { - return true; - } - if (not) { - return ((T) statistics.getMinValue()).compareTo(value1) < 0 - || ((T) statistics.getMaxValue()).compareTo(value2) > 0; - } else { - return ((T) statistics.getMaxValue()).compareTo(value1) >= 0 - && ((T) statistics.getMinValue()).compareTo(value2) <= 0; - } - } - } - - @Override - public boolean satisfy(long time, Object value) { - Object v = filterType == FilterType.TIME_FILTER ? time : value; - return (value1.compareTo((T) v) <= 0 && ((T) v).compareTo(value2) <= 0) ^ not; + public Filter copy() { + return new Between<>(value1, value2, filterType, not); } @Override - public boolean satisfyStartEndTime(long startTime, long endTime) { - if (filterType == FilterType.TIME_FILTER) { - long time1 = (Long) value1, time2 = (Long) value2; - if (not) { - return startTime < time1 || endTime > time2; - } else { - return endTime >= time1 && startTime <= time2; - } - } else { - return true; + public boolean equals(Object o) { + if (!(o instanceof Between)) { + return false; } + Between<?> between = (Between<?>) o; + return not == between.not + && value1.equals(between.value1) + && value2.equals(between.value2) + && filterType == between.filterType; } @Override - public boolean containStartEndTime(long startTime, long endTime) { - if (filterType == FilterType.TIME_FILTER) { - long time1 = (Long) value1, time2 = (Long) value2; - if (not) { - return endTime < time1 || startTime > time2; - } else { - return startTime >= time1 && endTime <= time2; - } - } else { - return true; - } + public int hashCode() { + return Objects.hash(value1, value2, not, filterType); } @Override - public Filter copy() { - return new Between(value1, value2, filterType, not); + public Filter reverse() { + return new Between<>(value1, value2, filterType, !not); } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java index 7a69be9468..922a102c9c 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Eq.java @@ -43,8 +43,7 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> { @Override public boolean satisfy(Statistics statistics) { if (filterType == FilterType.TIME_FILTER) { - return ((Long) value) >= statistics.getStartTime() - && ((Long) value) <= statistics.getEndTime(); + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } else { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return true; @@ -54,6 +53,19 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> { } } + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return value.compareTo((T) statistics.getMinValue()) == 0 + && value.compareTo((T) statistics.getMaxValue()) == 0; + } + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -76,13 +88,13 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> { long time = (Long) value; return time == startTime && time == endTime; } else { - return true; + return false; } } @Override public Filter copy() { - return new Eq(value, filterType); + return new Eq<>(value, filterType); } @Override @@ -94,4 +106,9 @@ public class Eq<T extends Comparable<T>> extends UnaryFilter<T> { public FilterSerializeId getSerializeId() { return FilterSerializeId.EQ; } + + @Override + public Filter reverse() { + return new NotEq<>(value, filterType); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java index 0719397e8c..ffb6c75eab 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Gt.java @@ -43,7 +43,7 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> { @Override public boolean satisfy(Statistics statistics) { if (filterType == FilterType.TIME_FILTER) { - return ((Long) value) < statistics.getEndTime(); + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } else { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return true; @@ -52,6 +52,18 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> { } } + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return value.compareTo((T) statistics.getMinValue()) < 0; + } + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -74,13 +86,13 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> { long time = (Long) value; return startTime > time; } else { - return true; + return false; } } @Override public Filter copy() { - return new Gt(value, filterType); + return new Gt<>(value, filterType); } @Override @@ -92,4 +104,9 @@ public class Gt<T extends Comparable<T>> extends UnaryFilter<T> { public FilterSerializeId getSerializeId() { return FilterSerializeId.GT; } + + @Override + public Filter reverse() { + return new LtEq<>(value, filterType); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java index 4e811e14fe..f6b3a2cd86 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/GtEq.java @@ -43,7 +43,7 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> { @Override public boolean satisfy(Statistics statistics) { if (filterType == FilterType.TIME_FILTER) { - return ((Long) value) <= statistics.getEndTime(); + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } else { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return true; @@ -52,6 +52,18 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> { } } + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return value.compareTo((T) statistics.getMinValue()) <= 0; + } + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -74,13 +86,13 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> { long time = (Long) value; return startTime >= time; } else { - return true; + return false; } } @Override public Filter copy() { - return new GtEq(value, filterType); + return new GtEq<>(value, filterType); } @Override @@ -92,4 +104,9 @@ public class GtEq<T extends Comparable<T>> extends UnaryFilter<T> { public FilterSerializeId getSerializeId() { return FilterSerializeId.GTEQ; } + + @Override + public Filter reverse() { + return new Lt<>(value, filterType); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java index 60e0747f5d..60d6eed75f 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/In.java @@ -26,11 +26,13 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Set; /** @@ -38,7 +40,7 @@ import java.util.Set; * * @param <T> comparable data type */ -public class In<T extends Comparable<T>> implements Filter { +public class In<T extends Comparable<T>> implements Filter, Serializable { private static final long serialVersionUID = 8572705136773595399L; @@ -61,6 +63,11 @@ public class In<T extends Comparable<T>> implements Filter { return true; } + @Override + public boolean allSatisfy(Statistics statistics) { + return false; + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -74,12 +81,12 @@ public class In<T extends Comparable<T>> implements Filter { @Override public boolean containStartEndTime(long startTime, long endTime) { - return true; + return false; } @Override public Filter copy() { - return new In(new HashSet(values), filterType, not); + return new In<>(new HashSet<>(values), filterType, not); } @Override @@ -110,17 +117,23 @@ public class In<T extends Comparable<T>> implements Filter { @Override public boolean equals(Object o) { - return o instanceof In - && ((In<?>) o).filterType == filterType - && ((In<?>) o).values.equals(values) - && ((In<?>) o).not == not; + if (!(o instanceof In)) { + return false; + } + In<?> in = (In<?>) o; + return in.filterType == filterType && in.values.equals(values) && in.not == not; + } + + @Override + public int hashCode() { + return Objects.hash(values, not, filterType); } @Override public String toString() { List<T> valueList = new ArrayList<>(values); Collections.sort(valueList); - return filterType + " < " + "reverse: " + not + ", " + valueList; + return filterType + (not ? " not in " : " in ") + valueList; } @Override @@ -128,6 +141,11 @@ public class In<T extends Comparable<T>> implements Filter { return FilterSerializeId.IN; } + @Override + public Filter reverse() { + return new In<>(new HashSet<>(values), filterType, !not); + } + public Set<T> getValues() { return values; } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java index fac3dfcf5b..d509c00862 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Like.java @@ -26,7 +26,9 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; +import java.util.Objects; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; @@ -35,7 +37,7 @@ import java.util.regex.PatternSyntaxException; * * @param <T> comparable data type */ -public class Like<T extends Comparable<T>> implements Filter { +public class Like<T extends Comparable<T>> implements Filter, Serializable { private static final long serialVersionUID = 2171102599229260789L; @@ -45,15 +47,18 @@ public class Like<T extends Comparable<T>> implements Filter { protected Pattern pattern; - private Like() {} + protected boolean not; + + public Like() {} /** * The main idea of this part comes from * https://codereview.stackexchange.com/questions/36861/convert-sql-like-to-regex/36864 */ - public Like(String value, FilterType filterType) { + public Like(String value, FilterType filterType, boolean not) { this.value = value; this.filterType = filterType; + this.not = not; try { String unescapeValue = unescapeString(value); String specialRegexStr = ".^$*+?{}[]|()"; @@ -89,12 +94,17 @@ public class Like<T extends Comparable<T>> implements Filter { return true; } + @Override + public boolean allSatisfy(Statistics statistics) { + return false; + } + @Override public boolean satisfy(long time, Object value) { if (filterType != FilterType.VALUE_FILTER) { - return false; + throw new UnsupportedOperationException(""); } - return pattern.matcher(value.toString()).find(); + return not != pattern.matcher(value.toString()).find(); } @Override @@ -104,12 +114,12 @@ public class Like<T extends Comparable<T>> implements Filter { @Override public boolean containStartEndTime(long startTime, long endTime) { - return true; + return false; } @Override public Filter copy() { - return new Like(value, filterType); + return new Like<>(value, filterType, not); } @Override @@ -117,9 +127,10 @@ public class Like<T extends Comparable<T>> implements Filter { try { outputStream.write(getSerializeId().ordinal()); outputStream.write(filterType.ordinal()); - ReadWriteIOUtils.writeObject(value, outputStream); - } catch (IOException ex) { - throw new IllegalArgumentException("Failed to serialize outputStream of type:", ex); + ReadWriteIOUtils.write(value, outputStream); + ReadWriteIOUtils.write(not, outputStream); + } catch (IOException ignored) { + // ignore } } @@ -127,11 +138,12 @@ public class Like<T extends Comparable<T>> implements Filter { public void deserialize(ByteBuffer buffer) { filterType = FilterType.values()[buffer.get()]; value = ReadWriteIOUtils.readString(buffer); + not = ReadWriteIOUtils.readBool(buffer); } @Override public String toString() { - return filterType + " is " + value; + return filterType + (not ? " not like " : " like ") + value; } @Override @@ -139,11 +151,30 @@ public class Like<T extends Comparable<T>> implements Filter { return FilterSerializeId.LIKE; } + @Override + public Filter reverse() { + return new Like<>(value, filterType, !not); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Like)) { + return false; + } + Like<?> like = (Like<?>) o; + return not == like.not && value.equals(like.value) && filterType == like.filterType; + } + + @Override + public int hashCode() { + return Objects.hash(value, filterType, not); + } + /** * This Method is for unescaping strings except '\' before special string '%', '_', '\', because * we need to use '\' to judege whether to replace this to regexp string */ - public String unescapeString(String value) { + private String unescapeString(String value) { String out = ""; for (int i = 0; i < value.length(); i++) { String ch = String.valueOf(value.charAt(i)); diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java index 9c83b720c2..c0151d3f63 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Lt.java @@ -43,7 +43,7 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> { @Override public boolean satisfy(Statistics statistics) { if (filterType == FilterType.TIME_FILTER) { - return ((Long) value) > statistics.getStartTime(); + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } else { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return true; @@ -52,6 +52,18 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> { } } + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return value.compareTo((T) statistics.getMaxValue()) > 0; + } + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -74,13 +86,13 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> { long time = (Long) value; return endTime < time; } else { - return true; + return false; } } @Override public Filter copy() { - return new Lt(value, filterType); + return new Lt<>(value, filterType); } @Override @@ -92,4 +104,9 @@ public class Lt<T extends Comparable<T>> extends UnaryFilter<T> { public FilterSerializeId getSerializeId() { return FilterSerializeId.LT; } + + @Override + public Filter reverse() { + return new GtEq<>(value, filterType); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java index b68b1b4759..4b35f154a1 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/LtEq.java @@ -43,7 +43,7 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> { @Override public boolean satisfy(Statistics statistics) { if (filterType == FilterType.TIME_FILTER) { - return ((Long) value) >= statistics.getStartTime(); + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } else { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return true; @@ -52,6 +52,18 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> { } } + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return value.compareTo((T) statistics.getMaxValue()) >= 0; + } + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -74,13 +86,13 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> { long time = (Long) value; return endTime <= time; } else { - return true; + return false; } } @Override public Filter copy() { - return new LtEq(value, filterType); + return new LtEq<>(value, filterType); } @Override @@ -92,4 +104,9 @@ public class LtEq<T extends Comparable<T>> extends UnaryFilter<T> { public FilterSerializeId getSerializeId() { return FilterSerializeId.LTEQ; } + + @Override + public Filter reverse() { + return new Gt<>(value, filterType); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java index 55abda179e..45148c5c03 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotEq.java @@ -43,8 +43,7 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> { @Override public boolean satisfy(Statistics statistics) { if (filterType == FilterType.TIME_FILTER) { - return !(((Long) value) == statistics.getStartTime() - && (Long) value == statistics.getEndTime()); + return satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } else { if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { return true; @@ -54,6 +53,19 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> { } } + @Override + public boolean allSatisfy(Statistics statistics) { + if (filterType == FilterType.TIME_FILTER) { + return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); + } else { + if (statistics.getType() == TSDataType.TEXT || statistics.getType() == TSDataType.BOOLEAN) { + return false; + } + return value.compareTo((T) statistics.getMinValue()) < 0 + || value.compareTo((T) statistics.getMaxValue()) > 0; + } + } + @Override public boolean satisfy(long time, Object value) { Object v = filterType == FilterType.TIME_FILTER ? time : value; @@ -76,13 +88,13 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> { long time = (Long) value; return time < startTime || time > endTime; } else { - return true; + return false; } } @Override public Filter copy() { - return new NotEq(value, filterType); + return new NotEq<>(value, filterType); } @Override @@ -94,4 +106,9 @@ public class NotEq<T extends Comparable<T>> extends UnaryFilter<T> { public FilterSerializeId getSerializeId() { return FilterSerializeId.NEQ; } + + @Override + public Filter reverse() { + return new Eq<>(value, filterType); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java index cd983dc329..ef21c14a0c 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/NotFilter.java @@ -32,12 +32,14 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -/** NotFilter necessary. Use InvertExpressionVisitor */ public class NotFilter implements Filter, Serializable { private static final long serialVersionUID = 584860326604020881L; private Filter that; + public static final String CONTAIN_NOT_ERR_MSG = + "This predicate contains a not! Did you forget to run this predicate through PredicateRemoveNotRewriter? "; + public NotFilter() {} public NotFilter(Filter that) { @@ -46,7 +48,12 @@ public class NotFilter implements Filter, Serializable { @Override public boolean satisfy(Statistics statistics) { - return !that.satisfy(statistics); + throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); + } + + @Override + public boolean allSatisfy(Statistics statistics) { + throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); } @Override @@ -54,18 +61,14 @@ public class NotFilter implements Filter, Serializable { return !that.satisfy(time, value); } - /** - * Notice that, if the not filter only contains value filter, this method may return false, this - * may cause misunderstanding. - */ @Override public boolean satisfyStartEndTime(long startTime, long endTime) { - return !that.satisfyStartEndTime(startTime, endTime); + throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); } @Override public boolean containStartEndTime(long startTime, long endTime) { - return !that.satisfyStartEndTime(startTime, endTime); + throw new UnsupportedOperationException(CONTAIN_NOT_ERR_MSG + this); } @Override @@ -79,7 +82,7 @@ public class NotFilter implements Filter, Serializable { @Override public String toString() { - return "NotFilter: " + that; + return "not (" + that + ")"; } @Override @@ -139,4 +142,9 @@ public class NotFilter implements Filter, Serializable { } return res; } + + @Override + public Filter reverse() { + return that; + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java index 17b0935510..0a16909e2c 100755 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/OrFilter.java @@ -22,6 +22,7 @@ import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.BinaryFilter; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; import org.apache.iotdb.tsfile.read.filter.factory.FilterSerializeId; import java.io.Serializable; @@ -55,6 +56,11 @@ public class OrFilter extends BinaryFilter implements Serializable { return left.satisfy(statistics) || right.satisfy(statistics); } + @Override + public boolean allSatisfy(Statistics statistics) { + return left.allSatisfy(statistics) || right.allSatisfy(statistics); + } + @Override public boolean satisfy(long time, Object value) { return left.satisfy(time, value) || right.satisfy(time, value); @@ -138,4 +144,9 @@ public class OrFilter extends BinaryFilter implements Serializable { return result; } + + @Override + public Filter reverse() { + return FilterFactory.and(left.reverse(), right.reverse()); + } } diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java index 31e7260469..3a154e7320 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Regexp.java @@ -27,6 +27,7 @@ import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; import java.io.DataOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Objects; import java.util.regex.Pattern; @@ -37,7 +38,9 @@ import java.util.regex.PatternSyntaxException; * * @param <T> comparable data type */ -public class Regexp<T extends Comparable<T>> implements Filter { +public class Regexp<T extends Comparable<T>> implements Filter, Serializable { + + private static final long serialVersionUID = -1168073851950524983L; protected String value; @@ -45,11 +48,14 @@ public class Regexp<T extends Comparable<T>> implements Filter { protected Pattern pattern; + protected boolean not; + public Regexp() {} - public Regexp(String value, FilterType filterType) { + public Regexp(String value, FilterType filterType, boolean not) { this.value = value; this.filterType = filterType; + this.not = not; try { this.pattern = Pattern.compile(this.value); } catch (PatternSyntaxException e) { @@ -62,12 +68,17 @@ public class Regexp<T extends Comparable<T>> implements Filter { return true; } + @Override + public boolean allSatisfy(Statistics statistics) { + return false; + } + @Override public boolean satisfy(long time, Object value) { if (filterType != FilterType.VALUE_FILTER) { - return false; + throw new UnsupportedOperationException(""); } - return pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find(); + return not != pattern.matcher(new MatcherInput(value.toString(), new AccessCount())).find(); } @Override @@ -77,12 +88,12 @@ public class Regexp<T extends Comparable<T>> implements Filter { @Override public boolean containStartEndTime(long startTime, long endTime) { - return true; + return false; } @Override public Filter copy() { - return new Regexp(value, filterType); + return new Regexp<>(value, filterType, not); } @Override @@ -91,8 +102,9 @@ public class Regexp<T extends Comparable<T>> implements Filter { outputStream.write(getSerializeId().ordinal()); outputStream.write(filterType.ordinal()); ReadWriteIOUtils.write(value, outputStream); - } catch (IOException ex) { - throw new IllegalArgumentException("Failed to serialize outputStream of type:", ex); + ReadWriteIOUtils.write(not, outputStream); + } catch (IOException ignored) { + // ignore } } @@ -100,6 +112,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { public void deserialize(ByteBuffer buffer) { filterType = FilterType.values()[buffer.get()]; value = ReadWriteIOUtils.readString(buffer); + not = ReadWriteIOUtils.readBool(buffer); if (value != null) { try { this.pattern = Pattern.compile(value); @@ -111,22 +124,34 @@ public class Regexp<T extends Comparable<T>> implements Filter { @Override public String toString() { - return filterType + " is " + value; + return filterType + (not ? " not match " : " match ") + value; + } + + @Override + public FilterSerializeId getSerializeId() { + return FilterSerializeId.REGEXP; + } + + @Override + public Filter reverse() { + return new Regexp<>(value, filterType, !not); } @Override public boolean equals(Object o) { - return o instanceof Regexp - && Objects.equals(((Regexp<?>) o).value, value) - && ((Regexp<?>) o).filterType == filterType; + if (!(o instanceof Regexp)) { + return false; + } + Regexp<?> regexp = (Regexp<?>) o; + return not == regexp.not && value.equals(regexp.value) && filterType == regexp.filterType; } @Override - public FilterSerializeId getSerializeId() { - return FilterSerializeId.REGEXP; + public int hashCode() { + return Objects.hash(value, filterType, not); } - private static class AccessCount { + public static class AccessCount { private int count; private final int accessThreshold = TSFileDescriptor.getInstance().getConfig().getPatternMatchingThreshold(); @@ -138,7 +163,7 @@ public class Regexp<T extends Comparable<T>> implements Filter { } } - private static class MatcherInput implements CharSequence { + public static class MatcherInput implements CharSequence { private final CharSequence value; diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index 795116ccb5..60b74a1870 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -163,16 +163,18 @@ public class PageReader implements IPageReader { } private boolean pageSatisfy() { - if (filter != null) { - return filter.satisfy(getStatistics()); - } else { - long rowCount = getStatistics().getCount(); + Statistics statistics = getStatistics(); + if (filter == null || filter.allSatisfy(statistics)) { + long rowCount = statistics.getCount(); if (paginationController.hasCurOffset(rowCount)) { paginationController.consumeOffset(rowCount); return false; + } else { + return true; } + } else { + return filter.satisfy(statistics); } - return true; } @Override diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java index 779e32d404..2d3b5c0abf 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterSerializeTest.java @@ -43,12 +43,17 @@ public class FilterSerializeTest { ValueFilter.gtEq("filter"), ValueFilter.lt(0.1), ValueFilter.ltEq(0.01f), - ValueFilter.not(ValueFilter.eq(true)), + FilterFactory.not(ValueFilter.eq(true)), ValueFilter.notEq(false), ValueFilter.notEq(false), - ValueFilter.in(new HashSet<>(Arrays.asList("a", "b")), false), - ValueFilter.in(new HashSet<>(Arrays.asList("c", "d")), true), + ValueFilter.in(new HashSet<>(Arrays.asList("a", "b"))), + ValueFilter.notIn(new HashSet<>(Arrays.asList("c", "d"))), ValueFilter.regexp("s.*"), + ValueFilter.like("s.*"), + ValueFilter.notRegexp("s.*"), + ValueFilter.notLike("s.*"), + ValueFilter.between(1, 100), + ValueFilter.notBetween(1, 100) }; for (Filter filter : filters) { validateSerialization(filter); @@ -64,11 +69,13 @@ public class FilterSerializeTest { TimeFilter.gtEq(3), TimeFilter.lt(4), TimeFilter.ltEq(5), - TimeFilter.not(ValueFilter.eq(6)), + FilterFactory.not(ValueFilter.eq(6)), TimeFilter.notEq(7), TimeFilter.notEq(7), - TimeFilter.in(new HashSet<>(Arrays.asList(1L, 2L)), false), - TimeFilter.in(new HashSet<>(Arrays.asList(3L, 4L)), true), + TimeFilter.in(new HashSet<>(Arrays.asList(1L, 2L))), + TimeFilter.notIn(new HashSet<>(Arrays.asList(3L, 4L))), + TimeFilter.between(1, 100), + TimeFilter.notBetween(1, 100) }; for (Filter filter : filters) { validateSerialization(filter); @@ -80,7 +87,7 @@ public class FilterSerializeTest { Filter[] filters = new Filter[] { FilterFactory.and(TimeFilter.eq(1), ValueFilter.eq(1)), - FilterFactory.or(ValueFilter.gt(2L), TimeFilter.not(ValueFilter.eq(6))) + FilterFactory.or(ValueFilter.gt(2L), FilterFactory.not(ValueFilter.eq(6))) }; for (Filter filter : filters) { validateSerialization(filter); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java index df4f9d06f3..cf1b31a9ef 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/MinTimeMaxTimeFilterTest.java @@ -25,6 +25,9 @@ import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; import org.junit.Assert; import org.junit.Test; +import static org.apache.iotdb.tsfile.read.filter.operator.NotFilter.CONTAIN_NOT_ERR_MSG; +import static org.junit.Assert.fail; + public class MinTimeMaxTimeFilterTest { long minTime = 100; @@ -81,7 +84,7 @@ public class MinTimeMaxTimeFilterTest { Filter valueEq = ValueFilter.gt(100); Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime)); - Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime)); + Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime)); } @Test @@ -104,11 +107,11 @@ public class MinTimeMaxTimeFilterTest { Filter valueEq = ValueFilter.gtEq(100); Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime)); - Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime)); + Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime)); valueEq = ValueFilter.gtEq(150); Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime)); - Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime)); + Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime)); } @Test @@ -131,7 +134,7 @@ public class MinTimeMaxTimeFilterTest { Filter valueEq = ValueFilter.lt(100); Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime)); - Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime)); + Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime)); } @Test @@ -154,7 +157,7 @@ public class MinTimeMaxTimeFilterTest { Filter valueEq = ValueFilter.ltEq(100); Assert.assertTrue(valueEq.satisfyStartEndTime(minTime, maxTime)); - Assert.assertTrue(valueEq.containStartEndTime(minTime, maxTime)); + Assert.assertFalse(valueEq.containStartEndTime(minTime, maxTime)); } @Test @@ -193,22 +196,17 @@ public class MinTimeMaxTimeFilterTest { @Test public void testNot() { Filter not = FilterFactory.not(TimeFilter.ltEq(10L)); - Assert.assertTrue(not.satisfyStartEndTime(minTime, maxTime)); - Assert.assertTrue(not.containStartEndTime(minTime, maxTime)); - - not = FilterFactory.not(TimeFilter.ltEq(100L)); - Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime)); - Assert.assertFalse(not.containStartEndTime(minTime, maxTime)); - - not = FilterFactory.not(TimeFilter.ltEq(200L)); - Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime)); - Assert.assertFalse(not.containStartEndTime(minTime, maxTime)); - - not = FilterFactory.not(TimeFilter.ltEq(300L)); - Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime)); - Assert.assertFalse(not.containStartEndTime(minTime, maxTime)); - - not = FilterFactory.not(ValueFilter.ltEq(100)); - Assert.assertFalse(not.satisfyStartEndTime(minTime, maxTime)); + try { + not.satisfyStartEndTime(minTime, maxTime); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); + } + try { + not.containStartEndTime(minTime, maxTime); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); + } } } diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java index 7caec9eaca..12c8162966 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/OperatorTest.java @@ -103,12 +103,12 @@ public class OperatorTest { @Test public void testNot() { - Filter timeLt = TimeFilter.not(TimeFilter.lt(TESTED_TIMESTAMP)); + Filter timeLt = FilterFactory.not(TimeFilter.lt(TESTED_TIMESTAMP)); Assert.assertFalse(timeLt.satisfy(TESTED_TIMESTAMP - 1, 100)); Assert.assertTrue(timeLt.satisfy(TESTED_TIMESTAMP, 100)); Assert.assertTrue(timeLt.satisfy(TESTED_TIMESTAMP + 1, 100)); - Filter valueLt = ValueFilter.not(ValueFilter.lt(100L)); + Filter valueLt = FilterFactory.not(ValueFilter.lt(100L)); Assert.assertFalse(valueLt.satisfy(TESTED_TIMESTAMP, 99L)); Assert.assertTrue(valueLt.satisfy(TESTED_TIMESTAMP, 100L)); Assert.assertTrue(valueLt.satisfy(TESTED_TIMESTAMP, 101L)); diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriterTest.java new file mode 100644 index 0000000000..f436a52238 --- /dev/null +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/PredicateRemoveNotRewriterTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tsfile.read.filter; + +import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; +import org.apache.iotdb.tsfile.read.filter.factory.FilterType; +import org.apache.iotdb.tsfile.read.filter.operator.Between; +import org.apache.iotdb.tsfile.read.filter.operator.Eq; +import org.apache.iotdb.tsfile.read.filter.operator.Gt; +import org.apache.iotdb.tsfile.read.filter.operator.GtEq; +import org.apache.iotdb.tsfile.read.filter.operator.In; +import org.apache.iotdb.tsfile.read.filter.operator.Like; +import org.apache.iotdb.tsfile.read.filter.operator.Lt; +import org.apache.iotdb.tsfile.read.filter.operator.LtEq; +import org.apache.iotdb.tsfile.read.filter.operator.NotEq; +import org.apache.iotdb.tsfile.read.filter.operator.Regexp; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashSet; + +public class PredicateRemoveNotRewriterTest { + + @Test + public void testReverse() { + Assert.assertEquals( + new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER).reverse()); + Assert.assertEquals( + new GtEq<>(1, FilterType.TIME_FILTER), new Lt<>(1, FilterType.TIME_FILTER).reverse()); + Assert.assertEquals( + new Lt<>(1, FilterType.TIME_FILTER), new GtEq<>(1, FilterType.TIME_FILTER).reverse()); + Assert.assertEquals( + new LtEq<>(1, FilterType.TIME_FILTER), new Gt<>(1, FilterType.TIME_FILTER).reverse()); + Assert.assertEquals( + new Eq<>(1, FilterType.TIME_FILTER), new NotEq<>(1, FilterType.TIME_FILTER).reverse()); + Assert.assertEquals( + new NotEq<>(1, FilterType.TIME_FILTER), new Eq<>(1, FilterType.TIME_FILTER).reverse()); + Assert.assertEquals( + new Like<>("s*", FilterType.TIME_FILTER, true), + new Like<>("s*", FilterType.TIME_FILTER, false).reverse()); + Assert.assertEquals( + new Like<>("s*", FilterType.TIME_FILTER, false), + new Like<>("s*", FilterType.TIME_FILTER, true).reverse()); + Assert.assertEquals( + new Regexp<>("s*", FilterType.TIME_FILTER, true), + new Regexp<>("s*", FilterType.TIME_FILTER, false).reverse()); + Assert.assertEquals( + new Regexp<>("s*", FilterType.TIME_FILTER, false), + new Regexp<>("s*", FilterType.TIME_FILTER, true).reverse()); + Assert.assertEquals( + new Between<>(1, 100, FilterType.TIME_FILTER, true), + new Between<>(1, 100, FilterType.TIME_FILTER, false).reverse()); + Assert.assertEquals( + new Between<>(1, 100, FilterType.TIME_FILTER, false), + new Between<>(1, 100, FilterType.TIME_FILTER, true).reverse()); + Assert.assertEquals( + new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, true), + new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, false).reverse()); + Assert.assertEquals( + new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, false), + new In<>(new HashSet<>(Arrays.asList("a", "b")), FilterType.TIME_FILTER, true).reverse()); + Assert.assertEquals( + new Gt<>(1, FilterType.TIME_FILTER), + FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER)).reverse()); + Assert.assertEquals( + FilterFactory.and( + new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)), + FilterFactory.or(new LtEq<>(1, FilterType.TIME_FILTER), new Gt<>(1, FilterType.TIME_FILTER)) + .reverse()); + Assert.assertEquals( + FilterFactory.or( + new LtEq<>(1, FilterType.TIME_FILTER), new Gt<>(1, FilterType.TIME_FILTER)), + FilterFactory.and( + new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)) + .reverse()); + } + + @Test + public void testRemoveNot() { + Assert.assertEquals( + new LtEq<>(1, FilterType.TIME_FILTER), + PredicateRemoveNotRewriter.rewrite(FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER)))); + Assert.assertEquals( + new Like<>("s*", FilterType.TIME_FILTER, false), + PredicateRemoveNotRewriter.rewrite( + FilterFactory.not(new Like<>("s*", FilterType.TIME_FILTER, true)))); + Assert.assertEquals( + FilterFactory.or( + new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)), + PredicateRemoveNotRewriter.rewrite( + FilterFactory.or( + FilterFactory.not(new LtEq<>(1, FilterType.TIME_FILTER)), + FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER))))); + Assert.assertEquals( + FilterFactory.and( + new Gt<>(1, FilterType.TIME_FILTER), new LtEq<>(1, FilterType.TIME_FILTER)), + PredicateRemoveNotRewriter.rewrite( + FilterFactory.and( + FilterFactory.not(new LtEq<>(1, FilterType.TIME_FILTER)), + FilterFactory.not(new Gt<>(1, FilterType.TIME_FILTER))))); + } +} diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java index 33e3f958da..d50b44a7a6 100644 --- a/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java +++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java @@ -27,10 +27,19 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.Serializable; + +import static org.apache.iotdb.tsfile.read.filter.operator.NotFilter.CONTAIN_NOT_ERR_MSG; +import static org.junit.Assert.fail; + public class StatisticsFilterTest { - private Statistics statistics1 = Statistics.getStatsByType(TSDataType.INT64); - private Statistics statistics2 = Statistics.getStatsByType(TSDataType.INT64); + private final Statistics<? extends Serializable> statistics1 = + Statistics.getStatsByType(TSDataType.INT64); + private final Statistics<? extends Serializable> statistics2 = + Statistics.getStatsByType(TSDataType.INT64); + private final Statistics<? extends Serializable> statistics3 = + Statistics.getStatsByType(TSDataType.INT64); @Before public void before() { @@ -38,6 +47,7 @@ public class StatisticsFilterTest { statistics1.update(100L, 100L); statistics2.update(101L, 101L); statistics2.update(200L, 200L); + statistics3.update(10L, 10L); } @Test @@ -45,32 +55,63 @@ public class StatisticsFilterTest { Filter timeEq = TimeFilter.eq(10L); Assert.assertTrue(timeEq.satisfy(statistics1)); Assert.assertFalse(timeEq.satisfy(statistics2)); + Assert.assertFalse(timeEq.allSatisfy(statistics1)); + Assert.assertFalse(timeEq.allSatisfy(statistics2)); + Assert.assertTrue(timeEq.allSatisfy(statistics3)); Filter valueEq = ValueFilter.eq(101L); Assert.assertFalse(valueEq.satisfy(statistics1)); Assert.assertTrue(valueEq.satisfy(statistics2)); + Assert.assertFalse(valueEq.allSatisfy(statistics1)); + Assert.assertFalse(valueEq.allSatisfy(statistics2)); + Assert.assertFalse(valueEq.allSatisfy(statistics3)); + } + + @Test + public void testNotEq() { + Filter timeNotEq = TimeFilter.notEq(10L); + Assert.assertTrue(timeNotEq.satisfy(statistics1)); + Assert.assertTrue(timeNotEq.satisfy(statistics2)); + Assert.assertFalse(timeNotEq.allSatisfy(statistics1)); + Assert.assertTrue(timeNotEq.allSatisfy(statistics2)); + Assert.assertFalse(timeNotEq.allSatisfy(statistics3)); + + Filter valueNotEq = ValueFilter.notEq(101L); + Assert.assertTrue(valueNotEq.satisfy(statistics1)); + Assert.assertTrue(valueNotEq.satisfy(statistics2)); + Assert.assertTrue(valueNotEq.allSatisfy(statistics1)); + Assert.assertFalse(valueNotEq.allSatisfy(statistics2)); + Assert.assertTrue(valueNotEq.allSatisfy(statistics3)); } @Test public void testGt() { - Filter timeGt = TimeFilter.gt(100L); - Assert.assertFalse(timeGt.satisfy(statistics1)); + Filter timeGt = TimeFilter.gt(10L); + Assert.assertTrue(timeGt.satisfy(statistics1)); Assert.assertTrue(timeGt.satisfy(statistics2)); + Assert.assertFalse(timeGt.allSatisfy(statistics1)); + Assert.assertTrue(timeGt.allSatisfy(statistics2)); Filter valueGt = ValueFilter.gt(100L); Assert.assertFalse(valueGt.satisfy(statistics1)); Assert.assertTrue(valueGt.satisfy(statistics2)); + Assert.assertFalse(valueGt.allSatisfy(statistics1)); + Assert.assertTrue(valueGt.allSatisfy(statistics2)); } @Test public void testGtEq() { - Filter timeGtEq = TimeFilter.gtEq(100L); + Filter timeGtEq = TimeFilter.gtEq(10L); Assert.assertTrue(timeGtEq.satisfy(statistics1)); Assert.assertTrue(timeGtEq.satisfy(statistics2)); + Assert.assertFalse(timeGtEq.allSatisfy(statistics1)); + Assert.assertTrue(timeGtEq.allSatisfy(statistics2)); Filter valueGtEq = ValueFilter.gtEq(100L); Assert.assertTrue(valueGtEq.satisfy(statistics1)); Assert.assertTrue(valueGtEq.satisfy(statistics2)); + Assert.assertFalse(valueGtEq.allSatisfy(statistics1)); + Assert.assertTrue(valueGtEq.allSatisfy(statistics2)); } @Test @@ -78,10 +119,14 @@ public class StatisticsFilterTest { Filter timeLt = TimeFilter.lt(101L); Assert.assertTrue(timeLt.satisfy(statistics1)); Assert.assertFalse(timeLt.satisfy(statistics2)); + Assert.assertTrue(timeLt.allSatisfy(statistics1)); + Assert.assertFalse(timeLt.allSatisfy(statistics2)); - Filter valueLt = ValueFilter.lt(101L); + Filter valueLt = ValueFilter.lt(11L); Assert.assertTrue(valueLt.satisfy(statistics1)); Assert.assertFalse(valueLt.satisfy(statistics2)); + Assert.assertFalse(valueLt.allSatisfy(statistics1)); + Assert.assertFalse(valueLt.allSatisfy(statistics2)); } @Test @@ -90,9 +135,11 @@ public class StatisticsFilterTest { Assert.assertTrue(timeLtEq.satisfy(statistics1)); Assert.assertTrue(timeLtEq.satisfy(statistics2)); - Filter valueLtEq = ValueFilter.ltEq(101L); + Filter valueLtEq = ValueFilter.ltEq(11L); Assert.assertTrue(valueLtEq.satisfy(statistics1)); - Assert.assertTrue(valueLtEq.satisfy(statistics2)); + Assert.assertFalse(valueLtEq.satisfy(statistics2)); + Assert.assertFalse(valueLtEq.allSatisfy(statistics1)); + Assert.assertFalse(valueLtEq.allSatisfy(statistics2)); } @Test @@ -105,4 +152,54 @@ public class StatisticsFilterTest { Assert.assertTrue(orFilter.satisfy(statistics1)); Assert.assertTrue(orFilter.satisfy(statistics2)); } + + @Test + public void testNot() { + Filter timeNotEq = FilterFactory.not(TimeFilter.eq(10L)); + try { + timeNotEq.satisfy(statistics1); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); + } + try { + timeNotEq.allSatisfy(statistics1); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); + } + + Filter valueNotEq = FilterFactory.not(ValueFilter.eq(101L)); + try { + valueNotEq.satisfy(statistics1); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); + } + try { + valueNotEq.allSatisfy(statistics1); + fail(); + } catch (Exception e) { + Assert.assertTrue(e.getMessage().contains(CONTAIN_NOT_ERR_MSG)); + } + } + + @Test + public void testBetweenAnd() { + Filter timeBetweenAnd = TimeFilter.between(0, 20); + Assert.assertTrue(timeBetweenAnd.satisfy(statistics1)); + Assert.assertFalse(timeBetweenAnd.satisfy(statistics2)); + Assert.assertTrue(timeBetweenAnd.satisfy(statistics3)); + Assert.assertFalse(timeBetweenAnd.allSatisfy(statistics1)); + Assert.assertFalse(timeBetweenAnd.allSatisfy(statistics2)); + Assert.assertTrue(timeBetweenAnd.allSatisfy(statistics3)); + + Filter timeNotBetweenAnd = TimeFilter.notBetween(0, 20); + Assert.assertTrue(timeNotBetweenAnd.satisfy(statistics1)); + Assert.assertTrue(timeNotBetweenAnd.satisfy(statistics2)); + Assert.assertFalse(timeNotBetweenAnd.satisfy(statistics3)); + Assert.assertFalse(timeNotBetweenAnd.allSatisfy(statistics1)); + Assert.assertTrue(timeNotBetweenAnd.allSatisfy(statistics2)); + Assert.assertFalse(timeNotBetweenAnd.allSatisfy(statistics3)); + } }
