JackieTien97 commented on code in PR #11631:
URL: https://github.com/apache/iotdb/pull/11631#discussion_r1410286025
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java:
##########
@@ -106,44 +107,51 @@ private void doFilter(Object row, int rowIndex, BatchData
batchData) {
}
}
- private boolean pageSatisfy() {
+ private boolean pageCanSkip() {
Statistics<? extends Serializable> statistics = getStatistics();
- if (valueFilter == null || valueFilter.allSatisfy(statistics)) {
- // For aligned series, When we only query some measurements under an
aligned device, if any
- // values of these queried measurements has the same value count as the
time column, the
- // timestamp will be selected.
- // NOTE: if we change the query semantic in the future for aligned
series, we need to remove
- // this check here.
- long rowCount = getTimeStatistics().getCount();
- boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty();
- if (!canUse) {
- for (Statistics<? extends Serializable> vStatistics :
getValueStatisticsList()) {
- if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
- canUse = true;
- break;
- }
- }
- }
- if (!canUse) {
- return true;
- }
- // When the number of points in all value pages is the same as that in
the time page, it means
- // that there is no null value, and all timestamps will be selected.
- if (paginationController.hasCurOffset(rowCount)) {
- paginationController.consumeOffset(rowCount);
+ if (valueFilter != null && !valueFilter.allSatisfy(statistics)) {
+ return valueFilter.canSkip(statistics);
+ }
+
+ if (!canSkipOffsetByStatistics()) {
+ return false;
+ }
+
+ long rowCount = getTimeStatistics().getCount();
+ if (paginationController.hasCurOffset(rowCount)) {
+ paginationController.consumeOffset(rowCount);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private boolean canSkipOffsetByStatistics() {
+ if (queryAllSensors || getValueStatisticsList().isEmpty()) {
+ return true;
+ }
+
+ // For aligned series, When we only query some measurements under an
aligned device, if any
+ // values of these queried measurements has the same value count as the
time column, the
+ // timestamp will be selected.
+ // NOTE: if we change the query semantic in the future for aligned series,
we need to remove
+ // this check here.
+ long rowCount = getTimeStatistics().getCount();
+ for (Statistics<? extends Serializable> vStatistics :
getValueStatisticsList()) {
+ if (vStatistics != null && vStatistics.hasNullValue(rowCount)) {
return false;
Review Comment:
```suggestion
if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
return true;
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java:
##########
@@ -64,21 +66,27 @@ public interface Filter {
*/
boolean satisfy(long time, Object value);
Review Comment:
```suggestion
```
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java:
##########
Review Comment:
Should this class implement `IStatisticsProvider`?
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.file.metadata;
+
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+
+import java.io.Serializable;
+
+public interface IStatisticsProvider {
+
+ Statistics<? extends Serializable> getTimeStatistics();
+
+ Statistics<? extends Serializable> getMeasurementStatistics(int
measurementIndex);
Review Comment:
```suggestion
Optional<Statistics<? extends Serializable>> getMeasurementStatistics(int
measurementIndex);
```
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java:
##########
@@ -106,44 +107,51 @@ private void doFilter(Object row, int rowIndex, BatchData
batchData) {
}
}
- private boolean pageSatisfy() {
+ private boolean pageCanSkip() {
Statistics<? extends Serializable> statistics = getStatistics();
- if (valueFilter == null || valueFilter.allSatisfy(statistics)) {
- // For aligned series, When we only query some measurements under an
aligned device, if any
- // values of these queried measurements has the same value count as the
time column, the
- // timestamp will be selected.
- // NOTE: if we change the query semantic in the future for aligned
series, we need to remove
- // this check here.
- long rowCount = getTimeStatistics().getCount();
- boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty();
- if (!canUse) {
- for (Statistics<? extends Serializable> vStatistics :
getValueStatisticsList()) {
- if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
- canUse = true;
- break;
- }
- }
- }
- if (!canUse) {
- return true;
- }
- // When the number of points in all value pages is the same as that in
the time page, it means
- // that there is no null value, and all timestamps will be selected.
- if (paginationController.hasCurOffset(rowCount)) {
- paginationController.consumeOffset(rowCount);
+ if (valueFilter != null && !valueFilter.allSatisfy(statistics)) {
+ return valueFilter.canSkip(statistics);
+ }
+
+ if (!canSkipOffsetByStatistics()) {
+ return false;
+ }
+
+ long rowCount = getTimeStatistics().getCount();
+ if (paginationController.hasCurOffset(rowCount)) {
+ paginationController.consumeOffset(rowCount);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private boolean canSkipOffsetByStatistics() {
+ if (queryAllSensors || getValueStatisticsList().isEmpty()) {
+ return true;
+ }
+
+ // For aligned series, When we only query some measurements under an
aligned device, if any
+ // values of these queried measurements has the same value count as the
time column, the
+ // timestamp will be selected.
+ // NOTE: if we change the query semantic in the future for aligned series,
we need to remove
+ // this check here.
+ long rowCount = getTimeStatistics().getCount();
+ for (Statistics<? extends Serializable> vStatistics :
getValueStatisticsList()) {
+ if (vStatistics != null && vStatistics.hasNullValue(rowCount)) {
return false;
- } else {
- return true;
}
- } else {
- return valueFilter.satisfy(statistics);
}
+
+ // When the number of points in all value pages is the same as that in the
time page, it means
+ // that there is no null value, and all timestamps will be selected.
+ return true;
Review Comment:
```suggestion
return false;
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java:
##########
@@ -64,21 +66,27 @@ public interface Filter {
*/
boolean satisfy(long time, Object value);
+ boolean satisfyRow(long time, Object[] values);
+
/**
* To examine whether there are data points satisfied with the filter.
*
* @param statistics statistics with min time, max time, min value, max
value.
* @return false if there are no data points satisfied with the filter
*/
- boolean satisfy(Statistics statistics);
+ boolean canSkip(Statistics<? extends Serializable> statistics);
Review Comment:
```suggestion
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java:
##########
@@ -50,28 +52,37 @@ private ValueFilterOperators() {
// forbidden construction
}
- private static final String MEASUREMENT_CANNOT_BE_NULL_MSG = "measurement
cannot be null";
private static final String CONSTANT_CANNOT_BE_NULL_MSG = "constant cannot
be null";
+ private static final boolean BLOCK_MIGHT_MATCH = false;
+ private static final boolean BLOCK_CANNOT_MATCH = true;
+ private static final boolean BLOCK_ALL_MATCH = true;
+
+ private static final String OPERATOR_TO_STRING_FORMAT = "measurements[%s] %s
%s";
+
+ private static final String CANNOT_PUSH_DOWN_MSG =
+ " operator can not be pushed down for non-aligned timeseries";
Review Comment:
why limited to `non-aligned timeseries`
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java:
##########
@@ -83,26 +94,35 @@ public boolean equals(Object o) {
if (o == null || getClass() != o.getClass()) {
return false;
}
+ if (!super.equals(o)) {
+ return false;
+ }
ValueColumnCompareFilter<?> that = (ValueColumnCompareFilter<?>) o;
- return measurement.equals(that.measurement);
+ return measurementIndex == that.measurementIndex;
}
@Override
public int hashCode() {
- return Objects.hash(measurement);
+ return Objects.hash(super.hashCode(), measurementIndex);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ OPERATOR_TO_STRING_FORMAT, measurementIndex,
getOperatorType().getSymbol(), constant);
}
}
public static final class ValueEq<T extends Comparable<T>> extends
ValueColumnCompareFilter<T> {
- // constant can be null
- // TODO: consider support IS NULL
- public ValueEq(String measurement, T constant) {
- super(measurement, constant);
+ // constant cannot be null
+ public ValueEq(int measurementIndex, T constant) {
+ super(measurementIndex, Objects.requireNonNull(constant,
CONSTANT_CANNOT_BE_NULL_MSG));
}
+ @SuppressWarnings("unchecked")
public ValueEq(ByteBuffer buffer) {
- this(ReadWriteIOUtils.readString(buffer), (T)
ReadWriteIOUtils.readObject(buffer));
+ this(ReadWriteIOUtils.readInt(buffer), (T)
ReadWriteIOUtils.readObject(buffer));
Review Comment:
never do this, you should write these two line by line, because the
execution order cannot be guaranteed.
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java:
##########
@@ -126,39 +127,45 @@ public BatchData getAllSatisfiedPageData(boolean
ascending) throws IOException {
return pageData.flip();
}
- private boolean pageSatisfy() {
- Statistics statistics = getStatistics();
- if (filter == null || filter.allSatisfy(statistics)) {
- // For aligned series, When we only query some measurements under an
aligned device, if any
- // values of these queried measurements has the same value count as the
time column, the
- // timestamp will be selected.
- // NOTE: if we change the query semantic in the future for aligned
series, we need to remove
- // this check here.
- long rowCount = getTimeStatistics().getCount();
- boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty();
- if (!canUse) {
- for (Statistics vStatistics : getValueStatisticsList()) {
- if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
- canUse = true;
- break;
- }
- }
- }
- if (!canUse) {
- return true;
- }
- // When the number of points in all value pages is the same as that in
the time page, it means
- // that there is no null value, and all timestamps will be selected.
- if (paginationController.hasCurOffset(rowCount)) {
- paginationController.consumeOffset(rowCount);
+ private boolean pageCanSkip() {
+ Statistics<? extends Serializable> statistics = getStatistics();
+ if (filter != null && !filter.allSatisfy(statistics)) {
+ return filter.canSkip(statistics);
+ }
+
+ if (!canSkipOffsetByStatistics()) {
+ return false;
+ }
+
+ long rowCount = getTimeStatistics().getCount();
+ if (paginationController.hasCurOffset(rowCount)) {
+ paginationController.consumeOffset(rowCount);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private boolean canSkipOffsetByStatistics() {
+ if (queryAllSensors || getValueStatisticsList().isEmpty()) {
+ return true;
+ }
+
+ // For aligned series, When we only query some measurements under an
aligned device, if any
+ // values of these queried measurements has the same value count as the
time column, the
+ // timestamp will be selected.
+ // NOTE: if we change the query semantic in the future for aligned series,
we need to remove
+ // this check here.
+ long rowCount = getTimeStatistics().getCount();
+ for (Statistics<? extends Serializable> statistics :
getValueStatisticsList()) {
+ if (statistics != null && statistics.hasNullValue(rowCount)) {
return false;
Review Comment:
```suggestion
if (statistics != null && !statistics.hasNullValue(rowCount)) {
return true;
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java:
##########
@@ -163,250 +183,431 @@ public boolean satisfy(long time, Object value) {
}
@Override
- public boolean satisfy(Statistics statistics) {
- if (statistics.getType() == TSDataType.TEXT || statistics.getType() ==
TSDataType.BOOLEAN) {
- return true;
+ @SuppressWarnings("unchecked")
+ public boolean canSkip(Statistics<? extends Serializable> statistics) {
+ if (statisticsNotAvailable(statistics)) {
+ return BLOCK_MIGHT_MATCH;
}
- return !(constant.compareTo((T) statistics.getMinValue()) == 0
- && constant.compareTo((T) statistics.getMaxValue()) == 0);
+
+ // drop if this is a column where min = max = value
+ return constant.compareTo((T) statistics.getMinValue()) == 0
+ && constant.compareTo((T) statistics.getMaxValue()) == 0;
}
@Override
- public boolean allSatisfy(Statistics statistics) {
- if (statistics.getType() == TSDataType.TEXT || statistics.getType() ==
TSDataType.BOOLEAN) {
- return false;
+ @SuppressWarnings("unchecked")
+ public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
+ if (statisticsNotAvailable(statistics)) {
+ return BLOCK_MIGHT_MATCH;
}
+
return constant.compareTo((T) statistics.getMinValue()) < 0
|| constant.compareTo((T) statistics.getMaxValue()) > 0;
}
@Override
public Filter reverse() {
- return new ValueEq<>(measurement, constant);
+ return new ValueEq<>(measurementIndex, constant);
}
@Override
public OperatorType getOperatorType() {
return OperatorType.VALUE_NEQ;
}
+ }
+
+ public static final class ValueIsNull<T extends Comparable<T>>
+ extends ValueColumnCompareFilter<T> {
+
+ // constant can be null
+ public ValueIsNull(int measurementIndex) {
+ super(measurementIndex, null);
+ }
+
+ public ValueIsNull(ByteBuffer buffer) {
+ this(ReadWriteIOUtils.readInt(buffer));
+ }
@Override
- public String toString() {
- return measurement + " != " + constant;
+ public boolean satisfy(long time, Object value) {
+ throw new IllegalArgumentException(getOperatorType().getSymbol() +
CANNOT_PUSH_DOWN_MSG);
+ }
+
+ @Override
+ public boolean satisfyRow(long time, Object[] values) {
+ return values[measurementIndex] == null;
+ }
+
+ @Override
+ public boolean canSkip(Statistics<? extends Serializable> statistics) {
+ throw new IllegalArgumentException(getOperatorType().getSymbol() +
CANNOT_PUSH_DOWN_MSG);
+ }
+
+ @Override
+ public boolean canSkip(IStatisticsProvider statisticsProvider) {
+ Statistics<? extends Serializable> statistics =
+ statisticsProvider.getMeasurementStatistics(measurementIndex);
+
+ if (statistics == null) {
+ // the measurement isn't in this block so all values are null.
+ return BLOCK_MIGHT_MATCH;
+ }
+
+ if (statisticsNotAvailable(statistics)) {
+ return BLOCK_MIGHT_MATCH;
+ }
Review Comment:
Why do we need to do this check here?
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.file.metadata;
+
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+
+import java.io.Serializable;
+
+public interface IStatisticsProvider {
+
+ Statistics<? extends Serializable> getTimeStatistics();
+
+ Statistics<? extends Serializable> getMeasurementStatistics(int
measurementIndex);
Review Comment:
use optional to remind caller that the return value may be null.
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java:
##########
@@ -83,26 +94,35 @@ public boolean equals(Object o) {
if (o == null || getClass() != o.getClass()) {
return false;
}
+ if (!super.equals(o)) {
+ return false;
+ }
ValueColumnCompareFilter<?> that = (ValueColumnCompareFilter<?>) o;
- return measurement.equals(that.measurement);
+ return measurementIndex == that.measurementIndex;
}
@Override
public int hashCode() {
- return Objects.hash(measurement);
+ return Objects.hash(super.hashCode(), measurementIndex);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ OPERATOR_TO_STRING_FORMAT, measurementIndex,
getOperatorType().getSymbol(), constant);
}
}
public static final class ValueEq<T extends Comparable<T>> extends
ValueColumnCompareFilter<T> {
- // constant can be null
- // TODO: consider support IS NULL
- public ValueEq(String measurement, T constant) {
- super(measurement, constant);
+ // constant cannot be null
+ public ValueEq(int measurementIndex, T constant) {
+ super(measurementIndex, Objects.requireNonNull(constant,
CONSTANT_CANNOT_BE_NULL_MSG));
}
+ @SuppressWarnings("unchecked")
public ValueEq(ByteBuffer buffer) {
- this(ReadWriteIOUtils.readString(buffer), (T)
ReadWriteIOUtils.readObject(buffer));
+ this(ReadWriteIOUtils.readInt(buffer), (T)
ReadWriteIOUtils.readObject(buffer));
Review Comment:
and check all other places.
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.file.metadata;
+
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+
+import java.io.Serializable;
+
+public interface IStatisticsProvider {
+
+ Statistics<? extends Serializable> getTimeStatistics();
+
+ Statistics<? extends Serializable> getMeasurementStatistics(int
measurementIndex);
+
+ default boolean hasNullValue(int measurementIndex) {
+ long rowCount = getTimeStatistics().getCount();
+ return getMeasurementStatistics(measurementIndex).hasNullValue(rowCount);
+ }
+
+ default boolean isAllNulls(int measurementIndex) {
Review Comment:
make it as an interface function instead of default method.
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java:
##########
@@ -64,21 +66,27 @@ public interface Filter {
*/
boolean satisfy(long time, Object value);
+ boolean satisfyRow(long time, Object[] values);
+
/**
* To examine whether there are data points satisfied with the filter.
*
* @param statistics statistics with min time, max time, min value, max
value.
* @return false if there are no data points satisfied with the filter
*/
- boolean satisfy(Statistics statistics);
+ boolean canSkip(Statistics<? extends Serializable> statistics);
+
+ boolean canSkip(IStatisticsProvider statisticsProvider);
/**
* To examine whether all data points are satisfied with the filter.
*
* @param statistics statistics with min time, max time, min value, max
value.
* @return true if all data points are satisfied with the filter
*/
- boolean allSatisfy(Statistics statistics);
+ boolean allSatisfy(Statistics<? extends Serializable> statistics);
Review Comment:
```suggestion
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java:
##########
Review Comment:
Why `Not` can't use `Statistics`?
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java:
##########
@@ -163,250 +183,431 @@ public boolean satisfy(long time, Object value) {
}
@Override
- public boolean satisfy(Statistics statistics) {
- if (statistics.getType() == TSDataType.TEXT || statistics.getType() ==
TSDataType.BOOLEAN) {
- return true;
+ @SuppressWarnings("unchecked")
+ public boolean canSkip(Statistics<? extends Serializable> statistics) {
+ if (statisticsNotAvailable(statistics)) {
+ return BLOCK_MIGHT_MATCH;
}
- return !(constant.compareTo((T) statistics.getMinValue()) == 0
- && constant.compareTo((T) statistics.getMaxValue()) == 0);
+
+ // drop if this is a column where min = max = value
+ return constant.compareTo((T) statistics.getMinValue()) == 0
+ && constant.compareTo((T) statistics.getMaxValue()) == 0;
}
@Override
- public boolean allSatisfy(Statistics statistics) {
- if (statistics.getType() == TSDataType.TEXT || statistics.getType() ==
TSDataType.BOOLEAN) {
- return false;
+ @SuppressWarnings("unchecked")
+ public boolean allSatisfy(Statistics<? extends Serializable> statistics) {
+ if (statisticsNotAvailable(statistics)) {
+ return BLOCK_MIGHT_MATCH;
}
+
return constant.compareTo((T) statistics.getMinValue()) < 0
|| constant.compareTo((T) statistics.getMaxValue()) > 0;
}
@Override
public Filter reverse() {
- return new ValueEq<>(measurement, constant);
+ return new ValueEq<>(measurementIndex, constant);
}
@Override
public OperatorType getOperatorType() {
return OperatorType.VALUE_NEQ;
}
+ }
+
+ public static final class ValueIsNull<T extends Comparable<T>>
+ extends ValueColumnCompareFilter<T> {
+
+ // constant can be null
+ public ValueIsNull(int measurementIndex) {
+ super(measurementIndex, null);
+ }
+
+ public ValueIsNull(ByteBuffer buffer) {
+ this(ReadWriteIOUtils.readInt(buffer));
+ }
@Override
- public String toString() {
- return measurement + " != " + constant;
+ public boolean satisfy(long time, Object value) {
+ throw new IllegalArgumentException(getOperatorType().getSymbol() +
CANNOT_PUSH_DOWN_MSG);
+ }
+
+ @Override
+ public boolean satisfyRow(long time, Object[] values) {
+ return values[measurementIndex] == null;
+ }
+
+ @Override
+ public boolean canSkip(Statistics<? extends Serializable> statistics) {
+ throw new IllegalArgumentException(getOperatorType().getSymbol() +
CANNOT_PUSH_DOWN_MSG);
+ }
+
+ @Override
+ public boolean canSkip(IStatisticsProvider statisticsProvider) {
+ Statistics<? extends Serializable> statistics =
+ statisticsProvider.getMeasurementStatistics(measurementIndex);
+
+ if (statistics == null) {
+ // the measurement isn't in this block so all values are null.
+ return BLOCK_MIGHT_MATCH;
Review Comment:
```suggestion
return BLOCK_ALL_MATCH;
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/OperatorType.java:
##########
@@ -21,41 +21,55 @@
public enum OperatorType {
// comparison
- TIME_EQ,
- VALUE_EQ,
- TIME_NEQ,
- VALUE_NEQ,
- TIME_GT,
- VALUE_GT,
- TIME_GTEQ,
- VALUE_GTEQ,
- TIME_LT,
- VALUE_LT,
- TIME_LTEQ,
- VALUE_LTEQ,
+ TIME_EQ("=="),
+ VALUE_EQ("=="),
Review Comment:
```suggestion
TIME_EQ("="),
VALUE_EQ("="),
```
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java:
##########
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.file.metadata;
+
+import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
+
+import java.io.Serializable;
+
+public interface IStatisticsProvider {
+
+ Statistics<? extends Serializable> getTimeStatistics();
+
+ Statistics<? extends Serializable> getMeasurementStatistics(int
measurementIndex);
+
+ default boolean hasNullValue(int measurementIndex) {
Review Comment:
for non-aligned, they don't need to do this, they just need to `return
false`.
##########
iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java:
##########
@@ -126,39 +127,45 @@ public BatchData getAllSatisfiedPageData(boolean
ascending) throws IOException {
return pageData.flip();
}
- private boolean pageSatisfy() {
- Statistics statistics = getStatistics();
- if (filter == null || filter.allSatisfy(statistics)) {
- // For aligned series, When we only query some measurements under an
aligned device, if any
- // values of these queried measurements has the same value count as the
time column, the
- // timestamp will be selected.
- // NOTE: if we change the query semantic in the future for aligned
series, we need to remove
- // this check here.
- long rowCount = getTimeStatistics().getCount();
- boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty();
- if (!canUse) {
- for (Statistics vStatistics : getValueStatisticsList()) {
- if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
- canUse = true;
- break;
- }
- }
- }
- if (!canUse) {
- return true;
- }
- // When the number of points in all value pages is the same as that in
the time page, it means
- // that there is no null value, and all timestamps will be selected.
- if (paginationController.hasCurOffset(rowCount)) {
- paginationController.consumeOffset(rowCount);
+ private boolean pageCanSkip() {
+ Statistics<? extends Serializable> statistics = getStatistics();
+ if (filter != null && !filter.allSatisfy(statistics)) {
+ return filter.canSkip(statistics);
+ }
+
+ if (!canSkipOffsetByStatistics()) {
+ return false;
+ }
+
+ long rowCount = getTimeStatistics().getCount();
+ if (paginationController.hasCurOffset(rowCount)) {
+ paginationController.consumeOffset(rowCount);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private boolean canSkipOffsetByStatistics() {
+ if (queryAllSensors || getValueStatisticsList().isEmpty()) {
+ return true;
+ }
+
+ // For aligned series, When we only query some measurements under an
aligned device, if any
+ // values of these queried measurements has the same value count as the
time column, the
+ // timestamp will be selected.
+ // NOTE: if we change the query semantic in the future for aligned series,
we need to remove
+ // this check here.
+ long rowCount = getTimeStatistics().getCount();
+ for (Statistics<? extends Serializable> statistics :
getValueStatisticsList()) {
+ if (statistics != null && statistics.hasNullValue(rowCount)) {
return false;
- } else {
- return true;
}
- } else {
- // TODO accept valueStatisticsList to filter
- return filter.satisfy(statistics);
}
+
+ // When the number of points in all value pages is the same as that in the
time page, it means
+ // that there is no null value, and all timestamps will be selected.
+ return true;
Review Comment:
```suggestion
return false;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
