This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/extendFilter in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c995ff624e5acd769257dc90314fc3f97fb8f46a Author: Minghui Liu <[email protected]> AuthorDate: Mon Dec 4 22:15:09 2023 +0800 tmp save --- .../operator/source/AlignedSeriesScanUtil.java | 38 ++++++++------------- .../execution/operator/source/SeriesScanUtil.java | 12 +++---- .../read/reader/chunk/MemAlignedPageReader.java | 33 ++++++------------ .../read/reader/chunk/MemPageReader.java | 17 ++++++++++ .../reader/chunk/MemAlignedChunkLoaderTest.java | 8 ++--- .../tsfile/file/metadata/AlignedChunkMetadata.java | 29 +++++++--------- .../file/metadata/AlignedTimeSeriesMetadata.java | 27 ++++++--------- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 22 ++++++++++++ .../iotdb/tsfile/file/metadata/IChunkMetadata.java | 4 +-- .../{IStatisticsProvider.java => IMetadata.java} | 26 ++++++++++++--- .../tsfile/file/metadata/IStatisticsProvider.java | 18 ++++++++-- .../tsfile/file/metadata/ITimeSeriesMetadata.java | 4 +-- .../tsfile/file/metadata/TimeseriesMetadata.java | 19 +++++++++++ .../tsfile/read/common/block/TsBlockBuilder.java | 7 +--- .../iotdb/tsfile/read/filter/basic/Filter.java | 6 ++-- .../tsfile/read/filter/basic/ITimeFilter.java | 10 +++--- .../tsfile/read/filter/basic/IValueFilter.java | 10 +++--- .../iotdb/tsfile/read/filter/operator/And.java | 10 +++--- .../iotdb/tsfile/read/filter/operator/Not.java | 6 ++-- .../iotdb/tsfile/read/filter/operator/Or.java | 10 +++--- .../read/filter/operator/ValueFilterOperators.java | 39 +++++++++++----------- .../iotdb/tsfile/read/reader/IPageReader.java | 5 ++- .../tsfile/read/reader/page/AlignedPageReader.java | 38 ++++++++------------- .../iotdb/tsfile/read/reader/page/PageReader.java | 17 ++++++++++ .../Preconditions.java} | 20 ++++++++--- .../tsfile/read/filter/StatisticsFilterTest.java | 6 ++-- 26 files changed, 251 insertions(+), 190 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java index b4b8fc11fe4..686ee690c1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java @@ -85,7 +85,9 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { @SuppressWarnings("squid:S3740") @Override protected Statistics currentFileStatistics(int index) { - return ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).getStatistics(index); + return ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata) + .getMeasurementStatistics(index) + .orElse(null); } @SuppressWarnings("squid:S3740") @@ -97,7 +99,7 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { @SuppressWarnings("squid:S3740") @Override protected Statistics currentChunkStatistics(int index) { - return ((AlignedChunkMetadata) firstChunkMetadata).getStatistics(index); + return ((AlignedChunkMetadata) firstChunkMetadata).getMeasurementStatistics(index).orElse(null); } @SuppressWarnings("squid:S3740") @@ -178,28 +180,21 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { // timestamp will be selected. // NOTE: if we change the query semantic in the future for aligned series, we need to remove // this check here. - long rowCount = - ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).getTimeStatistics().getCount(); boolean canUse = queryAllSensors - || ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata) - .getValueStatisticsList() - .isEmpty(); - if (!canUse) { - for (Statistics statistics : - ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).getValueStatisticsList()) { - if (statistics != null && !statistics.hasNullValue(rowCount)) { - canUse = true; - break; - } - } + || ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).getMeasurementCount() == 0; + if (!canUse && (((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).timeAllSelected())) { + canUse = true; } if (!canUse) { return; } + // When the number of points in all value chunk groups is the same as that in the time chunk // group, it means that there is no null value, and all timestamps will be selected. + long rowCount = + ((AlignedTimeSeriesMetadata) firstTimeSeriesMetadata).getTimeStatistics().getCount(); if (paginationController.hasCurOffset(rowCount)) { skipCurrentFile(); paginationController.consumeOffset(rowCount); @@ -228,16 +223,9 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil { // this check here. long rowCount = firstChunkMetadata.getStatistics().getCount(); boolean canUse = - queryAllSensors - || ((AlignedChunkMetadata) firstChunkMetadata).getValueStatisticsList().isEmpty(); - if (!canUse) { - for (Statistics statistics : - ((AlignedChunkMetadata) firstChunkMetadata).getValueStatisticsList()) { - if (statistics != null && !statistics.hasNullValue(rowCount)) { - canUse = true; - break; - } - } + queryAllSensors || ((AlignedChunkMetadata) firstChunkMetadata).getMeasurementCount() == 0; + if (!canUse && (((AlignedChunkMetadata) firstChunkMetadata).timeAllSelected())) { + canUse = true; } if (!canUse) { return; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 15806de7fbd..3aa0d67a02a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -42,9 +42,9 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlock; import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder; import org.apache.iotdb.tsfile.read.filter.basic.Filter; -import org.apache.iotdb.tsfile.read.reader.IAlignedPageReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; import org.apache.iotdb.tsfile.read.reader.IPointReader; +import org.apache.iotdb.tsfile.read.reader.page.AlignedPageReader; import org.apache.iotdb.tsfile.read.reader.series.PaginationController; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -1142,7 +1142,7 @@ public class SeriesScanUtil { this.version = new PriorityMergeReader.MergeReaderPriority(version, offset); this.data = data; this.isSeq = isSeq; - this.isAligned = data instanceof IAlignedPageReader; + this.isAligned = data instanceof AlignedPageReader || data instanceof MemAlignedPageReader; this.isMem = data instanceof MemPageReader || data instanceof MemAlignedPageReader; } @@ -1153,18 +1153,18 @@ public class SeriesScanUtil { @SuppressWarnings("squid:S3740") Statistics getStatistics(int index) throws IOException { - if (!(data instanceof IAlignedPageReader)) { + if (!isAligned) { throw new IOException("Can only get statistics by index from AlignedPageReader"); } - return ((IAlignedPageReader) data).getStatistics(index); + return data.getMeasurementStatistics(index).orElse(null); } @SuppressWarnings("squid:S3740") Statistics getTimeStatistics() throws IOException { - if (!(data instanceof IAlignedPageReader)) { + if (!isAligned) { throw new IOException("Can only get statistics of time column from AlignedPageReader"); } - return ((IAlignedPageReader) data).getTimeStatistics(); + return data.getTimeStatistics(); } TsBlock getAllSatisfiedPageData(boolean ascending) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 00e6af75fe2..1541cb31604 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -20,7 +20,6 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk; import org.apache.iotdb.tsfile.file.metadata.AlignedChunkMetadata; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.BatchData; @@ -31,7 +30,6 @@ import org.apache.iotdb.tsfile.read.common.block.column.Column; import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; -import org.apache.iotdb.tsfile.read.reader.IAlignedPageReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; import org.apache.iotdb.tsfile.read.reader.series.PaginationController; import org.apache.iotdb.tsfile.utils.TsPrimitiveType; @@ -39,10 +37,11 @@ import org.apache.iotdb.tsfile.utils.TsPrimitiveType; import java.io.IOException; import java.io.Serializable; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.tsfile.read.reader.series.PaginationController.UNLIMITED_PAGINATION_CONTROLLER; -public class MemAlignedPageReader implements IPageReader, IAlignedPageReader, IStatisticsProvider { +public class MemAlignedPageReader implements IPageReader { private final TsBlock tsBlock; private final AlignedChunkMetadata chunkMetadata; @@ -126,23 +125,14 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader, IS } private boolean canSkipOffsetByStatistics() { - if (queryAllSensors || getValueStatisticsList().isEmpty()) { + if (queryAllSensors || getMeasurementCount() == 0) { return true; } // For aligned series, we can use statistics to skip OFFSET only when all times are selected. // NOTE: if we change the query semantic in the future for aligned series, we need to remove // this check here. - long rowCount = getTimeStatistics().getCount(); - for (Statistics<? extends Serializable> statistics : getValueStatisticsList()) { - if (statistics != null && !statistics.hasNullValue(rowCount)) { - // When there is any value page point number that is the same as the time page, - // it means that all timestamps in time page will be selected. - return true; - } - } - - return false; + return timeAllSelected(); } @Override @@ -241,23 +231,20 @@ public class MemAlignedPageReader implements IPageReader, IAlignedPageReader, IS return chunkMetadata.getStatistics(); } - @Override - public Statistics<? extends Serializable> getStatistics(int index) { - return chunkMetadata.getStatistics(index); - } - @Override public Statistics<? extends Serializable> getTimeStatistics() { return chunkMetadata.getTimeStatistics(); } @Override - public Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex) { - return getStatistics(measurementIndex); + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + return chunkMetadata.getMeasurementStatistics(measurementIndex); } - private List<Statistics<? extends Serializable>> getValueStatisticsList() { - return chunkMetadata.getValueStatisticsList(); + @Override + public int getMeasurementCount() { + return chunkMetadata.getMeasurementCount(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java index 2c7705bbc5d..bb600b03421 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.tsfile.read.reader.series.PaginationController.UNLIMITED_PAGINATION_CONTROLLER; @@ -351,4 +352,20 @@ public class MemPageReader implements IPageReader { public void initTsBlockBuilder(List<TSDataType> dataTypes) { // non-aligned page reader don't need to init TsBlockBuilder at the very beginning } + + @Override + public Statistics<? extends Serializable> getTimeStatistics() { + return getStatistics(); + } + + @Override + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + return Optional.ofNullable(getStatistics()); + } + + @Override + public int getMeasurementCount() { + return 1; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java index ecf7d22849c..b9f538b3439 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java @@ -83,11 +83,7 @@ public class MemAlignedChunkLoaderTest { Mockito.when(chunkMetadata1.getStatistics()).thenReturn(timeStatistics); Mockito.when(chunkMetadata1.getTimeStatistics()).thenReturn(timeStatistics); - Mockito.when(chunkMetadata1.getStatistics(0)).thenReturn(statistics1); - Mockito.when(chunkMetadata1.getValueStatisticsList()) - .thenReturn( - Arrays.asList( - statistics1, statistics2, statistics3, statistics4, statistics5, statistics6)); + Mockito.when(chunkMetadata1.getMeasurementStatistics(0).orElse(null)).thenReturn(statistics1); MemAlignedChunkReader chunkReader = (MemAlignedChunkReader) memAlignedChunkLoader.getChunkReader(chunkMetadata1, null); @@ -137,7 +133,7 @@ public class MemAlignedChunkLoaderTest { assertEquals(2L, tsBlock.getTimeColumn().getLong(1)); assertEquals(timeStatistics, pageReader.getStatistics()); - assertEquals(statistics1, pageReader.getStatistics(0)); + assertEquals(statistics1, pageReader.getMeasurementStatistics(0).orElse(null)); assertEquals(timeStatistics, pageReader.getTimeStatistics()); assertFalse(pageReader.isModified()); pageReader.setLimitOffset(UNLIMITED_PAGINATION_CONTROLLER); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java index e26af6e3a41..0a724ce785b 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedChunkMetadata.java @@ -28,8 +28,9 @@ import java.io.OutputStream; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Optional; -public class AlignedChunkMetadata implements IChunkMetadata, IStatisticsProvider { +public class AlignedChunkMetadata implements IChunkMetadata { // ChunkMetadata for time column private final IChunkMetadata timeChunkMetadata; @@ -46,33 +47,27 @@ public class AlignedChunkMetadata implements IChunkMetadata, IStatisticsProvider } @Override - public Statistics<? extends Serializable> getStatistics() { + public <T extends Serializable> Statistics<T> getStatistics() { return valueChunkMetadataList.size() == 1 && valueChunkMetadataList.get(0) != null ? valueChunkMetadataList.get(0).getStatistics() : timeChunkMetadata.getStatistics(); } - public Statistics<? extends Serializable> getStatistics(int index) { - IChunkMetadata v = valueChunkMetadataList.get(index); - return v == null ? null : v.getStatistics(); - } - - public List<Statistics<? extends Serializable>> getValueStatisticsList() { - List<Statistics<? extends Serializable>> valueStatisticsList = new ArrayList<>(); - for (IChunkMetadata v : valueChunkMetadataList) { - valueStatisticsList.add(v == null ? null : v.getStatistics()); - } - return valueStatisticsList; + @Override + public <T extends Serializable> Statistics<T> getTimeStatistics() { + return timeChunkMetadata.getStatistics(); } @Override - public Statistics<? extends Serializable> getTimeStatistics() { - return timeChunkMetadata.getStatistics(); + public <T extends Serializable> Optional< Statistics<T>> getMeasurementStatistics( + int measurementIndex) { + IChunkMetadata chunkMetadata = valueChunkMetadataList.get(measurementIndex); + return Optional.ofNullable(chunkMetadata == null ? null : chunkMetadata.getStatistics()); } @Override - public Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex) { - return getStatistics(measurementIndex); + public int getMeasurementCount() { + return valueChunkMetadataList.size(); } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java index ea648910d3a..33e33ab05b1 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/AlignedTimeSeriesMetadata.java @@ -26,8 +26,9 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; -public class AlignedTimeSeriesMetadata implements ITimeSeriesMetadata, IStatisticsProvider { +public class AlignedTimeSeriesMetadata implements ITimeSeriesMetadata { // TimeSeriesMetadata for time column private final TimeseriesMetadata timeseriesMetadata; @@ -53,27 +54,21 @@ public class AlignedTimeSeriesMetadata implements ITimeSeriesMetadata, IStatisti : timeseriesMetadata.getStatistics(); } - public Statistics<? extends Serializable> getStatistics(int index) { - TimeseriesMetadata v = valueTimeseriesMetadataList.get(index); - return v == null ? null : v.getStatistics(); - } - - public List<Statistics> getValueStatisticsList() { - List<Statistics> valueStatisticsList = new ArrayList<>(); - for (TimeseriesMetadata v : valueTimeseriesMetadataList) { - valueStatisticsList.add(v == null ? null : v.getStatistics()); - } - return valueStatisticsList; - } - @Override public Statistics<? extends Serializable> getTimeStatistics() { return timeseriesMetadata.getStatistics(); } @Override - public Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex) { - return getStatistics(measurementIndex); + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + TimeseriesMetadata metadata = valueTimeseriesMetadataList.get(measurementIndex); + return Optional.ofNullable(metadata == null ? null : metadata.getStatistics()); + } + + @Override + public int getMeasurementCount() { + return valueTimeseriesMetadataList.size(); } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index 287b7f968ab..8d39da5aace 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -35,6 +35,9 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; + +import static org.apache.iotdb.tsfile.utils.Preconditions.checkArgument; /** Metadata of one chunk. */ public class ChunkMetadata implements IChunkMetadata { @@ -344,4 +347,23 @@ public class ChunkMetadata implements IChunkMetadata { isClosed, mask); } + + @Override + public Statistics<? extends Serializable> getTimeStatistics() { + return getStatistics(); + } + + @Override + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + checkArgument( + measurementIndex == 0, + "ChunkMetadata only has one measurement, but measurementIndex is %s"); + return Optional.ofNullable(statistics); + } + + @Override + public int getMeasurementCount() { + return 1; + } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java index 2dbedd1be1b..05621d2a940 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IChunkMetadata.java @@ -29,9 +29,7 @@ import java.io.OutputStream; import java.io.Serializable; import java.util.List; -public interface IChunkMetadata { - - Statistics<? extends Serializable> getStatistics(); +public interface IChunkMetadata extends IMetadata { boolean isModified(); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IMetadata.java similarity index 57% copy from iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java copy to iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IMetadata.java index d605b7b2487..c2184353ca1 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IMetadata.java @@ -22,19 +22,37 @@ package org.apache.iotdb.tsfile.file.metadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import java.io.Serializable; +import java.util.Optional; -public interface IStatisticsProvider { +public interface IMetadata { - Statistics<? extends Serializable> getTimeStatistics(); + <T extends Serializable> Statistics<T> getStatistics(); - Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex); + <T extends Serializable> Statistics<T> getTimeStatistics(); + + <T extends Serializable> Optional<Statistics<T>> getMeasurementStatistics(int measurementIndex); + + int getMeasurementCount(); default boolean hasNullValue(int measurementIndex) { long rowCount = getTimeStatistics().getCount(); - return getMeasurementStatistics(measurementIndex).hasNullValue(rowCount); + Optional<Statistics<Serializable>> statistics = + getMeasurementStatistics(measurementIndex); + return statistics.map(stat -> stat.hasNullValue(rowCount)).orElse(true); } default boolean isAllNulls(int measurementIndex) { return false; } + + default boolean timeAllSelected() { + for (int index = 0; index < getMeasurementCount(); index++) { + if (!hasNullValue(index)) { + // When there is any value page point number that is the same as the time page, + // it means that all timestamps in time page will be selected. + return true; + } + } + return false; + } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java index d605b7b2487..5dc304a733f 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/IStatisticsProvider.java @@ -22,19 +22,33 @@ package org.apache.iotdb.tsfile.file.metadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import java.io.Serializable; +import java.util.Optional; public interface IStatisticsProvider { Statistics<? extends Serializable> getTimeStatistics(); - Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex); + Optional<Statistics<? extends Serializable>> getMeasurementStatistics(int measurementIndex); + + int getMeasurementCount(); default boolean hasNullValue(int measurementIndex) { long rowCount = getTimeStatistics().getCount(); - return getMeasurementStatistics(measurementIndex).hasNullValue(rowCount); + Optional<Statistics<? extends Serializable>> statistics = + getMeasurementStatistics(measurementIndex); + return statistics.map(stat -> stat.hasNullValue(rowCount)).orElse(true); } default boolean isAllNulls(int measurementIndex) { return false; } + + default boolean timeAllSelected() { + for (int index = 0; index < getMeasurementCount(); index++) { + if (!hasNullValue(index)) { + return true; + } + } + return false; + } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java index c09f0e90b08..708318ad761 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ITimeSeriesMetadata.java @@ -24,9 +24,7 @@ import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader; import java.util.List; -public interface ITimeSeriesMetadata { - - Statistics getStatistics(); +public interface ITimeSeriesMetadata extends IMetadata { boolean isModified(); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java index b40868d4346..da279f4bf43 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java @@ -34,10 +34,12 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Set; import static io.airlift.slice.SizeOf.sizeOfCharArray; import static io.airlift.slice.SizeOf.sizeOfObjectArray; +import static org.apache.iotdb.tsfile.utils.Preconditions.checkArgument; public class TimeseriesMetadata implements ITimeSeriesMetadata { @@ -228,6 +230,23 @@ public class TimeseriesMetadata implements ITimeSeriesMetadata { this.statistics = statistics; } + @Override + public Statistics<? extends Serializable> getTimeStatistics() { + return getStatistics(); + } + + @Override + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + checkArgument(measurementIndex == 0); + return Optional.ofNullable(statistics); + } + + @Override + public int getMeasurementCount() { + return 1; + } + public void setChunkMetadataLoader(IChunkMetadataLoader chunkMetadataLoader) { this.chunkMetadataLoader = chunkMetadataLoader; } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java index bfe98d84c96..762fc47f4ed 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/TsBlockBuilder.java @@ -38,6 +38,7 @@ import java.util.List; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.tsfile.utils.Preconditions.checkArgument; public class TsBlockBuilder { @@ -326,10 +327,4 @@ public class TsBlockBuilder { getColumnBuilder(columnIndex).writeBinary(new Binary(value, TSFileConfig.STRING_CHARSET)); } } - - private static void checkArgument(boolean expression, String errorMessage) { - if (!expression) { - throw new IllegalArgumentException(errorMessage); - } - } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java index 04b43c70ce5..6fef37a3faf 100755 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/Filter.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.basic; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; @@ -76,7 +76,7 @@ public interface Filter { */ boolean canSkip(Statistics<? extends Serializable> statistics); - boolean canSkip(IStatisticsProvider statisticsProvider); + boolean canSkip(IMetadata metadata); /** * To examine whether all data points are satisfied with the filter. @@ -86,7 +86,7 @@ public interface Filter { */ boolean allSatisfy(Statistics<? extends Serializable> statistics); - boolean allSatisfy(IStatisticsProvider statisticsProvider); + boolean allSatisfy(IMetadata metadata); /** * To examine whether the min time and max time are satisfied with the filter. diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java index 455c690b186..2b58b889645 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.basic; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import java.io.Serializable; @@ -35,15 +35,15 @@ public interface ITimeFilter extends Filter { return !satisfyStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } - default boolean canSkip(IStatisticsProvider statisticsProvider) { - return canSkip(statisticsProvider.getTimeStatistics()); + default boolean canSkip(IMetadata metadata) { + return canSkip(metadata.getTimeStatistics()); } default boolean allSatisfy(Statistics<? extends Serializable> statistics) { return containStartEndTime(statistics.getStartTime(), statistics.getEndTime()); } - default boolean allSatisfy(IStatisticsProvider statisticsProvider) { - return allSatisfy(statisticsProvider.getTimeStatistics()); + default boolean allSatisfy(IMetadata metadata) { + return allSatisfy(metadata.getTimeStatistics()); } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java index 68c7370a051..7b646adfb48 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.basic; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.read.common.TimeRange; import java.util.List; @@ -37,12 +37,12 @@ public interface IValueFilter extends Filter { return satisfy(time, values[getMeasurementIndex()]); } - default boolean canSkip(IStatisticsProvider statisticsProvider) { - return canSkip(statisticsProvider.getMeasurementStatistics(getMeasurementIndex())); + default boolean canSkip(IMetadata metadata) { + return canSkip(metadata.getMeasurementStatistics(getMeasurementIndex())); } - default boolean allSatisfy(IStatisticsProvider statisticsProvider) { - return allSatisfy(statisticsProvider.getMeasurementStatistics(getMeasurementIndex())); + default boolean allSatisfy(IMetadata metadata) { + return allSatisfy(metadata.getMeasurementStatistics(getMeasurementIndex())); } default boolean satisfyStartEndTime(long startTime, long endTime) { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java index 07e215b8734..682df4e5787 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/And.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.operator; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -59,8 +59,8 @@ public class And extends BinaryLogicalFilter { } @Override - public boolean canSkip(IStatisticsProvider statisticsProvider) { - return left.canSkip(statisticsProvider) || right.canSkip(statisticsProvider); + public boolean canSkip(IMetadata metadata) { + return left.canSkip(metadata) || right.canSkip(metadata); } @Override @@ -69,8 +69,8 @@ public class And extends BinaryLogicalFilter { } @Override - public boolean allSatisfy(IStatisticsProvider statisticsProvider) { - return left.allSatisfy(statisticsProvider) && right.allSatisfy(statisticsProvider); + public boolean allSatisfy(IMetadata metadata) { + return left.allSatisfy(metadata) && right.allSatisfy(metadata); } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java index e2a66189c7d..711da50c21c 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Not.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.operator; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -66,7 +66,7 @@ public class Not implements Filter { } @Override - public boolean canSkip(IStatisticsProvider statisticsProvider) { + public boolean canSkip(IMetadata metadata) { throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } @@ -76,7 +76,7 @@ public class Not implements Filter { } @Override - public boolean allSatisfy(IStatisticsProvider statisticsProvider) { + public boolean allSatisfy(IMetadata metadata) { throw new IllegalArgumentException(CONTAIN_NOT_ERR_MSG + this); } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java index 9801b917806..4b2887cce37 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/Or.java @@ -19,7 +19,7 @@ package org.apache.iotdb.tsfile.read.filter.operator; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -59,8 +59,8 @@ public class Or extends BinaryLogicalFilter { } @Override - public boolean canSkip(IStatisticsProvider statisticsProvider) { - return left.canSkip(statisticsProvider) && right.canSkip(statisticsProvider); + public boolean canSkip(IMetadata metadata) { + return left.canSkip(metadata) && right.canSkip(metadata); } @Override @@ -69,8 +69,8 @@ public class Or extends BinaryLogicalFilter { } @Override - public boolean allSatisfy(IStatisticsProvider statisticsProvider) { - return left.allSatisfy(statisticsProvider) || right.allSatisfy(statisticsProvider); + public boolean allSatisfy(IMetadata metadata) { + return left.allSatisfy(metadata) || right.allSatisfy(metadata); } @Override diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java index 341aa44e1f9..07b35c73571 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/ValueFilterOperators.java @@ -20,7 +20,7 @@ package org.apache.iotdb.tsfile.read.filter.operator; import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.filter.basic.Filter; @@ -39,6 +39,7 @@ import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; @@ -244,22 +245,22 @@ public final class ValueFilterOperators { } @Override - public boolean canSkip(IStatisticsProvider statisticsProvider) { - Statistics<? extends Serializable> statistics = - statisticsProvider.getMeasurementStatistics(measurementIndex); + public boolean canSkip(IMetadata metadata) { + Optional<Statistics<? extends Serializable>> statistics = + metadata.getMeasurementStatistics(measurementIndex); - if (statistics == null) { + if (!statistics.isPresent()) { // the measurement isn't in this block so all values are null. return BLOCK_MIGHT_MATCH; } - if (statisticsNotAvailable(statistics)) { + if (statisticsNotAvailable(statistics.get())) { return BLOCK_MIGHT_MATCH; } // we are looking for records where v eq(null) // so drop if there are no nulls in this chunk - if (!statisticsProvider.hasNullValue(measurementIndex)) { + if (!metadata.hasNullValue(measurementIndex)) { return BLOCK_CANNOT_MATCH; } return BLOCK_MIGHT_MATCH; @@ -271,21 +272,21 @@ public final class ValueFilterOperators { } @Override - public boolean allSatisfy(IStatisticsProvider statisticsProvider) { - Statistics<? extends Serializable> statistics = - statisticsProvider.getMeasurementStatistics(measurementIndex); + public boolean allSatisfy(IMetadata metadata) { + Optional<Statistics<? extends Serializable>> statistics = + metadata.getMeasurementStatistics(measurementIndex); - if (statistics == null) { + if (statistics.isPresent()) { // the measurement isn't in this block so all values are null. // null is always equal to null return BLOCK_ALL_MATCH; } - if (statisticsNotAvailable(statistics)) { + if (statisticsNotAvailable(statistics.get())) { return BLOCK_MIGHT_MATCH; } - if (statisticsProvider.isAllNulls(measurementIndex)) { + if (metadata.isAllNulls(measurementIndex)) { return BLOCK_ALL_MATCH; } return BLOCK_MIGHT_MATCH; @@ -330,9 +331,9 @@ public final class ValueFilterOperators { } @Override - public boolean canSkip(IStatisticsProvider statisticsProvider) { + public boolean canSkip(IMetadata metadata) { Statistics<? extends Serializable> statistics = - statisticsProvider.getMeasurementStatistics(measurementIndex); + metadata.getMeasurementStatistics(measurementIndex); if (statistics == null) { // null is always equal to null @@ -345,7 +346,7 @@ public final class ValueFilterOperators { // we are looking for records where v notEq(null) // so, if this is a column of all nulls, we can drop it - if (statisticsProvider.isAllNulls(measurementIndex)) { + if (metadata.isAllNulls(measurementIndex)) { return BLOCK_CANNOT_MATCH; } return BLOCK_MIGHT_MATCH; @@ -357,9 +358,9 @@ public final class ValueFilterOperators { } @Override - public boolean allSatisfy(IStatisticsProvider statisticsProvider) { + public boolean allSatisfy(IMetadata metadata) { Statistics<? extends Serializable> statistics = - statisticsProvider.getMeasurementStatistics(measurementIndex); + metadata.getMeasurementStatistics(measurementIndex); if (statistics == null) { return BLOCK_MIGHT_MATCH; @@ -371,7 +372,7 @@ public final class ValueFilterOperators { // we are looking for records where v notEq(null) // so, if this is a column of all nulls, we can drop it - if (!statisticsProvider.hasNullValue(measurementIndex)) { + if (!metadata.hasNullValue(measurementIndex)) { return BLOCK_ALL_MATCH; } return BLOCK_MIGHT_MATCH; diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java index ac09b4cb9e6..1ceab56f0a2 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IPageReader.java @@ -19,6 +19,7 @@ package org.apache.iotdb.tsfile.read.reader; +import org.apache.iotdb.tsfile.file.metadata.IMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.BatchData; @@ -30,7 +31,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.List; -public interface IPageReader { +public interface IPageReader extends IMetadata { default BatchData getAllSatisfiedPageData() throws IOException { return getAllSatisfiedPageData(true); @@ -40,8 +41,6 @@ public interface IPageReader { TsBlock getAllSatisfiedData() throws IOException; - Statistics<? extends Serializable> getStatistics(); - void setFilter(Filter filter); boolean isModified(); diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java index 0e8ee616e7c..139f309c12b 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java @@ -21,7 +21,6 @@ package org.apache.iotdb.tsfile.read.reader.page; import org.apache.iotdb.tsfile.encoding.decoder.Decoder; import org.apache.iotdb.tsfile.file.header.PageHeader; -import org.apache.iotdb.tsfile.file.metadata.IStatisticsProvider; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.BatchData; @@ -31,7 +30,6 @@ import org.apache.iotdb.tsfile.read.common.block.TsBlock; import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; import org.apache.iotdb.tsfile.read.filter.basic.Filter; import org.apache.iotdb.tsfile.read.filter.factory.FilterFactory; -import org.apache.iotdb.tsfile.read.reader.IAlignedPageReader; import org.apache.iotdb.tsfile.read.reader.IPageReader; import org.apache.iotdb.tsfile.read.reader.IPointReader; import org.apache.iotdb.tsfile.read.reader.series.PaginationController; @@ -43,18 +41,18 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.tsfile.read.reader.series.PaginationController.UNLIMITED_PAGINATION_CONTROLLER; -public class AlignedPageReader implements IPageReader, IAlignedPageReader, IStatisticsProvider { +public class AlignedPageReader implements IPageReader { private final TimePageReader timePageReader; private final List<ValuePageReader> valuePageReaderList; private final int valueCount; // only used for limit and offset push down optimizer, if we select all columns from aligned - // device, we - // can use statistics to skip. + // device, we can use statistics to skip. // it's only exact while using limit & offset push down private final boolean queryAllSensors; @@ -130,7 +128,7 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader, IStat private boolean pageCanSkip() { Statistics<? extends Serializable> statistics = getStatistics(); if (filter != null && !filter.allSatisfy(statistics)) { - return filter.canSkip(statistics); + return filter.canSkip(this); } if (!canSkipOffsetByStatistics()) { @@ -147,23 +145,14 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader, IStat } private boolean canSkipOffsetByStatistics() { - if (queryAllSensors || getValueStatisticsList().isEmpty()) { + if (queryAllSensors || valueCount == 0) { return true; } // For aligned series, we can use statistics to skip OFFSET only when all times are selected. // NOTE: if we change the query semantic in the future for aligned series, we need to remove // this check here. - long rowCount = getTimeStatistics().getCount(); - for (Statistics<? extends Serializable> statistics : getValueStatisticsList()) { - if (statistics != null && !statistics.hasNullValue(rowCount)) { - // When there is any value page point number that is the same as the time page, - // it means that all timestamps in time page will be selected. - return true; - } - } - - return false; + return timeAllSelected(); } public IPointReader getLazyPointReader() throws IOException { @@ -386,19 +375,20 @@ public class AlignedPageReader implements IPageReader, IAlignedPageReader, IStat } @Override - public Statistics<? extends Serializable> getStatistics(int index) { - ValuePageReader valuePageReader = valuePageReaderList.get(index); - return valuePageReader == null ? null : valuePageReader.getStatistics(); + public Statistics<? extends Serializable> getTimeStatistics() { + return timePageReader.getStatistics(); } @Override - public Statistics<? extends Serializable> getTimeStatistics() { - return timePageReader.getStatistics(); + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + ValuePageReader valuePageReader = valuePageReaderList.get(measurementIndex); + return Optional.ofNullable(valuePageReader == null ? null : valuePageReader.getStatistics()); } @Override - public Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex) { - return getStatistics(measurementIndex); + public int getMeasurementCount() { + return valueCount; } private List<Statistics<? extends Serializable>> getValueStatisticsList() { diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java index fafbc4d9fbd..6dbb93a0d7e 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/PageReader.java @@ -43,6 +43,7 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.tsfile.read.reader.series.PaginationController.UNLIMITED_PAGINATION_CONTROLLER; @@ -380,4 +381,20 @@ public class PageReader implements IPageReader { } return false; } + + @Override + public Statistics<? extends Serializable> getTimeStatistics() { + return getStatistics(); + } + + @Override + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + return Optional.ofNullable(getStatistics()); + } + + @Override + public int getMeasurementCount() { + return 1; + } } diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IAlignedPageReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Preconditions.java similarity index 64% rename from iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IAlignedPageReader.java rename to iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Preconditions.java index 5b590e3b2a6..90443038be5 100644 --- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IAlignedPageReader.java +++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/Preconditions.java @@ -17,13 +17,23 @@ * under the License. */ -package org.apache.iotdb.tsfile.read.reader; +package org.apache.iotdb.tsfile.utils; -import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; +public class Preconditions { -public interface IAlignedPageReader { + private Preconditions() { + // private constructor for utility class + } - Statistics getStatistics(int index); + public static void checkArgument(boolean expression) { + if (!expression) { + throw new IllegalArgumentException(); + } + } - Statistics getTimeStatistics(); + public static void checkArgument(boolean expression, String errorMessage) { + if (!expression) { + throw new IllegalArgumentException(errorMessage); + } + } } diff --git a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java index 4449eeeab5c..57aeea789c7 100644 --- a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java +++ b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/StatisticsFilterTest.java @@ -34,6 +34,7 @@ import org.junit.Test; import java.io.Serializable; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.apache.iotdb.tsfile.read.filter.operator.Not.CONTAIN_NOT_ERR_MSG; import static org.junit.Assert.fail; @@ -277,8 +278,9 @@ public class StatisticsFilterTest { } @Override - public Statistics<? extends Serializable> getMeasurementStatistics(int measurementIndex) { - return statisticsList.get(measurementIndex); + public Optional<Statistics<? extends Serializable>> getMeasurementStatistics( + int measurementIndex) { + return Optional.ofNullable(statisticsList.get(measurementIndex)); } } }
