This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch addMoreCheckpointsInSeriesScan in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0d27ac0a6f61ac69475853b9e0545376f0f1b5e9 Author: shuwenwei <[email protected]> AuthorDate: Wed Jul 2 17:27:23 2025 +0800 fix ut --- .../execution/operator/source/SeriesScanUtil.java | 13 +- .../series/AbstractAlignedSeriesScanTest.java | 2 + .../AlignedSeriesScanLimitOffsetPushDownTest.java | 609 +++++++++--------- .../AlignedSeriesScanPredicatePushDownTest.java | 466 +++++++------- ...gleColumnSeriesScanLimitOffsetPushDownTest.java | 402 ++++++------ .../read/reader/series/SeriesReaderTestUtil.java | 17 + .../series/SeriesScanLimitOffsetPushDownTest.java | 500 +++++++-------- .../series/SeriesScanPredicatePushDownTest.java | 692 ++++++++++----------- 8 files changed, 1358 insertions(+), 1343 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 14b43814f88..b0c230f4e5a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -330,22 +330,19 @@ public class SeriesScanUtil implements Accountable { } Optional<Boolean> hasNextFileReturnValue = null; - while (firstChunkMetadata == null && hasNextFileReturnValue == null) { + while (firstChunkMetadata == null) { if (cachedChunkMetadata.isEmpty()) { + if (hasNextFileReturnValue != null) { + return Optional.empty(); + } hasNextFileReturnValue = hasNextFile(); if (!hasNextFileReturnValue.isPresent() || !hasNextFileReturnValue.get()) { - break; + return hasNextFileReturnValue; } } initFirstChunkMetadata(); - // filter chunk based on push-down conditions filterFirstChunkMetadata(); } - if (hasNextFileReturnValue != null - && !hasNextFileReturnValue.isPresent() - && firstChunkMetadata == null) { - return hasNextFileReturnValue; - } return Optional.of(firstChunkMetadata != null); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AbstractAlignedSeriesScanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AbstractAlignedSeriesScanTest.java index 74d5905efce..ba671128fb1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AbstractAlignedSeriesScanTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AbstractAlignedSeriesScanTest.java @@ -434,4 +434,6 @@ public abstract class AbstractAlignedSeriesScanTest { BloomFilterCache.getInstance().clear(); EnvironmentUtils.cleanAllDir(); } + + protected void assertWithOptionalReturnValue() {} } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java index d57139fb8cf..8dddda60c1e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanLimitOffsetPushDownTest.java @@ -1,304 +1,305 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -// package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; -// -// import org.apache.iotdb.commons.exception.IllegalPathException; -// import org.apache.iotdb.commons.path.AlignedFullPath; -// import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; -// import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; -// import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; -// import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; -// import org.apache.iotdb.db.utils.EnvironmentUtils; -// -// import org.apache.tsfile.enums.TSDataType; -// import org.apache.tsfile.read.common.block.TsBlock; -// import org.apache.tsfile.write.schema.MeasurementSchema; -// import org.junit.Assert; -// import org.junit.Test; -// -// import java.io.IOException; -// import java.util.Arrays; -// import java.util.HashSet; -// -// public class AlignedSeriesScanLimitOffsetPushDownTest extends AbstractAlignedSeriesScanTest { -// -// private AlignedSeriesScanUtil getAlignedSeriesScanUtil(long limit, long offset) -// throws IllegalPathException { -// AlignedFullPath scanPath = -// new AlignedFullPath( -// TEST_DEVICE, -// Arrays.asList("s1", "s2"), -// Arrays.asList( -// new MeasurementSchema("s1", TSDataType.INT32), -// new MeasurementSchema("s2", TSDataType.INT32))); -// -// SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); -// scanOptionsBuilder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); -// scanOptionsBuilder.withPushDownLimit(limit); -// scanOptionsBuilder.withPushDownOffset(offset); -// AlignedSeriesScanUtil seriesScanUtil = -// new AlignedSeriesScanUtil( -// scanPath, -// Ordering.ASC, -// scanOptionsBuilder.build(), -// EnvironmentUtils.TEST_QUERY_FI_CONTEXT); -// seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); -// return seriesScanUtil; -// } -// -// @Test -// public void testSkipFile() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 10); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 10; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testCannotSkipFile() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 20); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 20; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// -// @Test -// public void testSkipChunk() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 30); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 30; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// -// @Test -// public void testCannotSkipChunk() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 40); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 40; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// -// @Test -// public void testSkipPage() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 50); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 50; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// -// @Test -// public void testCannotSkipPage() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 60); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 60; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// -// @Test -// public void testSkipPoint() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(10, 75); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 75; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// expectedTime = 80; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// } +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; + +public class AlignedSeriesScanLimitOffsetPushDownTest extends AbstractAlignedSeriesScanTest { + + private AlignedSeriesScanUtil getAlignedSeriesScanUtil(long limit, long offset) + throws IllegalPathException { + AlignedFullPath scanPath = + new AlignedFullPath( + TEST_DEVICE, + Arrays.asList("s1", "s2"), + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32))); + + SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); + scanOptionsBuilder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + scanOptionsBuilder.withPushDownLimit(limit); + scanOptionsBuilder.withPushDownOffset(offset); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, + Ordering.ASC, + scanOptionsBuilder.build(), + EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + return seriesScanUtil; + } + + @Test + public void testSkipFile() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 10); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 10; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testCannotSkipFile() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 20); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 20; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } + + @Test + public void testSkipChunk() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 30); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 30; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } + + @Test + public void testCannotSkipChunk() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 40); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 40; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } + + @Test + public void testSkipPage() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 50); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 50; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } + + @Test + public void testCannotSkipPage() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(5, 60); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 60; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } + + @Test + public void testSkipPoint() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(10, 75); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 75; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + expectedTime = 80; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanPredicatePushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanPredicatePushDownTest.java index 9e71b24f7d3..8c1eab17dc5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanPredicatePushDownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanPredicatePushDownTest.java @@ -1,233 +1,233 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -// package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; -// -// import org.apache.iotdb.commons.exception.IllegalPathException; -// import org.apache.iotdb.commons.path.AlignedFullPath; -// import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; -// import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; -// import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; -// import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; -// import org.apache.iotdb.db.utils.EnvironmentUtils; -// -// import org.apache.tsfile.enums.TSDataType; -// import org.apache.tsfile.read.common.block.TsBlock; -// import org.apache.tsfile.read.filter.basic.Filter; -// import org.apache.tsfile.read.filter.factory.FilterFactory; -// import org.apache.tsfile.read.filter.factory.TimeFilterApi; -// import org.apache.tsfile.read.filter.factory.ValueFilterApi; -// import org.apache.tsfile.write.schema.MeasurementSchema; -// import org.junit.Assert; -// import org.junit.Test; -// -// import java.io.IOException; -// import java.util.Arrays; -// import java.util.HashSet; -// -// public class AlignedSeriesScanPredicatePushDownTest extends AbstractAlignedSeriesScanTest { -// -// private AlignedSeriesScanUtil getAlignedSeriesScanUtil( -// Filter globalTimeFilter, Filter pushDownFilter) throws IllegalPathException { -// AlignedFullPath scanPath = -// new AlignedFullPath( -// TEST_DEVICE, -// Arrays.asList("s1", "s2"), -// Arrays.asList( -// new MeasurementSchema("s1", TSDataType.INT32), -// new MeasurementSchema("s2", TSDataType.INT32))); -// -// SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); -// scanOptionsBuilder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); -// scanOptionsBuilder.withGlobalTimeFilter(globalTimeFilter); -// scanOptionsBuilder.withPushDownFilter(pushDownFilter); -// AlignedSeriesScanUtil seriesScanUtil = -// new AlignedSeriesScanUtil( -// scanPath, -// Ordering.ASC, -// scanOptionsBuilder.build(), -// EnvironmentUtils.TEST_QUERY_FI_CONTEXT); -// seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); -// return seriesScanUtil; -// } -// -// @Test -// @SuppressWarnings("squid:S5961") // Suppress "Test methods should not contain too many -// assertions" -// public void testNoFilter() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(null, null); -// -// // File 1 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 1 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 1 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 2 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 2 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 2 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 3 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 3 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // File 3 - Chunk 2 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 3 - Chunk 2 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 4 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 4 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 4 - Chunk 1 - Page 1 (chunk actually) -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // File 4 - Chunk 1 - Page 2 (chunk actually) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // File 4 - Chunk 1 - Page 3 (chunk actually) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // (File 4 - Chunk 2) merge (File 5 - Chunk 1) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // (File 4 - Chunk 2 - Page 1) merge (File 5 - Chunk 1 - Page 1) -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// -// // File 5 - Chunk 1 - Page 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipWithFilter() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = -// getAlignedSeriesScanUtil( -// TimeFilterApi.gt(10), -// FilterFactory.and( -// ValueFilterApi.gtEq(0, 20, TSDataType.INT32), -// ValueFilterApi.lt(1, 30, TSDataType.INT32))); -// -// // File 1 skipped -// // File 2 skipped -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 3 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 3 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(20, tsBlock.getTimeByIndex(0)); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // File 3 - Chunk 2 skipped -// // File 4 - Chunk 1 skipped -// // (File 4 - Chunk 2) merge (File 5 - Chunk 1) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// } +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.factory.FilterFactory; +import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.apache.tsfile.read.filter.factory.ValueFilterApi; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; + +public class AlignedSeriesScanPredicatePushDownTest extends AbstractAlignedSeriesScanTest { + + private AlignedSeriesScanUtil getAlignedSeriesScanUtil( + Filter globalTimeFilter, Filter pushDownFilter) throws IllegalPathException { + AlignedFullPath scanPath = + new AlignedFullPath( + TEST_DEVICE, + Arrays.asList("s1", "s2"), + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32))); + + SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); + scanOptionsBuilder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + scanOptionsBuilder.withGlobalTimeFilter(globalTimeFilter); + scanOptionsBuilder.withPushDownFilter(pushDownFilter); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, + Ordering.ASC, + scanOptionsBuilder.build(), + EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + return seriesScanUtil; + } + + @Test + @SuppressWarnings( + "squid:S5961") // Suppress "Test methods should not contain too manya assertions" + public void testNoFilter() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSeriesScanUtil(null, null); + + // File 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 1 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 1 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 2 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 2 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 3 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 3 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // File 3 - Chunk 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 3 - Chunk 2 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 4 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 4 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 4 - Chunk 1 - Page 1 (chunk actually) + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // File 4 - Chunk 1 - Page 2 (chunk actually) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // File 4 - Chunk 1 - Page 3 (chunk actually) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // (File 4 - Chunk 2) merge (File 5 - Chunk 1) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); + + // (File 4 - Chunk 2 - Page 1) merge (File 5 - Chunk 1 - Page 1) + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + + // File 5 - Chunk 1 - Page 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipWithFilter() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = + getAlignedSeriesScanUtil( + TimeFilterApi.gt(10), + FilterFactory.and( + ValueFilterApi.gtEq(0, 20, TSDataType.INT32), + ValueFilterApi.lt(1, 30, TSDataType.INT32))); + + // File 1 skipped + // File 2 skipped + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 3 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 3 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(20, tsBlock.getTimeByIndex(0)); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // File 3 - Chunk 2 skipped + // File 4 - Chunk 1 skipped + // (File 4 - Chunk 2) merge (File 5 - Chunk 1) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSingleColumnSeriesScanLimitOffsetPushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSingleColumnSeriesScanLimitOffsetPushDownTest.java index 159745dc074..44e7b87d903 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSingleColumnSeriesScanLimitOffsetPushDownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSingleColumnSeriesScanLimitOffsetPushDownTest.java @@ -1,201 +1,201 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -// package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; -// -// import org.apache.iotdb.commons.exception.IllegalPathException; -// import org.apache.iotdb.commons.path.AlignedFullPath; -// import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; -// import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; -// import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; -// import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; -// import org.apache.iotdb.db.utils.EnvironmentUtils; -// -// import org.apache.tsfile.enums.TSDataType; -// import org.apache.tsfile.read.common.block.TsBlock; -// import org.apache.tsfile.write.schema.MeasurementSchema; -// import org.junit.Assert; -// import org.junit.Test; -// -// import java.io.IOException; -// import java.util.Collections; -// import java.util.HashSet; -// -// public class AlignedSingleColumnSeriesScanLimitOffsetPushDownTest -// extends AbstractAlignedSeriesScanTest { -// -// private static final int TEST_LIMIT = 5; -// -// private AlignedSeriesScanUtil getAlignedSingleColumnSeriesScanUtil(long offset) -// throws IllegalPathException { -// AlignedFullPath scanPath = -// new AlignedFullPath( -// TEST_DEVICE, -// Collections.singletonList("s1"), -// Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT32))); -// -// SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); -// scanOptionsBuilder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); -// scanOptionsBuilder.withPushDownLimit(TEST_LIMIT); -// scanOptionsBuilder.withPushDownOffset(offset); -// AlignedSeriesScanUtil seriesScanUtil = -// new AlignedSeriesScanUtil( -// scanPath, -// Ordering.ASC, -// scanOptionsBuilder.build(), -// EnvironmentUtils.TEST_QUERY_FI_CONTEXT); -// seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); -// return seriesScanUtil; -// } -// -// @Test -// public void testSkipFile() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSingleColumnSeriesScanUtil(20); -// -// // File 1 skipped -// // File 2 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 24; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// -// @Test -// public void testSkipChunk() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSingleColumnSeriesScanUtil(30); -// -// // File 1 skipped (10 points) -// // File 2 skipped (6 points) -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// // File 3 Chunk 1 skipped (10 points) -// // File 3 Chunk 2 (6 points should skip 4 points) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(2, tsBlock.getPositionCount()); -// long expectedTime = 34; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// // remaining 3 points selected -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(3, tsBlock.getPositionCount()); -// expectedTime = 40; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipPage() throws IllegalPathException, IOException { -// AlignedSeriesScanUtil seriesScanUtil = getAlignedSingleColumnSeriesScanUtil(45); -// -// // File 1 skipped (10 points) -// // File 2 skipped (6 points) -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// -// // File 3 - Chunk 1 skipped (10 points) -// // File 3 - Chunk 2 skipped (6 points) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(0, tsBlock.getPositionCount()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 4 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// -// // File 4 - Chunk 1 - Page 1 skipped (10 points) -// // File 4 - Chunk 1 - Page 2 (6 points should skip 3 points) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(3, tsBlock.getPositionCount()); -// long expectedTime = 53; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // File 4 - Chunk 1 - Page 2 (remaining 2 points) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(2, tsBlock.getPositionCount()); -// expectedTime = 60; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// } -// } +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; + +public class AlignedSingleColumnSeriesScanLimitOffsetPushDownTest + extends AbstractAlignedSeriesScanTest { + + private static final int TEST_LIMIT = 5; + + private AlignedSeriesScanUtil getAlignedSingleColumnSeriesScanUtil(long offset) + throws IllegalPathException { + AlignedFullPath scanPath = + new AlignedFullPath( + TEST_DEVICE, + Collections.singletonList("s1"), + Collections.singletonList(new MeasurementSchema("s1", TSDataType.INT32))); + + SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); + scanOptionsBuilder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + scanOptionsBuilder.withPushDownLimit(TEST_LIMIT); + scanOptionsBuilder.withPushDownOffset(offset); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, + Ordering.ASC, + scanOptionsBuilder.build(), + EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + return seriesScanUtil; + } + + @Test + public void testSkipFile() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSingleColumnSeriesScanUtil(20); + + // File 1 skipped + // File 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 24; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } + + @Test + public void testSkipChunk() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSingleColumnSeriesScanUtil(30); + + // File 1 skipped (10 points) + // File 2 skipped (6 points) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + // File 3 Chunk 1 skipped (10 points) + // File 3 Chunk 2 (6 points should skip 4 points) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(2, tsBlock.getPositionCount()); + long expectedTime = 34; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + // remaining 3 points selected + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(3, tsBlock.getPositionCount()); + expectedTime = 40; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipPage() throws IllegalPathException, IOException { + AlignedSeriesScanUtil seriesScanUtil = getAlignedSingleColumnSeriesScanUtil(45); + + // File 1 skipped (10 points) + // File 2 skipped (6 points) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + + // File 3 - Chunk 1 skipped (10 points) + // File 3 - Chunk 2 skipped (6 points) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(0, tsBlock.getPositionCount()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 4 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + + // File 4 - Chunk 1 - Page 1 skipped (10 points) + // File 4 - Chunk 1 - Page 2 (6 points should skip 3 points) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(3, tsBlock.getPositionCount()); + long expectedTime = 53; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // File 4 - Chunk 1 - Page 2 (remaining 2 points) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(2, tsBlock.getPositionCount()); + expectedTime = 60; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java index 66e96378f09..c6c3bcb698a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesReaderTestUtil.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; @@ -216,4 +217,20 @@ public class SeriesReaderTestUtil { FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders(); } + + static void assertWithHasNext(SeriesScanHasNextSupplier supplier, boolean value) + throws IOException { + while (true) { + Optional<Boolean> b = supplier.get(); + if (!b.isPresent()) { + continue; + } + Assert.assertEquals(b.get(), value); + break; + } + } + + interface SeriesScanHasNextSupplier { + Optional<Boolean> get() throws IOException; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanLimitOffsetPushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanLimitOffsetPushDownTest.java index 14c14b73a51..c44d728640f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanLimitOffsetPushDownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanLimitOffsetPushDownTest.java @@ -1,250 +1,250 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -// package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; -// -// import org.apache.iotdb.commons.exception.IllegalPathException; -// import org.apache.iotdb.commons.path.IFullPath; -// import org.apache.iotdb.commons.path.MeasurementPath; -// import org.apache.iotdb.db.queryengine.execution.operator.source.SeriesScanUtil; -// import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; -// import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; -// import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; -// import org.apache.iotdb.db.utils.EnvironmentUtils; -// -// import org.apache.tsfile.enums.TSDataType; -// import org.apache.tsfile.read.common.block.TsBlock; -// import org.junit.Assert; -// import org.junit.Test; -// -// import java.io.IOException; -// import java.util.Collections; -// -// public class SeriesScanLimitOffsetPushDownTest extends AbstractSeriesScanTest { -// -// private SeriesScanUtil getSeriesScanUtil(long limit, long offset, Ordering scanOrder) -// throws IllegalPathException { -// MeasurementPath scanPath = new MeasurementPath(TEST_PATH, TSDataType.INT32); -// -// SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); -// scanOptionsBuilder.withAllSensors(Collections.singleton(scanPath.getMeasurement())); -// scanOptionsBuilder.withPushDownLimit(limit); -// scanOptionsBuilder.withPushDownOffset(offset); -// SeriesScanUtil seriesScanUtil = -// new SeriesScanUtil( -// IFullPath.convertToIFullPath(scanPath), -// scanOrder, -// scanOptionsBuilder.build(), -// EnvironmentUtils.TEST_QUERY_FI_CONTEXT); -// seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); -// return seriesScanUtil; -// } -// -// @Test -// public void testSkipFile() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(5, 10, Ordering.ASC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 10; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipChunk() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(5, 20, Ordering.ASC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 20; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipPage() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(5, 30, Ordering.ASC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 30; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipPoint1() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 45, Ordering.ASC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 45; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipPoint2() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 55, Ordering.ASC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// long expectedTime = 55; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipPointDesc1() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 5, Ordering.DESC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// -// long expectedTime = 64; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// -// expectedTime = 59; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipPointDesc2() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 25, Ordering.DESC); -// -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// -// long expectedTime = 44; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// -// expectedTime = 39; -// for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { -// Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); -// } -// -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// } +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.IFullPath; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.SeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; + +public class SeriesScanLimitOffsetPushDownTest extends AbstractSeriesScanTest { + + private SeriesScanUtil getSeriesScanUtil(long limit, long offset, Ordering scanOrder) + throws IllegalPathException { + MeasurementPath scanPath = new MeasurementPath(TEST_PATH, TSDataType.INT32); + + SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); + scanOptionsBuilder.withAllSensors(Collections.singleton(scanPath.getMeasurement())); + scanOptionsBuilder.withPushDownLimit(limit); + scanOptionsBuilder.withPushDownOffset(offset); + SeriesScanUtil seriesScanUtil = + new SeriesScanUtil( + IFullPath.convertToIFullPath(scanPath), + scanOrder, + scanOptionsBuilder.build(), + EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + return seriesScanUtil; + } + + @Test + public void testSkipFile() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(5, 10, Ordering.ASC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 10; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipChunk() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(5, 20, Ordering.ASC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 20; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipPage() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(5, 30, Ordering.ASC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 30; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipPoint1() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 45, Ordering.ASC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 45; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipPoint2() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 55, Ordering.ASC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + long expectedTime = 55; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime++, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipPointDesc1() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 5, Ordering.DESC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + + long expectedTime = 64; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); + } + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + + expectedTime = 59; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipPointDesc2() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(10, 25, Ordering.DESC); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + + long expectedTime = 44; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); + } + + Assert.assertTrue(seriesScanUtil.hasNextPage()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + + expectedTime = 39; + for (int i = 0, size = tsBlock.getPositionCount(); i < size; i++) { + Assert.assertEquals(expectedTime--, tsBlock.getTimeByIndex(i)); + } + + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanPredicatePushDownTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanPredicatePushDownTest.java index 4c27244ecf1..4a546d78790 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanPredicatePushDownTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanPredicatePushDownTest.java @@ -1,347 +1,345 @@ -/// * -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -// package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; -// -// import org.apache.iotdb.commons.exception.IllegalPathException; -// import org.apache.iotdb.commons.path.IFullPath; -// import org.apache.iotdb.commons.path.MeasurementPath; -// import org.apache.iotdb.db.queryengine.execution.operator.source.SeriesScanUtil; -// import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; -// import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; -// import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; -// import org.apache.iotdb.db.utils.EnvironmentUtils; -// -// import org.apache.tsfile.enums.TSDataType; -// import org.apache.tsfile.read.common.block.TsBlock; -// import org.apache.tsfile.read.filter.basic.Filter; -// import org.apache.tsfile.read.filter.factory.TimeFilterApi; -// import org.apache.tsfile.read.filter.factory.ValueFilterApi; -// import org.junit.Assert; -// import org.junit.Test; -// -// import java.io.IOException; -// import java.util.Collections; -// -// import static org.apache.tsfile.read.filter.factory.ValueFilterApi.DEFAULT_MEASUREMENT_INDEX; -// -// public class SeriesScanPredicatePushDownTest extends AbstractSeriesScanTest { -// -// private SeriesScanUtil getSeriesScanUtil(Filter globalTimeFilter, Filter pushDownFilter) -// throws IllegalPathException { -// MeasurementPath scanPath = new MeasurementPath(TEST_PATH, TSDataType.INT32); -// -// SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); -// scanOptionsBuilder.withAllSensors(Collections.singleton(scanPath.getMeasurement())); -// scanOptionsBuilder.withGlobalTimeFilter(globalTimeFilter); -// scanOptionsBuilder.withPushDownFilter(pushDownFilter); -// SeriesScanUtil seriesScanUtil = -// new SeriesScanUtil( -// IFullPath.convertToIFullPath(scanPath), -// Ordering.ASC, -// scanOptionsBuilder.build(), -// EnvironmentUtils.TEST_QUERY_FI_CONTEXT); -// seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); -// return seriesScanUtil; -// } -// -// @Test -// @SuppressWarnings("squid:S5961") // Suppress "Test methods should not contain too many -// assertions" -// public void testNoFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(null, null); -// -// // File 1 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 1 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 1 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 2 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 2 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 2 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // File 2 - Chunk 2 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 2 - Chunk 2 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 3 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 3 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// -// // File 3 - Chunk 1 - Page 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// -// // (File 3 - Chunk 2) merge (File 4 - Chunk 1) -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // (File 3 - Chunk 2 - Page 1) merge (File 4 - Chunk 1 - Page 1) -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// -// // File 4 - Chunk 1 - Page 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertFalse(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.hasNextFile()); -// } -// -// @Test -// public void testSkipFileByGlobalTimeFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(10), null); -// checkFile1Skipped(seriesScanUtil); -// } -// -// @Test -// public void testSkipFileByPushDownFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = -// getSeriesScanUtil( -// TimeFilterApi.gt(0), -// ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 10, TSDataType.INT32)); -// checkFile1Skipped(seriesScanUtil); -// } -// -// private void checkFile1Skipped(SeriesScanUtil seriesScanUtil) throws IOException { -// // File 1 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 1 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 1 - Chunk 1 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(10, tsBlock.getTimeByIndex(0)); -// } -// -// @Test -// public void testSkipChunkByGlobalTimeFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(20), null); -// checkFile2Chunk1Skipped(seriesScanUtil); -// } -// -// @Test -// public void testSkipChunkByPushDownFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = -// getSeriesScanUtil( -// TimeFilterApi.gt(0), -// ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 20, TSDataType.INT32)); -// checkFile2Chunk1Skipped(seriesScanUtil); -// } -// -// private void checkFile2Chunk1Skipped(SeriesScanUtil seriesScanUtil) throws IOException { -// // File 1 skipped -// // File 2 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 2 - Chunk 1 skipped -// // File 2 - Chunk 2 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); -// -// // File 2 - Chunk 2 - Page 1 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(20, tsBlock.getTimeByIndex(0)); -// } -// -// @Test -// public void testSkipPageByGlobalTimeFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(40), null); -// checkFile1AndFile2Skipped(seriesScanUtil); -// -// // File 3 - Chunk 1 - Page 1 skipped -// // File 3 - Chunk 1 - Page 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(40, tsBlock.getTimeByIndex(0)); -// } -// -// @Test -// public void testSkipPageByPushDownFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = -// getSeriesScanUtil( -// TimeFilterApi.gt(0), -// ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 40, TSDataType.INT32)); -// checkFile1AndFile2Skipped(seriesScanUtil); -// -// // File 3 - Chunk 1 - Page 1 skipped -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertNull(tsBlock); -// -// // File 3 - Chunk 1 - Page 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(40, tsBlock.getTimeByIndex(0)); -// } -// -// private void checkFile1AndFile2Skipped(SeriesScanUtil seriesScanUtil) throws IOException { -// // File 1 skipped -// // File 2 skipped -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 3 - Chunk 1 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); -// } -// -// @Test -// public void testSkipMergeReaderPointByGlobalTimeFilter() -// throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(55), null); -// checkFile1AndFile2AndMergeReaderPointSkipped(seriesScanUtil); -// } -// -// @Test -// public void testSkipMergeReaderPointByPushDownFilter() throws IllegalPathException, IOException -// { -// SeriesScanUtil seriesScanUtil = -// getSeriesScanUtil( -// TimeFilterApi.gt(0), -// ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 55, TSDataType.INT32)); -// checkFile1AndFile2AndMergeReaderPointSkipped(seriesScanUtil); -// } -// -// private void checkFile1AndFile2AndFile3Chunk1Skipped(SeriesScanUtil seriesScanUtil) -// throws IOException { -// // File 1 skipped -// // File 2 skipped -// // File 3 -// Assert.assertTrue(seriesScanUtil.hasNextFile()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); -// -// // File 3 - Chunk 1 skipped -// // File 3 - Chunk 2 -// Assert.assertTrue(seriesScanUtil.hasNextChunk()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); -// } -// -// private void checkFile1AndFile2AndMergeReaderPointSkipped(SeriesScanUtil seriesScanUtil) -// throws IOException { -// checkFile1AndFile2AndFile3Chunk1Skipped(seriesScanUtil); -// -// // (File 3 - Chunk 2) merge (File 4 - Chunk 1) -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(5, tsBlock.getPositionCount()); -// Assert.assertEquals(55, tsBlock.getTimeByIndex(0)); -// } -// -// @Test -// public void testSkipMergeReaderByGlobalTimeFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(60), null); -// checkFile1AndFile2AndFile3Chunk1Skipped(seriesScanUtil); -// -// // (File 3 - Chunk 1) merge (File 4 - Chunk 1) skipped -// // File 4 - Chunk 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(60, tsBlock.getTimeByIndex(0)); -// } -// -// @Test -// public void testSkipMergeReaderByPushDownFilter() throws IllegalPathException, IOException { -// SeriesScanUtil seriesScanUtil = -// getSeriesScanUtil( -// TimeFilterApi.gt(0), -// ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 60, TSDataType.INT32)); -// -// checkFile1AndFile2AndFile3Chunk1Skipped(seriesScanUtil); -// -// // (File 3 - Chunk 1) merge (File 4 - Chunk 1) -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); -// TsBlock tsBlock = seriesScanUtil.nextPage(); -// Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); -// -// // File 4 - Chunk 2 -// Assert.assertTrue(seriesScanUtil.hasNextPage()); -// Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); -// tsBlock = seriesScanUtil.nextPage(); -// Assert.assertEquals(10, tsBlock.getPositionCount()); -// Assert.assertEquals(60, tsBlock.getTimeByIndex(0)); -// } -// } +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.IFullPath; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.SeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.utils.EnvironmentUtils; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.filter.basic.Filter; +import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.apache.tsfile.read.filter.factory.ValueFilterApi; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collections; + +import static org.apache.tsfile.read.filter.factory.ValueFilterApi.DEFAULT_MEASUREMENT_INDEX; + +public class SeriesScanPredicatePushDownTest extends AbstractSeriesScanTest { + + private SeriesScanUtil getSeriesScanUtil(Filter globalTimeFilter, Filter pushDownFilter) + throws IllegalPathException { + MeasurementPath scanPath = new MeasurementPath(TEST_PATH, TSDataType.INT32); + + SeriesScanOptions.Builder scanOptionsBuilder = new SeriesScanOptions.Builder(); + scanOptionsBuilder.withAllSensors(Collections.singleton(scanPath.getMeasurement())); + scanOptionsBuilder.withGlobalTimeFilter(globalTimeFilter); + scanOptionsBuilder.withPushDownFilter(pushDownFilter); + SeriesScanUtil seriesScanUtil = + new SeriesScanUtil( + IFullPath.convertToIFullPath(scanPath), + Ordering.ASC, + scanOptionsBuilder.build(), + EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource(new QueryDataSource(seqResources, unSeqResources)); + return seriesScanUtil; + } + + @Test + @SuppressWarnings("squid:S5961") // Suppress "Test methods should not contain too many assertions" + public void testNoFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(null, null); + + // File 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 1 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 1 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 2 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 2 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // File 2 - Chunk 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 2 - Chunk 2 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 3 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 3 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + + // File 3 - Chunk 1 - Page 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + + // (File 3 - Chunk 2) merge (File 4 - Chunk 1) + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); + + // (File 3 - Chunk 2 - Page 1) merge (File 4 - Chunk 1 - Page 1) + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + + // File 4 - Chunk 1 - Page 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertFalse(seriesScanUtil.hasNextPage()); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, false); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, false); + } + + @Test + public void testSkipFileByGlobalTimeFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(10), null); + checkFile1Skipped(seriesScanUtil); + } + + @Test + public void testSkipFileByPushDownFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = + getSeriesScanUtil( + TimeFilterApi.gt(0), + ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 10, TSDataType.INT32)); + checkFile1Skipped(seriesScanUtil); + } + + private void checkFile1Skipped(SeriesScanUtil seriesScanUtil) throws IOException { + // File 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 1 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 1 - Chunk 1 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(10, tsBlock.getTimeByIndex(0)); + } + + @Test + public void testSkipChunkByGlobalTimeFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(20), null); + checkFile2Chunk1Skipped(seriesScanUtil); + } + + @Test + public void testSkipChunkByPushDownFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = + getSeriesScanUtil( + TimeFilterApi.gt(0), + ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 20, TSDataType.INT32)); + checkFile2Chunk1Skipped(seriesScanUtil); + } + + private void checkFile2Chunk1Skipped(SeriesScanUtil seriesScanUtil) throws IOException { + // File 1 skipped + // File 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 2 - Chunk 1 skipped + // File 2 - Chunk 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.canUseCurrentChunkStatistics()); + + // File 2 - Chunk 2 - Page 1 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(20, tsBlock.getTimeByIndex(0)); + } + + @Test + public void testSkipPageByGlobalTimeFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(40), null); + checkFile1AndFile2Skipped(seriesScanUtil); + + // File 3 - Chunk 1 - Page 1 skipped + // File 3 - Chunk 1 - Page 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(40, tsBlock.getTimeByIndex(0)); + } + + @Test + public void testSkipPageByPushDownFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = + getSeriesScanUtil( + TimeFilterApi.gt(0), + ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 40, TSDataType.INT32)); + checkFile1AndFile2Skipped(seriesScanUtil); + + // File 3 - Chunk 1 - Page 1 skipped + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertNull(tsBlock); + + // File 3 - Chunk 1 - Page 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(40, tsBlock.getTimeByIndex(0)); + } + + private void checkFile1AndFile2Skipped(SeriesScanUtil seriesScanUtil) throws IOException { + // File 1 skipped + // File 2 skipped + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 3 - Chunk 1 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); + } + + @Test + public void testSkipMergeReaderPointByGlobalTimeFilter() + throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(55), null); + checkFile1AndFile2AndMergeReaderPointSkipped(seriesScanUtil); + } + + @Test + public void testSkipMergeReaderPointByPushDownFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = + getSeriesScanUtil( + TimeFilterApi.gt(0), + ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 55, TSDataType.INT32)); + checkFile1AndFile2AndMergeReaderPointSkipped(seriesScanUtil); + } + + private void checkFile1AndFile2AndFile3Chunk1Skipped(SeriesScanUtil seriesScanUtil) + throws IOException { + // File 1 skipped + // File 2 skipped + // File 3 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentFileStatistics()); + + // File 3 - Chunk 1 skipped + // File 3 - Chunk 2 + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertFalse(seriesScanUtil.canUseCurrentChunkStatistics()); + } + + private void checkFile1AndFile2AndMergeReaderPointSkipped(SeriesScanUtil seriesScanUtil) + throws IOException { + checkFile1AndFile2AndFile3Chunk1Skipped(seriesScanUtil); + + // (File 3 - Chunk 2) merge (File 4 - Chunk 1) + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(5, tsBlock.getPositionCount()); + Assert.assertEquals(55, tsBlock.getTimeByIndex(0)); + } + + @Test + public void testSkipMergeReaderByGlobalTimeFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = getSeriesScanUtil(TimeFilterApi.gtEq(60), null); + checkFile1AndFile2AndFile3Chunk1Skipped(seriesScanUtil); + + // (File 3 - Chunk 1) merge (File 4 - Chunk 1) skipped + // File 4 - Chunk 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(60, tsBlock.getTimeByIndex(0)); + } + + @Test + public void testSkipMergeReaderByPushDownFilter() throws IllegalPathException, IOException { + SeriesScanUtil seriesScanUtil = + getSeriesScanUtil( + TimeFilterApi.gt(0), + ValueFilterApi.gtEq(DEFAULT_MEASUREMENT_INDEX, 60, TSDataType.INT32)); + + checkFile1AndFile2AndFile3Chunk1Skipped(seriesScanUtil); + + // (File 3 - Chunk 1) merge (File 4 - Chunk 1) + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertFalse(seriesScanUtil.canUseCurrentPageStatistics()); + TsBlock tsBlock = seriesScanUtil.nextPage(); + Assert.assertTrue(tsBlock == null || tsBlock.isEmpty()); + + // File 4 - Chunk 2 + Assert.assertTrue(seriesScanUtil.hasNextPage()); + Assert.assertTrue(seriesScanUtil.canUseCurrentPageStatistics()); + tsBlock = seriesScanUtil.nextPage(); + Assert.assertEquals(10, tsBlock.getPositionCount()); + Assert.assertEquals(60, tsBlock.getTimeByIndex(0)); + } +}
