This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch IOTDB-6115
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/IOTDB-6115 by this push:
new 387f8eaa94b opt
387f8eaa94b is described below
commit 387f8eaa94bd26ac3b62cec22bf809291938efc3
Author: JackieTien97 <[email protected]>
AuthorDate: Wed Aug 16 20:00:29 2023 +0800
opt
---
.../operator/source/AlignedSeriesScanUtil.java | 4 ++--
.../memtable/AlignedReadOnlyMemChunk.java | 12 +++++++++++-
.../read/reader/chunk/DiskAlignedChunkLoader.java | 13 +++++++++++--
.../read/reader/chunk/MemAlignedChunkLoader.java | 11 +++++++++--
.../read/reader/chunk/MemAlignedChunkReader.java | 8 +++++---
.../read/reader/chunk/MemAlignedPageReader.java | 22 ++++++++++++++++------
.../metadata/DiskAlignedChunkMetadataLoader.java | 16 ++++++++++++++--
.../metadata/MemAlignedChunkMetadataLoader.java | 19 ++++++++++++++++---
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 21 ++++++++++++++++-----
.../reader/chunk/MemAlignedChunkLoaderTest.java | 2 +-
.../read/TsFileAlignedSeriesReaderIterator.java | 2 +-
.../read/reader/chunk/AlignedChunkReader.java | 16 ++++++++++------
.../tsfile/read/reader/page/AlignedPageReader.java | 22 ++++++++++++++++------
.../read/reader/series/FileSeriesReader.java | 2 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 3 ++-
15 files changed, 131 insertions(+), 42 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
index 7e20a1e4e82..300412081f4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AlignedSeriesScanUtil.java
@@ -53,7 +53,7 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil {
// device, we
// can use statistics to skip.
// it's only exact while using limit & offset push down
- private boolean queryAllSensors = false;
+ private final boolean queryAllSensors;
public AlignedSeriesScanUtil(
PartialPath seriesPath,
@@ -138,7 +138,7 @@ public class AlignedSeriesScanUtil extends SeriesScanUtil {
Set<String> allSensors)
throws IOException {
return FileLoaderUtils.loadTimeSeriesMetadata(
- resource, (AlignedPath) seriesPath, context, filter);
+ resource, (AlignedPath) seriesPath, context, filter, queryAllSensors);
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
index 04d9a2b367b..cf899441d49 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java
@@ -45,6 +45,12 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
private final List<TSDataType> dataTypes;
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private boolean queryAllSensors;
+
/**
* The constructor for Aligned type.
*
@@ -146,7 +152,7 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
}
IChunkMetadata alignedChunkMetadata =
new AlignedChunkMetadata(timeChunkMetadata, valueChunkMetadataList);
- alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(this));
+ alignedChunkMetadata.setChunkLoader(new MemAlignedChunkLoader(this,
queryAllSensors));
alignedChunkMetadata.setVersion(Long.MAX_VALUE);
cachedMetaData = alignedChunkMetadata;
}
@@ -160,4 +166,8 @@ public class AlignedReadOnlyMemChunk extends
ReadOnlyMemChunk {
public IPointReader getPointReader() {
return tsBlock.getTsBlockAlignedRowIterator();
}
+
+ public void setQueryAllSensors(boolean queryAllSensors) {
+ this.queryAllSensors = queryAllSensors;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
index 18907c22b3e..5b0165b2f7c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java
@@ -40,11 +40,19 @@ import static
org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.INI
public class DiskAlignedChunkLoader implements IChunkLoader {
private final boolean debug;
+
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private final boolean queryAllSensors;
+
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
- public DiskAlignedChunkLoader(boolean debug) {
+ public DiskAlignedChunkLoader(boolean debug, boolean queryAllSensors) {
this.debug = debug;
+ this.queryAllSensors = queryAllSensors;
}
@Override
@@ -75,7 +83,8 @@ public class DiskAlignedChunkLoader implements IChunkLoader {
}
long t2 = System.nanoTime();
- IChunkReader chunkReader = new AlignedChunkReader(timeChunk,
valueChunkList, timeFilter);
+ IChunkReader chunkReader =
+ new AlignedChunkReader(timeChunk, valueChunkList, timeFilter,
queryAllSensors);
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
INIT_CHUNK_READER_ALIGNED_DISK, System.nanoTime() - t2);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java
index a93a25bb7b1..63feb2220f1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoader.java
@@ -36,11 +36,18 @@ public class MemAlignedChunkLoader implements IChunkLoader {
private final AlignedReadOnlyMemChunk chunk;
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private final boolean queryAllSensors;
+
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
- public MemAlignedChunkLoader(AlignedReadOnlyMemChunk chunk) {
+ public MemAlignedChunkLoader(AlignedReadOnlyMemChunk chunk, boolean
queryAllSensors) {
this.chunk = chunk;
+ this.queryAllSensors = queryAllSensors;
}
@Override
@@ -57,7 +64,7 @@ public class MemAlignedChunkLoader implements IChunkLoader {
public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter
timeFilter) {
long startTime = System.nanoTime();
try {
- return new MemAlignedChunkReader(chunk, timeFilter);
+ return new MemAlignedChunkReader(chunk, timeFilter, queryAllSensors);
} finally {
long duration = System.nanoTime() - startTime;
SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
index b7ff4cc89ed..66f36fea0b2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkReader.java
@@ -33,16 +33,18 @@ import java.util.List;
/** To read aligned chunk data in memory. */
public class MemAlignedChunkReader implements IChunkReader {
- private List<IPageReader> pageReaderList;
+ private final List<IPageReader> pageReaderList;
- public MemAlignedChunkReader(AlignedReadOnlyMemChunk readableChunk, Filter
filter) {
+ public MemAlignedChunkReader(
+ AlignedReadOnlyMemChunk readableChunk, Filter filter, boolean
queryAllSensors) {
// we treat one ReadOnlyMemChunk as one Page
this.pageReaderList =
Collections.singletonList(
new MemAlignedPageReader(
readableChunk.getTsBlock(),
(AlignedChunkMetadata) readableChunk.getChunkMetaData(),
- filter));
+ filter,
+ queryAllSensors));
}
@Override
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
index 2f08be13c84..3f305da610d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java
@@ -46,15 +46,23 @@ public class MemAlignedPageReader implements IPageReader,
IAlignedPageReader {
private final TsBlock tsBlock;
private final AlignedChunkMetadata chunkMetadata;
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private final boolean queryAllSensors;
+
private Filter valueFilter;
private PaginationController paginationController =
UNLIMITED_PAGINATION_CONTROLLER;
private TsBlockBuilder builder;
- public MemAlignedPageReader(TsBlock tsBlock, AlignedChunkMetadata
chunkMetadata, Filter filter) {
+ public MemAlignedPageReader(
+ TsBlock tsBlock, AlignedChunkMetadata chunkMetadata, Filter filter,
boolean queryAllSensors) {
this.tsBlock = tsBlock;
this.chunkMetadata = chunkMetadata;
this.valueFilter = filter;
+ this.queryAllSensors = queryAllSensors;
}
@Override
@@ -107,11 +115,13 @@ public class MemAlignedPageReader implements IPageReader,
IAlignedPageReader {
// NOTE: if we change the query semantic in the future for aligned
series, we need to remove
// this check here.
long rowCount = getTimeStatistics().getCount();
- boolean canUse = getValueStatisticsList().isEmpty();
- for (Statistics<? extends Serializable> vStatistics :
getValueStatisticsList()) {
- if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
- canUse = true;
- break;
+ boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty();
+ if (!canUse) {
+ for (Statistics<? extends Serializable> vStatistics :
getValueStatisticsList()) {
+ if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
+ canUse = true;
+ break;
+ }
}
}
if (!canUse) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
index 96326fb9443..95dbfd6e4e2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/DiskAlignedChunkMetadataLoader.java
@@ -51,16 +51,27 @@ public class DiskAlignedChunkMetadataLoader implements
IChunkMetadataLoader {
// time filter or value filter, only used to check time range
private final Filter filter;
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private final boolean queryAllSensors;
+
private static final Logger DEBUG_LOGGER =
LoggerFactory.getLogger("QUERY_DEBUG");
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
public DiskAlignedChunkMetadataLoader(
- TsFileResource resource, AlignedPath seriesPath, QueryContext context,
Filter filter) {
+ TsFileResource resource,
+ AlignedPath seriesPath,
+ QueryContext context,
+ Filter filter,
+ boolean queryAllSensors) {
this.resource = resource;
this.seriesPath = seriesPath;
this.context = context;
this.filter = filter;
+ this.queryAllSensors = queryAllSensors;
}
@Override
@@ -111,7 +122,8 @@ public class DiskAlignedChunkMetadataLoader implements
IChunkMetadataLoader {
if (chunkMetadata.needSetChunkLoader()) {
chunkMetadata.setFilePath(resource.getTsFilePath());
chunkMetadata.setClosed(resource.isClosed());
- chunkMetadata.setChunkLoader(new
DiskAlignedChunkLoader(context.isDebug()));
+ chunkMetadata.setChunkLoader(
+ new DiskAlignedChunkLoader(context.isDebug(),
queryAllSensors));
}
});
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
index 90bbc3136ad..17bd551c7ab 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/metadata/MemAlignedChunkMetadataLoader.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.metadata;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
+import
org.apache.iotdb.db.storageengine.dataregion.memtable.AlignedReadOnlyMemChunk;
import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
import
org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -41,16 +42,26 @@ public class MemAlignedChunkMetadataLoader implements
IChunkMetadataLoader {
private final PartialPath seriesPath;
private final QueryContext context;
private final Filter timeFilter;
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private final boolean queryAllSensors;
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
public MemAlignedChunkMetadataLoader(
- TsFileResource resource, PartialPath seriesPath, QueryContext context,
Filter timeFilter) {
+ TsFileResource resource,
+ PartialPath seriesPath,
+ QueryContext context,
+ Filter timeFilter,
+ boolean queryAllSensors) {
this.resource = resource;
this.seriesPath = seriesPath;
this.context = context;
this.timeFilter = timeFilter;
+ this.queryAllSensors = queryAllSensors;
}
@Override
@@ -66,7 +77,8 @@ public class MemAlignedChunkMetadataLoader implements
IChunkMetadataLoader {
if (chunkMetadata.needSetChunkLoader()) {
chunkMetadata.setFilePath(resource.getTsFilePath());
chunkMetadata.setClosed(resource.isClosed());
- chunkMetadata.setChunkLoader(new
DiskAlignedChunkLoader(context.isDebug()));
+ chunkMetadata.setChunkLoader(
+ new DiskAlignedChunkLoader(context.isDebug(),
queryAllSensors));
}
});
@@ -76,7 +88,8 @@ public class MemAlignedChunkMetadataLoader implements
IChunkMetadataLoader {
List<ReadOnlyMemChunk> memChunks =
resource.getReadOnlyMemChunk(seriesPath);
if (memChunks != null) {
for (ReadOnlyMemChunk readOnlyMemChunk : memChunks) {
- if (!memChunks.isEmpty()) {
+ if (!readOnlyMemChunk.isEmpty()) {
+ ((AlignedReadOnlyMemChunk)
readOnlyMemChunk).setQueryAllSensors(queryAllSensors);
chunkMetadataList.add(readOnlyMemChunk.getChunkMetaData());
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index 2f243b2b508..55d9f04e30e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -201,7 +201,11 @@ public class FileLoaderUtils {
* @throws IOException IOException may be thrown while reading it from disk.
*/
public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
- TsFileResource resource, AlignedPath alignedPath, QueryContext context,
Filter filter)
+ TsFileResource resource,
+ AlignedPath alignedPath,
+ QueryContext context,
+ Filter filter,
+ boolean queryAllSensors)
throws IOException {
final long t1 = System.nanoTime();
boolean loadFromMem = false;
@@ -209,14 +213,16 @@ public class FileLoaderUtils {
AlignedTimeSeriesMetadata alignedTimeSeriesMetadata;
// If the tsfile is closed, we need to load from tsfile
if (resource.isClosed()) {
- alignedTimeSeriesMetadata = loadFromDisk(resource, alignedPath,
context, filter);
+ alignedTimeSeriesMetadata =
+ loadFromDisk(resource, alignedPath, context, filter,
queryAllSensors);
} else { // if the tsfile is unclosed, we just get it directly from
TsFileResource
loadFromMem = true;
alignedTimeSeriesMetadata =
(AlignedTimeSeriesMetadata)
resource.getTimeSeriesMetadata(alignedPath);
if (alignedTimeSeriesMetadata != null) {
alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new MemAlignedChunkMetadataLoader(resource, alignedPath,
context, filter));
+ new MemAlignedChunkMetadataLoader(
+ resource, alignedPath, context, filter, queryAllSensors));
}
}
@@ -252,7 +258,11 @@ public class FileLoaderUtils {
}
private static AlignedTimeSeriesMetadata loadFromDisk(
- TsFileResource resource, AlignedPath alignedPath, QueryContext context,
Filter filter)
+ TsFileResource resource,
+ AlignedPath alignedPath,
+ QueryContext context,
+ Filter filter,
+ boolean queryAllSensors)
throws IOException {
AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
// load all the TimeseriesMetadata of vector, the first one is for time
column and the
@@ -293,7 +303,8 @@ public class FileLoaderUtils {
alignedTimeSeriesMetadata =
new AlignedTimeSeriesMetadata(timeColumn,
valueTimeSeriesMetadataList);
alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new DiskAlignedChunkMetadataLoader(resource, alignedPath, context,
filter));
+ new DiskAlignedChunkMetadataLoader(
+ resource, alignedPath, context, filter, queryAllSensors));
}
}
return alignedTimeSeriesMetadata;
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
index e6f4b4475e6..8e1ba4f1bbf 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedChunkLoaderTest.java
@@ -52,7 +52,7 @@ public class MemAlignedChunkLoaderTest {
AlignedReadOnlyMemChunk chunk =
Mockito.mock(AlignedReadOnlyMemChunk.class);
ChunkMetadata chunkMetadata = Mockito.mock(ChunkMetadata.class);
- MemAlignedChunkLoader memAlignedChunkLoader = new
MemAlignedChunkLoader(chunk);
+ MemAlignedChunkLoader memAlignedChunkLoader = new
MemAlignedChunkLoader(chunk, false);
try {
memAlignedChunkLoader.loadChunk(chunkMetadata);
fail();
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileAlignedSeriesReaderIterator.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileAlignedSeriesReaderIterator.java
index 2113a523fcd..f150bfd19e4 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileAlignedSeriesReaderIterator.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileAlignedSeriesReaderIterator.java
@@ -82,7 +82,7 @@ public class TsFileAlignedSeriesReaderIterator {
}
AlignedChunkReader chunkReader =
- new AlignedChunkReader(timeChunk, Arrays.asList(valueChunks), null);
+ new AlignedChunkReader(timeChunk, Arrays.asList(valueChunks), null,
false);
return new NextAlignedChunkInfo(chunkReader, totalSize, notNullChunkNum,
totalPointNum);
}
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java
index 86bd53f5616..7a1074562e9 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AlignedChunkReader.java
@@ -92,7 +92,8 @@ public class AlignedChunkReader implements IChunkReader {
*
* @param filter filter
*/
- public AlignedChunkReader(Chunk timeChunk, List<Chunk> valueChunkList,
Filter filter)
+ public AlignedChunkReader(
+ Chunk timeChunk, List<Chunk> valueChunkList, Filter filter, boolean
queryAllSensors)
throws IOException {
this.filter = filter;
this.timeChunkDataBuffer = timeChunk.getData();
@@ -108,7 +109,7 @@ public class AlignedChunkReader implements IChunkReader {
valueChunkStatisticsList.add(chunk == null ? null :
chunk.getChunkStatistic());
valueDeleteIntervalList.add(chunk == null ? null :
chunk.getDeleteIntervalList());
});
- initAllPageReaders(timeChunk.getChunkStatistic(),
valueChunkStatisticsList);
+ initAllPageReaders(timeChunk.getChunkStatistic(),
valueChunkStatisticsList, queryAllSensors);
}
/**
@@ -132,12 +133,14 @@ public class AlignedChunkReader implements IChunkReader {
valueChunkStatisticsList.add(chunk == null ? null :
chunk.getChunkStatistic());
valueDeleteIntervalList.add(chunk == null ? null :
chunk.getDeleteIntervalList());
});
- initAllPageReaders(timeChunk.getChunkStatistic(),
valueChunkStatisticsList);
+ initAllPageReaders(timeChunk.getChunkStatistic(),
valueChunkStatisticsList, false);
}
/** construct all the page readers in this chunk */
private void initAllPageReaders(
- Statistics timeChunkStatistics, List<Statistics>
valueChunkStatisticsList)
+ Statistics timeChunkStatistics,
+ List<Statistics> valueChunkStatisticsList,
+ boolean queryAllSensors)
throws IOException {
// construct next satisfied page header
while (timeChunkDataBuffer.remaining() > 0) {
@@ -176,7 +179,7 @@ public class AlignedChunkReader implements IChunkReader {
// if the current page satisfies
if (exits && timePageSatisfied(timePageHeader)) {
AlignedPageReader alignedPageReader =
- constructPageReaderForNextPage(timePageHeader,
valuePageHeaderList);
+ constructPageReaderForNextPage(timePageHeader,
valuePageHeaderList, queryAllSensors);
if (alignedPageReader != null) {
pageReaderList.add(alignedPageReader);
}
@@ -213,7 +216,8 @@ public class AlignedChunkReader implements IChunkReader {
}
private AlignedPageReader constructPageReaderForNextPage(
- PageHeader timePageHeader, List<PageHeader> valuePageHeader) throws
IOException {
+ PageHeader timePageHeader, List<PageHeader> valuePageHeader, boolean
queryAllSensors)
+ throws IOException {
PageInfo timePageInfo = new PageInfo();
getPageInfo(timePageHeader, timeChunkDataBuffer, timeChunkHeader,
timePageInfo);
PageInfo valuePageInfo = new PageInfo();
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
index f0e7b497acb..8aaa3484f83 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/page/AlignedPageReader.java
@@ -50,6 +50,12 @@ public class AlignedPageReader implements IPageReader,
IAlignedPageReader {
private final List<ValuePageReader> valuePageReaderList;
private final int valueCount;
+ // only used for limit and offset push down optimizer, if we select all
columns from aligned
+ // device, we
+ // can use statistics to skip.
+ // it's only exact while using limit & offset push down
+ private final boolean queryAllSensors;
+
private Filter filter;
private PaginationController paginationController =
UNLIMITED_PAGINATION_CONTROLLER;
@@ -67,7 +73,8 @@ public class AlignedPageReader implements IPageReader,
IAlignedPageReader {
List<ByteBuffer> valuePageDataList,
List<TSDataType> valueDataTypeList,
List<Decoder> valueDecoderList,
- Filter filter) {
+ Filter filter,
+ boolean queryAllSensors) {
timePageReader = new TimePageReader(timePageHeader, timePageData,
timeDecoder);
isModified = timePageReader.isModified();
valuePageReaderList = new ArrayList<>(valuePageHeaderList.size());
@@ -87,6 +94,7 @@ public class AlignedPageReader implements IPageReader,
IAlignedPageReader {
}
this.filter = filter;
this.valueCount = valuePageReaderList.size();
+ this.queryAllSensors = queryAllSensors;
}
@Override
@@ -126,11 +134,13 @@ public class AlignedPageReader implements IPageReader,
IAlignedPageReader {
// NOTE: if we change the query semantic in the future for aligned
series, we need to remove
// this check here.
long rowCount = getTimeStatistics().getCount();
- boolean canUse = getValueStatisticsList().isEmpty();
- for (Statistics vStatistics : getValueStatisticsList()) {
- if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
- canUse = true;
- break;
+ boolean canUse = queryAllSensors || getValueStatisticsList().isEmpty();
+ if (!canUse) {
+ for (Statistics vStatistics : getValueStatisticsList()) {
+ if (vStatistics != null && !vStatistics.hasNullValue(rowCount)) {
+ canUse = true;
+ break;
+ }
}
}
if (!canUse) {
diff --git
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java
index 222cc59a5c9..2e1e5090bd8 100644
---
a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java
+++
b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/series/FileSeriesReader.java
@@ -56,7 +56,7 @@ public class FileSeriesReader extends
AbstractFileSeriesReader {
for (IChunkMetadata metadata :
alignedChunkMetadata.getValueChunkMetadataList()) {
valueChunkList.add(chunkLoader.loadChunk((ChunkMetadata) metadata));
}
- this.chunkReader = new AlignedChunkReader(timeChunk, valueChunkList,
filter);
+ this.chunkReader = new AlignedChunkReader(timeChunk, valueChunkList,
filter, false);
}
}
diff --git
a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIntegrityCheckingTool.java
b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIntegrityCheckingTool.java
index be55f4a0449..c753508b8f0 100644
---
a/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIntegrityCheckingTool.java
+++
b/iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileIntegrityCheckingTool.java
@@ -224,7 +224,8 @@ public class TsFileIntegrityCheckingTool {
reader.readMemChunk((ChunkMetadata)
valueChunkMetadataList.get(chunkIdx));
// construct an aligned chunk reader using time chunk and value
chunk
IChunkReader chunkReader =
- new AlignedChunkReader(timeChunk,
Collections.singletonList(valueChunk), null);
+ new AlignedChunkReader(
+ timeChunk, Collections.singletonList(valueChunk), null,
false);
// verify the values
List<Pair<Long, TsPrimitiveType>> originValue =
originDataChunks.get(chunkIdx);
for (int valIdx = 0; chunkReader.hasNextSatisfiedPage(); ) {
