JingsongLi commented on code in PR #6060:
URL: https://github.com/apache/paimon/pull/6060#discussion_r2282012074
##########
paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java:
##########
@@ -792,23 +796,67 @@ public ColumnIndexStore getColumnIndexStore(int
blockIndex) {
}
private RowRanges getRowRanges(int blockIndex) {
- assert FilterCompat.isFilteringRequired(options.getRecordFilter())
- : "Should not be invoked if filter is null or NOOP";
+ boolean filteringRequired =
FilterCompat.isFilteringRequired(options.getRecordFilter());
+ if (!filteringRequired && selection == null) {
+ throw new IllegalArgumentException("Should not be invoked if
filter is null or NOOP");
+ }
+
RowRanges rowRanges = blockRowRanges.get(blockIndex);
if (rowRanges == null) {
- rowRanges =
- ColumnIndexFilter.calculateRowRanges(
- options.getRecordFilter(),
- getColumnIndexStore(blockIndex),
- paths.keySet(),
- blocks.get(blockIndex).getRowCount(),
- blocks.get(blockIndex).getRowIndexOffset(),
- selection);
+ BlockMetaData block = blocks.get(blockIndex);
+ rowRanges = RowRanges.createSingle(block.getRowCount());
+ if (selection != null) {
+ RowRanges result = calculateRowRanges(blockIndex, selection);
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
+
+ if (filteringRequired) {
+ RowRanges result =
+ ColumnIndexFilter.calculateRowRanges(
+ options.getRecordFilter(),
+ getColumnIndexStore(blockIndex),
+ paths.keySet(),
+ block.getRowCount());
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
blockRowRanges.set(blockIndex, rowRanges);
}
return rowRanges;
}
+ private RowRanges calculateRowRanges(int blockIndex, RoaringBitmap32
selection) {
+ List<OffsetIndex> offsets;
+ BlockMetaData block = blocks.get(blockIndex);
+ if (paths.isEmpty()) {
Review Comment:
`SELECT COUNT(*) FROM T LIMIT 10;` will just use pushAggregation and just
read manifests?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@paimon.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
