Tan-JiaLiang commented on code in PR #6060:
URL: https://github.com/apache/paimon/pull/6060#discussion_r2281867165
##########
paimon-format/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java:
##########
@@ -792,23 +796,67 @@ public ColumnIndexStore getColumnIndexStore(int
blockIndex) {
}
private RowRanges getRowRanges(int blockIndex) {
- assert FilterCompat.isFilteringRequired(options.getRecordFilter())
- : "Should not be invoked if filter is null or NOOP";
+ boolean filteringRequired =
FilterCompat.isFilteringRequired(options.getRecordFilter());
+ if (!filteringRequired && selection == null) {
+ throw new IllegalArgumentException("Should not be invoked if
filter is null or NOOP");
+ }
+
RowRanges rowRanges = blockRowRanges.get(blockIndex);
if (rowRanges == null) {
- rowRanges =
- ColumnIndexFilter.calculateRowRanges(
- options.getRecordFilter(),
- getColumnIndexStore(blockIndex),
- paths.keySet(),
- blocks.get(blockIndex).getRowCount(),
- blocks.get(blockIndex).getRowIndexOffset(),
- selection);
+ BlockMetaData block = blocks.get(blockIndex);
+ rowRanges = RowRanges.createSingle(block.getRowCount());
+ if (selection != null) {
+ RowRanges result = calculateRowRanges(blockIndex, selection);
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
+
+ if (filteringRequired) {
+ RowRanges result =
+ ColumnIndexFilter.calculateRowRanges(
+ options.getRecordFilter(),
+ getColumnIndexStore(blockIndex),
+ paths.keySet(),
+ block.getRowCount());
+ rowRanges = RowRanges.intersection(result, rowRanges);
+ }
blockRowRanges.set(blockIndex, rowRanges);
}
return rowRanges;
}
+ private RowRanges calculateRowRanges(int blockIndex, RoaringBitmap32
selection) {
+ List<OffsetIndex> offsets;
+ BlockMetaData block = blocks.get(blockIndex);
+ if (paths.isEmpty()) {
+ Optional<ColumnChunkMetaData> first =
block.getColumns().stream().findFirst();
+ if (first.isPresent()) {
+ ColumnPath path = first.get().getPath();
+ OffsetIndex index =
+ ColumnIndexStoreImpl.create(this, block,
Collections.singleton(path))
+ .getOffsetIndex(path);
+ offsets = Collections.singletonList(index);
+ } else {
+ offsets = Collections.emptyList();
+ }
+ } else {
+ ColumnIndexStore store = getColumnIndexStore(blockIndex);
+ offsets =
+
paths.keySet().stream().map(store::getOffsetIndex).collect(Collectors.toList());
+ }
+
+ long rowCount = block.getRowCount();
+ long rowIndexOffset = block.getRowIndexOffset();
+ RowRanges rowRanges = RowRanges.createSingle(rowCount);
+ for (OffsetIndex offset : offsets) {
+ if (offset != null) {
+ RowRanges result = RowRanges.create(rowCount, rowIndexOffset,
offset, selection);
Review Comment:
Yes, If using the selection directly, too much rowranges fragment.
e.g. selection: [1,3,5,7,9] ==> rowranges: [[1,1], [3,3], [5,5], [7,7],
[9,9]]
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@paimon.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
