JingsongLi commented on code in PR #1469:
URL: https://github.com/apache/paimon-rust/pull/1469#discussion_r4643614687
##########
crates/paimon/src/arrow/format/parquet.rs:
##########
@@ -2807,51 +2827,41 @@ fn build_predicate_page_selection(
});
}
- Ok(combined)
-}
-
-/// Collect the file-field indices referenced by leaf predicates (recursing
-/// through compound nodes). Indices are already file-level (see caller).
-fn collect_leaf_field_indices(predicate: &Predicate, out: &mut Vec<usize>) {
- match predicate {
- Predicate::NestedLeaf { index, .. } => out.push(*index),
- Predicate::Leaf { index, .. } => out.push(*index),
- Predicate::And(children) | Predicate::Or(children) => {
- for child in children {
- collect_leaf_field_indices(child, out);
- }
- }
- Predicate::Not(child) => collect_leaf_field_indices(child, out),
- Predicate::AlwaysTrue | Predicate::AlwaysFalse => {}
- }
+ combined
}
-/// Validate that an OffsetIndex's page row boundaries are well-formed for a
row
-/// group of `rg_rows` rows: the first page starts at row 0, `first_row_index`
-/// is strictly increasing, and every value stays within `[0, rg_rows]`.
-///
-/// Malformed metadata (negative, non-monotonic, or out-of-range boundaries)
-/// would otherwise produce invalid row ranges — huge values from `i64 as
usize`
-/// underflow, or ranges past the row group — that panic
-/// `RowSelection::from_consecutive_ranges`. Callers fail open when this
returns
-/// `false`.
-fn page_boundaries_valid(
+fn page_locations_valid(
pages: &[parquet::file::page_index::offset_index::PageLocation],
- rg_rows: usize,
+ row_count: i64,
+ column: &ColumnChunkMetaData,
+ file_size: u64,
) -> bool {
- let rg_rows = rg_rows as i64;
- let mut prev: i64 = -1;
- for (page_idx, page) in pages.iter().enumerate() {
- let first_row = page.first_row_index;
- if page_idx == 0 && first_row != 0 {
- return false;
- }
- if first_row <= prev || first_row > rg_rows {
- return false;
- }
- prev = first_row;
+ let Some(first) = pages.first() else {
+ return row_count == 0;
+ };
+ let data_start = column.data_page_offset();
+ let chunk_start = column.dictionary_page_offset().unwrap_or(data_start);
+ let Some(chunk_end) = chunk_start.checked_add(column.compressed_size())
else {
+ return false;
+ };
+ if !(0..=data_start).contains(&chunk_start)
+ || !u64::try_from(chunk_end).is_ok_and(|end| end <= file_size)
+ || first.first_row_index != 0
+ || first.offset != data_start
+ {
+ return false;
}
- true
+ pages.iter().enumerate().all(|(index, page)| {
+ let (next_row, next_byte) = pages.get(index + 1).map_or((row_count,
chunk_end), |next| {
+ (next.first_row_index, next.offset)
+ });
+ page.first_row_index < next_row
+ && page.compressed_page_size > 0
+ && page
+ .offset
+ .checked_add(i64::from(page.compressed_page_size))
+ .is_some_and(|end| end <= next_byte)
Review Comment:
An undersized `compressed_page_size` still passes this check. I reproduced
this by reducing the last page's recorded size by one byte in the projected
column's OffsetIndex while leaving the data page intact. With merged-read stats
pruning disabled, `rating = 4095` returns the expected row. With pruning
enabled, Arrow reads the shortened range and fails with `EOF: Not enough bytes
to decode`.
Could we reject unexpected gaps between pages in a contiguous column chunk,
drop the optional OffsetIndex, and fall back to row-group reads? The
damaged-file test currently covers an oversized entry. Please add the
undersized case as well.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]